From 77b6b63b99e4ae258686aa11a6622c9e3607bec5 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 10:33:56 +0100 Subject: [PATCH 01/71] rollout WIP --- documentation/configs/infer.md | 5 + ...nforcement-learning-implementation-plan.md | 477 ++++++++++++++++++ documentation/rollout.md | 184 +++++++ src/sequifier/config/infer_config.py | 66 ++- src/sequifier/config/rollout_config.py | 68 +++ src/sequifier/consolidated-docs.md | 5 + src/sequifier/infer.py | 134 ++++- src/sequifier/rollout/__init__.py | 1 + src/sequifier/rollout/contracts.py | 42 ++ src/sequifier/rollout/coordinator.py | 166 ++++++ src/sequifier/rollout/dependent.py | 50 ++ src/sequifier/rollout/factories.py | 29 ++ src/sequifier/rollout/history.py | 44 ++ src/sequifier/rollout/preprocessing.py | 114 +++++ src/sequifier/rollout_api.py | 33 ++ src/sequifier/trajectory/__init__.py | 6 + src/sequifier/trajectory/schema.py | 147 ++++++ src/sequifier/trajectory/store.py | 33 ++ 18 files changed, 1597 insertions(+), 7 deletions(-) create mode 100644 documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md create mode 100644 documentation/rollout.md create mode 100644 src/sequifier/config/rollout_config.py create mode 100644 src/sequifier/rollout/__init__.py create mode 100644 src/sequifier/rollout/contracts.py create mode 100644 src/sequifier/rollout/coordinator.py create mode 100644 src/sequifier/rollout/dependent.py create mode 100644 src/sequifier/rollout/factories.py create mode 100644 src/sequifier/rollout/history.py create mode 100644 src/sequifier/rollout/preprocessing.py create mode 100644 src/sequifier/rollout_api.py create mode 100644 src/sequifier/trajectory/__init__.py create mode 100644 src/sequifier/trajectory/schema.py create mode 100644 src/sequifier/trajectory/store.py diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index 408c336f..9f333c1d 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -1,5 +1,10 @@ # Infer Command Guide +For CPU numeric inference with externally calculated next-step inputs, see +[local forecast rollouts](../rollout.md). An explicit `rollout` configuration +selects a single seed window and saves a versioned trajectory through the +existing inference command. + Models with `static_conditioning` read the processed static feature table named in the preprocessing metadata. Each inference window receives the configured static values for its `sequenceId`. diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md new file mode 100644 index 00000000..069b00cd --- /dev/null +++ b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md @@ -0,0 +1,477 @@ +# Rollouts, return-conditioned policies, IQL, and PPO + +Status: implementation specification. Date: 2026-11-07. + +First implementation slice: opt-in CPU numeric forecast collection with explicit +bindings, a persistent local single-environment adapter, shared prediction/step +ordering, seed history and fixed preprocessing, and validated JSON trajectories. +See [local forecast rollouts](../rollout.md) for configuration and public APIs. +The broader stages below remain planned. No tests or acceptance fixtures were +written or run for this slice, as requested. + +Deliver this as one coordinated update, built and verified in the three dependency-ordered stages below. Completion means all three stages work together; stage boundaries are engineering checkpoints, not separate proposed releases. This document specifies new behavior and proposed API names, not functionality already present. + +## 1. Outcome and scope + +The update must enable: + +1. Autoregressive inference when some next-step inputs come from predictions and others come from external dependent-variable calculations or simulation. Support scalar features, categorical features, image views, static conditioning, and depth inputs through the existing execution schema. +2. A persistent, batched environment interface shared by inference, training-time rollout evaluation, trajectory collection, and PPO. A separate optional reward provider can supply rewards from external software. +3. Versioned trajectories that can be collected online, imported from existing data, evaluated, reward-labeled, and consumed by offline learners without temporal leakage. +4. Return-conditioned sequence modeling (RCSM), complete offline Implicit Q-Learning (IQL), and full synchronous clipped PPO with generalized advantage estimation (GAE). +5. One training plan that can run supervised/return-conditioned pretraining, offline IQL, and online PPO phases, with explicit state transfer and optimizer lifecycle rules. +6. PyTorch learning and PT/ONNX policy deployment, including consistent preprocessing and action distributions. Training algorithms and external environments remain outside the exported model graph. + +Supported RL actions: ordinary categorical variables, bounded and unbounded continuous vectors, and joint policies combining these. Include independent categorical heads and explicitly ordered autoregressive categorical heads. All learnable action components have a joint likelihood. Existing ordinary regression heads remain available to supervised modeling and dependent-variable inference; RL phases require an explicit probability distribution. + +The release includes single-process CPU/CUDA execution and synchronous DDP learning with separate environment slots per rank. Existing supervised distributed strategies must keep working. New IQL/PPO phases reject FSDP initially with a specific error until multi-network optimizer/checkpoint semantics are implemented for it; do not advertise FSDP RL support through inheritance from supervised training. + +Not part of this update: SAC/TD3/AWAC, online IQL fine-tuning, learned world models, differentiable simulators, asynchronous actors, policy-gradient methods for diffusion/flow policies, constrained/multi-objective RL, dynamic action spaces, multi-agent learning, or hard real-time robot scheduling. Hash-decoded categorical outputs remain supported in existing inference but are rejected as RL actions unless they expose a tested normalized joint probability; candidate ranking scores are insufficient. + +One RL decision executes one action vector and receives one transition. The adapter may hold that action for a configured fixed number of physics substeps and aggregate its reward. Temporal action chunks with intervening replanning or variable-duration decisions need explicit semi-Markov discounts and are deferred. Ordered action dimensions at one instant are supported and are distinct from temporal chunks. + +## 2. Repository integration points + +Current constraints established by source inspection: + +- `infer.py:get_probs_preds_from_dict` owns a NumPy generation loop, assumes each changing input exists in `preds_for_step`, and rejects future image frames. Extract orchestration rather than adding independent loops to each consumer. +- `config/infer_config.py` separately enforces identical inputs/targets in autoregression and generation-step validators; depth autoregression is rejected elsewhere. Replace these restrictions with source coverage and execution-schema checks for supported rollout paths. Preserve forward-looking, one-step generative constraints where still required. +- `model/network.py` exposes encoding/decoding primitives and `ModelOutput`. Preserve the supervised output contract; add policy/value/Q wrappers rather than changing every scalar output into an RL distribution. +- `io/batch.py` and `training/loss.py` describe supervised batches and masked supervised losses. RL needs dedicated transition/rollout batches; do not overload target masks with termination semantics. +- `training/engine.py`, `training/runtime.py`, `training/optimization.py`, and `train.py` already separate orchestration from model, data, and update services. Extend that separation with algorithm runtimes. +- `integration/specifications.py` already defines persistent `module:attribute` factories and state policies. Extract reusable factory loading/identity validation; environments are not `TrainingObserver` or `TrainingController` instances and must not be forced into those protocols. +- `evaluation/service.py` and its monitor currently describe supervised loss. Extend results/monitors with named metrics rather than placing returns in `total_loss`. +- `artifacts/run_checkpoint.py` currently stores one model and optimization state in format version 2. Introduce an explicit compatible checkpoint migration and an algorithm-owned bundle for additional networks/optimizers. +- `hyperparameter_search.py` launches `evaluation_script` once and reads named JSON metrics. Keep that contract. Share factory conventions with hooks, not its subprocess-per-evaluation execution model. + +Proposed new modules: + +```text +config/rollout_config.py Environment/reward factories, bindings, budgets +config/rl_config.py Policy specs and discriminated algorithm phases +rollout/{contracts,bindings,history,coordinator,returns}.py +rollout/{factories,dependent,rewards,remote}.py +rollout/adapters/{gymnasium,tensor_vector}.py +trajectory/{schema,store,importers,dataset,relabel}.py +rl/{policy,distributions,models,normalization,metrics}.py +rl/{runtime,checkpoint,transfer}.py +rl/algorithms/{return_conditioned,iql,ppo}.py +rl/{ppo_buffer,offline_sampling}.py +rollout_api.py Stable external adapter/collector API +rl_api.py Stable policy and learner API +``` + +Names can be adjusted during implementation, but the ownership boundaries and contracts below are requirements. + +## 3. Common contracts to settle before implementation + +### 3.1 Time and observation/action alignment + +Use the following canonical transition everywhere: + +```text +h_t = observations through o_t, actions only through a_(t-1), validity and time metadata +a_t ~ policy(h_t) +environment.step(a_t) -> o_(t+1), reward_t, terminated_t, truncated_t +transition = (h_t, a_t, reward_t, o_(t+1), boundary flags) +``` + +The policy and V must never see `a_t`, its reward, or next observation when computing their step-t outputs. Q may receive `a_t` through a separate action encoder. Introduce an explicit policy window view that puts current observations and previous actions at one decision position and supervises the current action there. Do not reinterpret ordinary `target_offset` silently. For existing forecast models, keep their training alignment and use an explicit forecast binding; do not claim that arbitrary next-item checkpoints can be converted into current-observation policies without retraining. + +For robotics, the row contains current images/proprioception and the previous motor command; the output is the next command to execute. For order-book forecasting, a forecast binding appends predicted level-4 features and adapter-computed level-2 features into the same next context position. Both use the coordinator, with different bindings. + +### 3.2 Environment and reward protocols + +Use typed dataclasses/protocols with a protocol version. Illustrative signatures: + +```python +class EnvironmentAdapter(Protocol): + def describe(self) -> EnvironmentSpec: ... + def reset(self, request: ResetRequest) -> ObservationBatch: ... + def step(self, request: StepRequest) -> TransitionBatch: ... + def close(self) -> None: ... + +class RewardProvider(Protocol): + def score(self, transitions: TransitionBatch) -> RewardBatch: ... + def finalize_episode(self, episode: EpisodeSummary) -> EpisodeOutcome: ... + def close(self) -> None: ... +``` + +`EnvironmentSpec` declares ordered fields, shapes, dtypes, units, bounds, batch/device support, partial-reset/autoreset semantics, seed support, and snapshot capabilities. `StepRequest` includes slot IDs, episode IDs, transition indices, an idempotency/request ID, actions, and relevant context. The coordinator steps only active slots. GPU adapters use fixed slots plus an active mask; adapters unable to step subsets must freeze inactive worlds or keep all slots active during collection. Never silently advance a live trajectory without recording it. + +`TransitionBatch` contains true next observations, named rewards if supplied, separate `terminated`/`truncated` masks, optional declared privileged critic observations, elapsed simulated time, and diagnostics. Return final pre-reset observations explicitly. Autoreset adapters normalize their outputs into this contract and separately expose reset observations. A state that was reset is never used as the successor of the preceding episode. + +Every hook is instantiated once per rank/run or evaluation session and closed in `finally`. Partial reset is explicit and changes only selected slots. Episode IDs are unique across ranks/restarts. A stateless dependent-variable adapter uses the same protocol, seeded from the initial history, but can simply calculate a deterministic function of predictions without a physics engine. + +Environment and reward factory specifications share `factory`, `config`, `id`, and version fields. Use `module:factory` as the CLI/config path and allow supplied instances for local Python runs. Construct instances inside workers for DDP. Adapter configuration specifies connection/launch details; no per-step process startup. + +The public value representation is domain values: original categorical labels, physical continuous units, and image arrays/tensors. One shared codec maps to/from model representation using fixed training metadata. Provide a numeric tensor fast path with declared integer vocabularies and scales; use a metadata handshake once to avoid decoding labels through Python on every GPU step. Reject changed units/vocabularies/shapes and non-finite values. Never refit preprocessing on rollout/evaluation data. + +Provide optional adapter `state_dict`/`load_state_dict` and explicit capabilities for exact snapshots, including remote simulator RNG and reward state. Exact snapshots are optional; never infer snapshot support from the existence of a Python method alone. + +### 3.3 Bindings and field ownership + +Each dynamic model-input field, image view, and structured input has exactly one next-step source: `prediction`, `environment`, or `provided` (an explicitly scheduled exogenous input). Static fields remain constant within an episode. Return conditioning is owned by the coordinator. Reject missing coverage, duplicate producers, ambiguous prediction/environment overrides, and exhausted exogenous schedules before the affected transition. + +Mappings include prediction-to-action names and codecs, input update names, and structured masks. A model can predict outputs that are only actions and never fed back, or outputs that are appended directly. Environment outputs can include full corrected state when simulator state is authoritative. This choice must be declared per field; do not automatically overwrite predicted values. + +Image frame-valid masks, attention masks, depth occupancy masks, and time coordinates advance atomically with their features. Reuse `ExecutionSchema` for shape/dtype checks. Depth updates replace a complete next-position structured value plus its mask. Prefix/preconditioning images remain static unless the policy interface explicitly models them as dynamic image views. + +### 3.4 Rewards, outcomes, and discounts + +Store raw named reward channels and episode metrics separately. Algorithms optimize one scalar reward selected by explicit fixed channel weights and transforms. Version and persist this scalarization; multiple channels do not imply a multi-objective learner. Avoid double-counting a channel produced by both the simulator and reward provider. + +Support sparse terminal rewards: the episode outcome callback runs on termination/truncation before the terminal transition is sealed and may supply an explicitly configured terminal reward adjustment. Once emitted to a learner, a transition is immutable. Scores available only after a whole collection run are evaluation metrics unless an offline relabeling job constructs a new dataset. PPO rejects unresolved delayed reward providers; it cannot update from provisional rewards and revise them later. + +Distinguish task termination from time-limit truncation and collection boundaries. Store a `bootstrap_allowed` flag validated against those boundaries: ordinary true termination disables bootstrap; time-limit truncation with a valid final observation permits it. If task semantics make a time limit genuinely terminal, encode that explicitly. Collector horizon cuts are fragments, not artificial episode ends. + +If an adapter reports both termination and truncation on the same transition, preserve both facts and give true termination precedence for bootstrap and GAE. Import validation rejects contradictory discount/next-observation claims, not the simultaneous flags by themselves. + +Use fixed gamma per decision for this release. Record physics substep count and duration, and reject variable-duration RL decisions rather than applying a misleading fixed discount. Raw rewards remain unchanged in artifacts; any learner reward scaling is a fixed, checkpointed transform. + +## 4. Stage 1 — shared rollout and trajectory infrastructure + +### 4.1 Coordinator and history storage + +Implement a coordinator independent of CLI, model format, and optimizer. It owns slot identities, bindings, codec calls, context windows, masks, environment/reward lifecycle, collection budgets, and sink writes. It calls a one-step predictor interface; PT/ONNX forecasting and RL policy runtimes implement that interface. + +Use a per-slot history ring buffer with explicit decision positions and padding. Each collection starts from either validated historical seed windows or adapter reset observations. Default for reset-only starts: left padding with false attention-valid masks, then one real observation and a declared previous-action start token/value. Longer warm-up requires an explicit source/controller and is recorded. Never fabricate valid historical camera frames or prior actions. + +Budget modes are explicit: maximum transitions, completed episodes, and per-episode maximum decisions, plus inference `generation_steps`. Prediction output budgets count emitted predictions, even when trajectories end early. Ragged results carry episode/step identity; padded tensors carry valid masks. For fixed-shape GPU collection, collect the configured decisions per slot and reset ended episodes in place. + +Terminal predictions must still be executed and rewarded. Refactor the existing generation loop's early break so the final requested action/forecast gets a transition and outcome even though no next model call is needed. In pure legacy prediction-feedback mode use a no-op adapter and avoid externally visible extra work. + +Build one coordinator path for eager PT/ONNX and media/tensor inputs. Avoid eager CPU conversion for the torch policy path. Separate output formatting from collection so existing flat prediction/probability output remains unchanged when there are no episode boundaries. Include episode IDs and masks in the new trajectory outputs. + +### 4.2 Adapters and service transport + +Ship these adapters/examples: + +- A dependent-feature adapter for level-4 to level-2 order-book feature updates, with deterministic synthetic data and no trading-system dependency. +- A native torch vector adapter for small analytic control environments, with CPU/CUDA support and partial resets. This is the reference for high-throughput learning. +- An optional Gymnasium single/vector adapter covering discrete, continuous Box, MultiDiscrete, and supported dictionary observations. Pin/test the optional dependency and normalize supported autoreset modes explicitly. +- A deterministic toy robotics adapter with RGB views and motor commands, demonstrating image/mask updates without a heavyweight simulator installation. +- A generic persistent remote adapter and example server using versioned, framed requests/responses, request IDs, deadlines, handshake schemas, and bounded payload sizes. Numeric arrays use a binary representation preserving shape/dtype; avoid JSON pixel lists. Support start/connect/close and demonstrate crash handling. No transparent retry of a non-idempotent step after an ambiguous timeout; require server deduplication or terminate collection with a clear error. + +Third-party robotics integrations implement the same adapter protocol. The release claims complete simulator integration plumbing, with tested local/vector/remote examples; it does not claim bundled production integrations for every robotics engine. Keep Gymnasium and media dependencies optional and lazy-loaded. Timeouts are specified per reset/step/reward request; a synchronous local call cannot be forcibly interrupted in-process, so adapters requiring hard deadlines must use process/service isolation. + +### 4.3 Trajectory artifact version 1 + +Define a manifest plus atomic, sharded records. Use Arrow/Parquet for tabular columns and indexed binary/tensor shards for images and depth payloads; do not require all media in RAM. Include a final committed manifest, shard checksums, counts, codec/schema hashes, and immutable source provenance. Incomplete shards are recoverable but never silently treated as a finished dataset. + +Required logical fields: + +| Scope | Fields | +|---|---| +| Dataset | Format version, field/units/action specs, preprocessing/codec hash, reward definition and scalarization, task ID, split IDs, simulator/adapter versions, collection policy artifact ID | +| Episode | Globally unique ID, slot/rank, seed/reset conditions, static context, start/end status, outcome metrics, simulated duration | +| Transition | Episode ID, t, current observation reference, action, next/final observation reference, named rewards, terminal/truncation/bootstrap flags, time, validity, behavior policy version | +| Optional learner data | Joint old log-probability, value estimate, pre-transform continuous action, action masks, privileged critic observations, collection generation | +| Fragment | Starting history references, continuation/end reason, preceding action, whether complete episode outcome/return is known | + +Store observations once and reference adjacent frames where practical. References preserve true terminal successor observations even when the next stored record is a reset. Serialize action component order, allowed categories, bounds, transforms, and masks as part of compatibility hashes. Sinks support configurable flush intervals and bounded queues; backpressure cannot drop transitions. + +Add strict importers for existing tabular/PT histories with explicit mappings for episode, observation, action, reward, and boundary fields. Reject gaps, conflicting terminal flags, missing successors for bootstrapped records, and ambiguous action timing. Never assume every file is one complete episode. For final terminal steps without observable successors, permit an explicit missing-next mask only when no bootstrap is used. Split by whole episode (and optionally task/collection group), not windows; detect duplicate episode IDs across train/evaluation splits. + +Offline reward relabeling produces a new manifest with parent dataset ID and new reward version. Episode metrics are not automatically copied into every step as rewards. Add return/transition validation utilities and inspectable trajectory summaries. + +### 4.4 Inference, evaluation, and configuration + +Add shared rollout config referenced by `infer`, new `sequifier rollout --config-path`, and training rollout evaluation sources. `infer` keeps its existing arguments and output behavior when rollout config is absent. `rollout` can start from environments without a file-backed seed dataset. + +Replace input/target equality validation with binding coverage only when a rollout binding is present; legacy feedback paths keep their existing checks until routed through an equivalent generated binding. Keep BERT/embedding and multi-position forecast restrictions explicit. No relaxing a validator without implementing the associated image/depth history behavior. + +Training-time rollouts run against an immutable evaluation snapshot, separate environment slots and RNG streams, and no learner gradients. Restore model mode after evaluation. Metrics include channel returns, scalarized return, success rate when defined, episode length, completed/censored counts, simulated time, and collection/policy/transport timing. Aggregate weighted sums/counts across ranks; report incomplete episodes separately. + +Extend evaluation sources to a discriminated supervised/rollout union and monitors to select source + metric + direction. Preserve legacy total-loss selectors by normalization. Training schedules use epoch/batch triggers for supervised phases and update/environment-step triggers for RL phases. Hyperparameter search can run rollout evaluation using `evaluation_inference_config` pointing to an inference config with rollout settings, or consume native named evaluation metrics; keep legacy `evaluation_script` JSON output intact. Pruning uses the configured comparable metric, not an accidental PPO loss. Disallow incompatible per-trial metric schedules. + +Replace loss-specific best-model bookkeeping in `RunState` with a typed monitor identity/value/direction and a migration for existing `best_validation_loss` state. Update best-checkpoint selection, export naming/selection, early stopping, visualization, and search consumers together. Continue to emit legacy supervised loss metrics alongside the new named metric stream so existing integrations remain usable. + +### 4.5 Stage 1 acceptance + +- An order-book model with level-4 outputs and level-4/level-2 inputs generates multiple steps; synthetic adapter outputs prove that the next model call sees the derived features. +- Camera observations returned after a motor command become the next policy input, including masks, on CPU and the optional GPU path. +- Different episode lengths, partial resets, terminal final observations, truncations, and collection fragments survive collection → storage → loading unchanged. +- PT and ONNX forecast rollouts have matched deterministic results within documented tolerances; all existing inference tests pass. +- A terminal reward is recorded for the final emitted action, adapters close on success/error, and remote timeout/retry behavior is tested. +- Rollout evaluation can select a best checkpoint and return custom hyperparameter-search metrics. + +## 5. Stage 2 — return conditioning and complete offline IQL + +### 5.1 Policy distributions and model wrappers + +Implement these before either IQL policy extraction or PPO: + +```python +PolicyRuntime.act(history, *, deterministic, generator) -> PolicyStep +PolicyRuntime.evaluate_actions(history, actions, auxiliary) -> ActionEvaluation +``` + +`PolicyStep` includes domain action, internal action, joint log-probability, optional value, action mask, and any continuous latent needed for numerically stable likelihood evaluation. `ActionEvaluation` evaluates exactly the same recorded joint action. Its joint log-probability sums over action components, never averages them. + +Distributions: + +- Categorical and independent MultiDiscrete components, excluding special tokens and respecting state-dependent masks. Reject an all-masked component. Store masks used during collection. +- Ordered autoregressive categorical actions: sample in declared component order; evaluate recorded actions by teacher forcing that same prefix; sum conditional log-probabilities. No sampling a new prefix when scoring a recorded action. +- Unbounded diagonal Gaussian continuous actions with learned/clamped log standard deviation. +- Affine tanh-transformed diagonal Gaussian for bounded continuous actions. Account for the tanh and affine Jacobians and retain pre-tanh actions for PPO. Never externally clip a sampled Gaussian and then score it as if unclipped. Offline actions exactly at bounds require an explicit, reported inverse-transform epsilon convention or rejection; no silent arbitrary clamping. +- Joint mixed actions: explicit factorization and component order. Initially continuous blocks are independent conditional on history; any dependence on sampled categorical actions must be explicitly represented and identically evaluated. + +Use FP32 probability math even with lower-precision networks. Analytic entropy is available for independent categorical/unbounded Gaussian policies; for transformed or autoregressive distributions use a documented current-policy Monte Carlo estimator, not entropy evaluated on stale PPO actions. Temperature, masks, normalization, and action bounds are fixed within a PPO collection/update generation. + +Freeze dropout and stochastic augmentation for RL action collection and likelihood evaluation, while retaining autograd during optimization (`eval()` does not mean `no_grad()`). Freeze mutable observation-normalization state within a PPO iteration. Use deterministic per-record transforms where any image preprocessing is required. Before the first optimizer update, old/new log-probabilities must match numerically. + +Compose separate actor, V, Q1, and Q2 networks using existing ingestion/backbone/encode primitives. Default to no shared trainable parameters between actor, V, and Q networks; this prevents unintended optimizer interaction. Q encodes history first, then combines its representation with the recorded current action using an action encoder. V receives history only. Q1/Q2 may be implemented as a validated twin module but must have independent learned estimates. Share immutable preprocessing specifications, not mutable normalization buffers. + +Allow explicitly separate actor and critic observation schemas for asymmetric simulator critics. A privileged critic input must be available in collection, its genuine successor state, offline data when used for IQL, and resume snapshots. It must never enter the actor codec, actor encoder, or exported actor artifact. Validate and test this isolation. Default all networks to the ordinary observation history to keep dataset-only workflows straightforward. + +Initialize actor from a compatible supervised policy artifact; initialize critics independently or from explicitly copied encoders. Register all parameters in the parameter catalog, freezing/optimizer routing, initialization provenance, and checkpoint bundle. Ordinary point-regression actor transfer can initialize Gaussian means only under an explicit conversion policy, with new standard-deviation parameters recorded. + +### 5.2 Return-conditioned sequence modeling + +Implement a named algorithm runtime `return_conditioned` using the policy-aligned window view. This is Decision-Transformer-style return conditioning using Sequifier's multivariate rows; do not claim identical interleaved return/state/action token architecture. + +For complete recorded episodes compute: + +```text +G_t = reward_t + gamma * G_(t+1), with G_T = 0 +``` + +Gamma is explicit in `(0, 1]`, default 1 for this mode. Compute rewards with the configured fixed scalarization; persist return scale and definition. For incomplete/truncated episodes, require a declared finite-horizon objective or exclude them from exact return-to-go training. Never present an unknown future tail as a known zero return. IQL may still consume bootstrappable fragments from the same dataset. + +Add `desired_return` as a dynamic input at the current decision position; actions remain targets, with only previous actions in features. Use the existing supervised optimization services for masked action likelihood loss. Optional deterministic regression loss can be exposed only for the standalone return-conditioned learner; such an artifact is not directly PPO/IQL-compatible until converted into a distributional actor. Default to the shared distributional policy for seamless phase transfer. + +Inference requires an initial desired return in raw configured reward units and updates it after each observed reward: + +```text +desired_return_(t+1) = (desired_return_t - reward_t) / gamma +``` + +Apply scaling only at the encoder boundary and reset the budget per episode. Optional clamps must be authored configuration and logged; do not infer attainable targets from evaluation episodes. Unknown future realized returns are never used online. + +Compare with behavior cloning and success-filtered behavior cloning on a controlled dataset with conflicting actions and known return differences. Test gamma/scaling arithmetic, episode boundaries, and absence of future-action/reward leakage. + +### 5.3 IQL data and update implementation + +Add an `iql` runtime that samples fixed-dataset transitions plus sufficient history for both h_t and h_(t+1). Sampling a transition includes up to the configured context length of history and one successor observation; never reconstruct successor context by concatenating arbitrary independently sampled windows. Respect masks, previous-action alignment, static context, and image/depth payload references. + +Implement the canonical losses using scalarized rewards, learned V, twin Qs, and slowly updated target Qs. At each update, compute detached targets/advantages from a consistent pre-update snapshot: + +```text +q_bar = min(Q1_target(h_t, a_t), Q2_target(h_t, a_t)) +delta = stop_gradient(q_bar) - V(h_t) +L_V = mean(abs(expectile - indicator(delta < 0)) * delta^2) + +y = reward_t + gamma * bootstrap_allowed_t * stop_gradient(V(h_next)) +L_Q = mean((Q1(h_t,a_t)-y)^2 + (Q2(h_t,a_t)-y)^2) + +advantage = stop_gradient(q_bar - V(h_t)) +weight = min(exp(inverse_temperature * advantage), max_weight) +L_actor = -mean(stop_gradient(weight) * log policy(a_t | h_t)) +``` + +Evaluate the exponential stably by capping its input at `log(max_weight)`. Reject invalid expectile/temperature/target-update settings. Target Qs are frozen copies, updated by Polyak averaging only after a successful critic optimizer step. Define one algorithm update as the configured V/Q/actor transactions and record individual update counters. Recompute forward graphs as necessary; do not backpropagate through detached targets, target networks, or the weight calculation. If AMP skips a transaction, dependent counters/target updates follow the actual applied steps; checkpoint only after the complete update boundary. + +Keep initial scope faithful to offline IQL: a fixed training dataset, an independently held-out evaluation environment/dataset, uniform transition sampling, and no fresh data injected during the phase. Expose expectile, inverse temperature, maximum weight, gamma, Polyak factor, batch size, gradient clipping, and separate optimizer/scheduler plans. Reward transforms are fixed. Log Q/V ranges, Bellman residuals, advantage/weight distributions, weight-cap frequency, actor likelihood, update counts, and held-out rollout performance. + +RC actor to IQL transfer requires a declared conditioning policy: either retain desired-return conditioning with identical semantics for all actor/critic contexts, or initialize a separately specified unconditioned actor via an explicit conversion/distillation step. Never silently remove an input. The straightforward default workflow is behavior-cloned unconditioned actor → IQL; return-conditioned training is a separate supported route, not a mandatory IQL prerequisite. + +### 5.4 Training-plan changes + +Replace resolved phase branching with a discriminated union normalized from authored config: + +- `supervised`: existing sources, epochs, and source scheduling, unchanged defaults. +- `return_conditioned`: trajectory sources, policy interface, epochs, return definition. +- `iql`: fixed trajectory sources, updates, actor/V/Q specifications, algorithm settings. +- `ppo`: environment source, policy/value specifications, environment-step budget, rollout and update settings (Stage 3). + +Legacy phases without an algorithm normalize to supervised. Give online phases their own source type and budget; they must not require dummy `dataset_training` files, criteria, epochs, or validation loaders. Adjust singleton normalization, canonical search config, authored/resolved validation, phase compilation, docs, and CLI routing together. Search spaces can address new discriminated config fields after ordinary canonical composition rules. + +Introduce `AlgorithmRuntime` with explicit collect/prepare/update/evaluate/checkpoint boundaries and named counters. Reuse optimization transaction mechanics, random-state handling, distributed strategy, metrics, and integration events. Keep supervised `LossService` intact; algorithm loss functions return their own structured result with accounting counts. Do not implement RL as observer callbacks changing gradients behind the engine's back. + +Extend update event identities with algorithm, network/optimizer role, algorithm-update index, and environment-step count. Keep existing supervised identity fields/behavior compatible. Controller directives apply only to the named optimizer transaction; observer/controller state is saved once per declared instance. Validate execution requirements per algorithm so a controller assuming one optimizer cannot silently change the wrong network. Define scheduler clocks explicitly per optimizer update, not in fabricated online epochs. + +Phase transitions retain actor weights and compatible codecs, but reset optimizer moments by default when algorithms change. Copy/reset value state only under explicit compatible specs. Record transfer provenance and validate action/observation schema, context length, return conditioning, and bounds before copying. Artifact warm-start and exact run resume are different operations. + +### 5.5 Stage 2 acceptance + +- Return-conditioned policies change action choices with requested return on a deterministic fixture; online return-budget arithmetic matches offline labels. +- IQL losses, gradient ownership, expectile asymmetry, weight caps, and target updates match hand-computed examples. +- A tiny tabular/analytic control task demonstrates learned value ordering and policy improvement against a deliberately mixed-quality behavior dataset over fixed seeds; include tolerances and report variance rather than asserting monotonic improvement per update. +- Actor/V outputs are invariant to replacing current-action labels or future observations; Q changes appropriately with its explicit action input. +- Distribution sampling/scoring round trips, categorical prefixes, action masks, and bounded Gaussian Jacobians pass numerical tests. +- Exact offline resume reproduces uninterrupted sampling and updates within device-appropriate deterministic tolerances; actor-only exports reproduce pre-export policy outputs. + +## 6. Stage 3 — full synchronous PPO + +### 6.1 Collection and rollout buffer + +Freeze a behavior policy version per iteration. Collect `num_envs × rollout_length` decisions per rank using the Stage 1 coordinator. Run collection without autograd; store observations/history references, sampled actions and latent values, joint old log-probabilities, old V estimates, scalar/raw rewards, final observations, terminal/truncation flags, masks, and policy version. + +A completed episode resets immediately in its slot and starts a new history. A collection cut preserves live context for the next iteration. PPO may change the policy between fragments of the same episode; each transition retains its behavior version and only the current iteration's buffer is optimized. Treat environment-step count as the total across ranks. Validate a total budget compatible with fixed-size synchronous collection or explicitly define a final smaller masked rollout; never silently overshoot the authored budget. + +Preallocate device rollout tensors/rings for numeric/image payloads and avoid duplicating full history windows per transition. Retain enough pre-rollout context to reconstruct every training history exactly. If storage is insufficient, fail before collecting, with a memory estimate. Permit CPU/pinned staging where configured; benchmark transport separately. Seal the buffer before calculating targets or updating. Reuse each buffer only for its configured PPO optimization epochs, then discard it from on-policy training (optional trajectory archival is independent). + +### 6.2 Bootstrap and GAE + +Evaluate V on each genuine successor/final observation using the corresponding successor history, with the same frozen value snapshot used during collection. Let b_t indicate whether bootstrap is permitted; let c_t indicate whether GAE recursion continues into the next stored transition of this same episode and collection segment: + +```text +td_t = reward_t + gamma * b_t * V_old(h_next) - V_old(h_t) +A_t = td_t + gamma * lambda * c_t * A_(t+1) +value_target_t = stop_gradient(A_t + V_old(h_t)) +``` + +True terminal: b=0, c=0. Time-limit truncation with valid final state: b=1, c=0. Ordinary transition inside the segment: b=1, c=1. Last live transition at a collection cut: b=1, c=0, so it bootstraps without accessing the next collection's advantages. Never bootstrap using reset observations or carry GAE across episode IDs. + +Compute GAE in FP32. Normalize advantages once over globally valid transitions per iteration using DDP sums/counts (configurable off); do not normalize rewards implicitly or re-normalize each minibatch. Keep returns and old values fixed through optimization. + +### 6.3 PPO loss and optimization + +Evaluate each recorded action on the exact same observation history, mask, codecs, and frozen preprocessing state. Joint ratio and losses: + +```text +ratio = exp(logp_new - logp_old) +L_policy = -mean(min(ratio * A, + clip(ratio, 1-epsilon, 1+epsilon) * A)) +L_value = 0.5 * mean((V_new - value_target)^2) +L_total = L_policy + value_coefficient * L_value + - entropy_coefficient * entropy_current +``` + +Expose optional value clipping around `V_old` as a documented maximum-of-clipped/unclipped loss, disabled by default. Implement optimizer gradient clipping and optional target-KL early stopping. Estimate KL consistently (for example `mean((ratio-1)-log_ratio)`) and aggregate globally; every rank stops at the same boundary. Log clipping fraction, ratio tails, entropy, KL, gradient norm, explained variance, value loss, returns, episode success, steps/sec, and timings. Detect non-finite ratios/losses and fail the iteration; do not hide them with arbitrary ratio clamping outside PPO's specified clipped objective. + +Use separate actor and value modules/optimizers by default, with no shared parameters. Combined mathematical loss can be split into separate transactions; both optimize the same sealed targets. Support configurable optimization epochs, minibatch size, and gradient accumulation with correct valid-sample weighting. Randomly sample decision indices and reconstruct complete fixed-length causal contexts; do not shuffle time positions inside a context. This is a finite-context transformer policy, not a persistent recurrent state requiring truncated BPTT. Any future hidden-state/KV-cache optimization must prove equivalence after parameter updates and resets. + +The complete PPO schema includes collection horizon/environment count, total budget, optimization epochs, minibatch/accumulation sizes, gamma, lambda, clipping epsilon, optional value-clip range, value/entropy coefficients, optional target KL, advantage normalization, actor/value optimizer and scheduler plans, gradient limits, action standard-deviation initialization/bounds, evaluation intervals, and checkpoint/resume policy. Validate positivity/ranges, actor/value parameter disjointness, batch coverage, divisibility or explicit remainder masks, and supported execution/distribution combinations before instantiating expensive simulators. Do not hide these controls in free-form adapter configuration. + +Use the shared distributions for discrete, continuous, ordered categorical, and mixed policies. A density-free deterministic regression head is invalid for PPO. Default evaluation uses mode/mean actions and no exploration; report separately if stochastic evaluation is requested. Deployment returns the same bounded/domain actions as collection. + +### 6.4 Distributed synchronization and simulator execution + +Each DDP rank has its own simulator batch, unique episode/slot IDs, and derived seeds based on run, rank, slot, and reset count. Actor/value parameters are synchronized before collection. All ranks perform compatible collect/update schedules, use globally weighted valid-sample losses, and agree on KL stopping, errors, pruning, and checkpoint boundaries. Avoid rank-zero-only environment collection feeding a nominal DDP learner. + +For IQL, partition/sample offline data with deterministic rank-specific samplers and synchronized update counts; target networks update from synchronized Q parameters. For all new runtimes, distinguish valid transition counts from padded context lengths in loss denominators and metric reductions. + +Remote adapters use batched RPC calls and one connection/session per declared rank namespace. A slow service may reduce throughput but must not silently mix policy generations. Support GPU tensor adapters without mandatory CPU synchronization per field. Simulator APIs, transport, rewards, and preprocessing have separate timings so performance problems are diagnosable. + +### 6.5 Checkpoints and resume + +Version the algorithm checkpoint payload and retain a reader/migration for existing supervised format-v2 artifacts. Save model specs/weights for actor, V, Q/target Q where relevant; separate optimizer/scheduler/scaler states; algorithm counters; dataset samplers; policy version; random streams; normalization; codec/action spec hashes; phase transfer provenance; and environment/reward state capabilities. + +For PPO, write durable checkpoints at complete iteration boundaries after optimizer updates, target computation, and metric accounting have finished. Exact resume additionally requires snapshots of live simulator states, histories, reset counters, reward state, and all rank RNG streams. If the adapter cannot snapshot, provide an explicit `reset_environments` resume mode: preserve learner state, reset environments/history, discard incomplete episode metrics, and label the run as a continuation without trajectory-equivalent resume. `exact` mode must fail rather than claim reproducibility. Mid-collection/mid-minibatch resume is not required in this release; a crash restarts from the last committed boundary. + +Keep actor-only model artifacts small and independent of critic state. Learner checkpoints contain the complete training bundle. Validate every network and optimizer assignment during restore; no missing-critic fallback in exact resume. Atomic manifests must not point to partially written rank state. + +### 6.6 Stage 3 acceptance + +- Hand-computed PPO clipping tests cover positive/negative advantages, joint action likelihoods, optional value clipping, entropy, and KL stop behavior. +- GAE tests cover true terminals, truncations, collection cuts, autoresets, nonuniform episode lengths, and missing invalid successor observations. +- The identity-ratio invariant holds before any update, including image inputs, ordered action heads, and checkpoint round trips. +- Discrete and bounded continuous control environments show reliable learning over multiple seeded runs. Keep short deterministic CPU learning checks in CI; run longer multi-seed quality benchmarks separately and save results. +- An RGB toy control policy completes collection → PPO update → evaluation → checkpoint → reload → rollout. A structured/depth observation fixture proves the same schema path. +- DDP results agree with single-process global-batch loss calculations within tolerance; one rank hitting KL/error/termination does not hang others. +- Exact-snapshot adapters reproduce an uninterrupted continuation; non-snapshot adapters correctly reject exact resume and pass explicit reset-mode continuation. +- Remote vector environments can train PPO using the same algorithm runtime as in-process tensor environments. + +## 7. Authoring surface and operational workflow + +All names below are proposed new fields. Implement schema-validated examples with the complete existing model configuration in documentation/examples; these snippets show the new surface only. + +```yaml +rollouts: + robot: + environment: + id: toy_robot + factory: examples.toy_robot:make_environment + config: {num_envs: 32} + reward: + scalarization: {task_success: 1.0, effort: -0.01} + bindings: + actions: {motor_command: motor_command} + inputs: + camera: {source: environment, field: camera} + joint_position: {source: environment, field: joint_position} + previous_command: {source: prediction, field: motor_command} + +training_plan: + phases: + - name: offline_improvement + algorithm: iql + updates: 100000 + trajectory_sources: [recorded_robot_training] + policy: robot_policy + value: robot_value + critics: robot_q_twins + gamma: 0.99 + expectile: 0.7 + inverse_temperature: 3.0 + max_weight: 100.0 + target_update_rate: 0.005 + - name: simulated_refinement + algorithm: ppo + environment_source: robot + policy: robot_policy + value: robot_value + total_environment_steps: 1048576 + rollout_length: 128 + optimization_epochs: 4 + minibatch_size: 512 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 +``` + +Stage 2 adds named trajectory sources and policy/value/critic registries to the canonical config. Existing model interface definitions are referenced where suitable; avoid embedding unrelated interface authoring syntax in each algorithm config. Model registry validation must catch accidental parameter sharing and incompatible copied interfaces. `rollouts` is a shared named registry consumed by inference, rollout collection, and evaluation. Expose consistent factory resolution relative to the project root/import environment, with explicit error messages for missing modules. + +Commands: + +```text +sequifier rollout --config-path configs/collect.yaml +sequifier preprocess --config-path configs/import-trajectories.yaml +sequifier train --config-path configs/train-return-conditioned.yaml +sequifier train --config-path configs/train-iql-ppo.yaml +sequifier infer --config-path configs/infer-with-dependent-features.yaml +``` + +Extend preprocess with an explicit trajectory import/relabel operation union rather than treating transition metadata as ordinary selected features. A Python API supports the same operations and direct local adapter instances. Update `sequifier docs`, `make` templates, consolidated docs generation, documentation/configs, and release notes. Publish examples for dependent scalar variables, RGB control, offline return conditioning, IQL warm-start, and PPO with local and remote simulators. + +Clarify numerical versus behavioral compatibility: ONNX exports expose policy parameters with action-spec/codec metadata, and the shared runtime performs sampling/scoring. Exported stochastic trajectories need not match PT draw-for-draw across backend RNGs; distribution parameters and deterministic actions must match within tested tolerances. Store ordered categorical decoding dependencies and implement equivalent stepwise ONNX sampling, or reject the particular export at preflight. Never label an exported logits tensor as a complete bounded-action policy without its transforms. + +## 8. Implementation order within the single update + +| Work package | Depends on | Concrete deliverable | +|---|---|---| +| A. Contracts and schema | None | Typed rollout/trajectory/action specs, phase union design, alignment and boundary fixtures | +| B. Coordinator and adapters | A | Dependent-variable inference, tensor/Gym/remote adapters, history/mask handling | +| C. Trajectory data | A/B | Atomic store/import/relabel, episode splits, history-aware loading | +| D. Evaluation integration | B/C | Rollout metrics, monitors, search/pruning support, inference regressions | +| E. Policy primitives | A | Distributions, actor/V/Q wrappers, codecs, log-probability tests, actor exports | +| F. Learner orchestration | C/E | Algorithm runtime, config union, optimizer/checkpoint bundle and migration | +| G. Return conditioning | C/E/F | Return labels, training, inference budget updates and examples | +| H. Offline IQL | C/E/F | Complete learner, targets, weighted actor extraction, resume and evaluation | +| I. PPO | B/D/E/F | Collection buffer, GAE, PPO optimization, DDP and resume semantics | +| J. Integrated release validation | G/H/I | End-to-end examples, numerical/learning/performance checks, docs and migration notes | + +Build contracts and small mathematical fixtures first; add learning tests once the corresponding algorithm exists. Do not combine algorithm validation with heavy simulator installation. Stage gates should be reviewable commits within the same update, with the release gated on J. + +## 9. Verification, performance, and release criteria + +Preserve all existing supported preprocessing, supervised learning, inference, artifact, and export behavior. Run the existing required suite and targeted tests for every changed subsystem. New tests must prove mathematical/boundary behavior rather than mirror private implementation details. + +Maintain three validation tiers: + +1. Fast CPU tests: schema validation, temporal alignment, codec round trips, distributions/gradients, return arithmetic, IQL/PPO/GAE formulas, partial resets, artifact migration, remote failure handling. +2. Integration tests: CLI/Python parity, file/tensor/media sources, actor export, exact/reset-mode resume, training phase transfer, rollout monitors/hyperparameter search, two-process CPU DDP and optional CUDA DDP. +3. Reproducible learning/performance jobs: fixed datasets/tasks/seeds, BC and return-conditioned and IQL comparisons, PPO discrete/continuous/image tasks, local versus remote adapter equivalence, and measured throughput/memory. + +Before optimization, record an agreed baseline on specified hardware with context length, model size, image resolution, simulator count, and DDP topology fixed. Measure policy inference, environment/reward stepping, transfer, serialization, update time, peak memory, and total valid transitions/sec separately. Require no accidental per-field GPU-to-CPU copies in the native tensor collection path and bounded memory over long runs. Set quantitative throughput regression limits after the baseline is measured; do not invent a hardware-independent performance promise. + +The update is complete when all Stage 1/2/3 acceptance criteria pass, the old test suite remains green, the five documented workflows run from clean optional-dependency installations, checkpoints/exports have tested compatibility, and unsupported algorithm/model combinations fail before long collection or training begins. Publish benchmark configuration and results, including seeds and uncertainty, without claiming general robotics SOTA from toy tasks. + +## 10. Algorithm references and interpretation + +- [Decision Transformer](https://arxiv.org/abs/2106.01345): return-conditioned causal action modeling. The proposed multivariate-row representation is a Sequifier-specific implementation choice. +- [Implicit Q-Learning](https://arxiv.org/abs/2110.06169): expectile value learning, Q backups, and advantage-weighted policy extraction. The separate-network, runtime, and storage designs above are proposed integration decisions. +- [Proximal Policy Optimization](https://arxiv.org/abs/1707.06347): clipped policy objective and repeated minibatch optimization of recent rollouts. +- [Generalized Advantage Estimation](https://arxiv.org/abs/1506.02438): value-based advantage estimation. The explicit adapter boundaries, checkpoint contracts, and terminal-observation handling are requirements of this implementation plan. diff --git a/documentation/rollout.md b/documentation/rollout.md new file mode 100644 index 00000000..b66e0544 --- /dev/null +++ b/documentation/rollout.md @@ -0,0 +1,184 @@ +# Local numeric forecast rollouts + +The first rollout slice collects one seed-initialized episode or episode fragment +on CPU. It supports real scalar features and real static conditioning, with +one-step `causal` or `final_value` forecast models (`target_offset: 1`). Model +predictions and a persistent local adapter jointly produce the next input row. +There is no learning algorithm in this path. + +## Inference configuration + +Add this to an otherwise valid inference configuration for a model with inputs +`x`, `d` and target `x`: + +```yaml +device: cpu +model_type: generative +autoregressive: true +generation_steps: 3 +rollout: + mode: forecast + seed_index: 0 + trajectory_path: outputs/trajectory.json + adapter: + factory: sequifier.rollout.dependent:LinearDependentAdapter + protocol_version: 1 + config: + prediction: x + external_input: d + multiplier: 2.0 + terminate_after: 3 + bindings: + - {input: x, source: prediction, key: x} + - {input: d, source: environment, key: d} +``` + +Run the existing `sequifier infer --config-path configs/infer.yaml` command. +PT and existing schema-bearing ONNX models use the same coordinator and their +existing inference backends. This feature adds no export format. + +`seed_index` is required and selects one prepared model window, using zero-based +dataset/chunk order. Unlike legacy autoregression, selection happens before the +first-subsequence-per-sequence filter. It uses the usual inference window +selection rules, including `window_stride`. An out-of-range index fails. The +seed must have at least one valid position, end in a valid position, and contain +only left padding before its valid observations. Every feature and the attention +mask must have the same context length. Static values come from the existing +processed static feature table and stay constant. + +Each changing input must have exactly one binding. `prediction` keys name model +targets; `environment` keys name adapter outputs. Static inputs cannot have +bindings. Missing producers, duplicate input bindings, unknown predictions, and +extra or missing adapter fields fail clearly. Outputs that are only consumed by +the adapter do not need a feedback binding. + +The seed dataset uses the existing **preprocessed training representation**. +The adapter and stored trajectory use **original units**. The inference wrapper +decodes the seed and forecasts and encodes every successor using the fixed +training `mean`, `std`, and `normalize_real_columns` metadata. It uses the same +`std + 1e-9` transform as preprocessing, without fitting statistics on rollout +values. Float overflow, non-finite values, arrays in place of scalars, and +incompatible model inputs are rejected. Model padding stays zero with a false +attention mask. Features and attention masks slide together, appending a valid +successor position even for the last executed prediction. + +With `rollout` present, inference writes the trajectory to `trajectory_path` +(relative to `project_root` unless absolute). It replaces that file atomically +after successful collection and validation. This path collects one selected +seed with one model; it does not produce legacy flat prediction/probability +files. Omitting `rollout` preserves existing inference output and configuration +rules. + +## Adapter protocol + +Adapters declare `protocol_version = 1` and implement: + +```python +from sequifier.rollout_api import ResetRequest, StepRequest, StepResult + +def reset(self, request: ResetRequest) -> None: ... +def step(self, request: StepRequest) -> StepResult: ... +def close(self) -> None: ... +``` + +A `module:factory` receives `adapter.config` as keyword arguments and is called +once per collection. The adapter stays alive for all steps and closes on success +or failure, including reset, prediction, and result-validation failures. A +factory that fails during construction owns cleanup of any partial resources. +Python callers can pass a supplied adapter instance to `collect_rollout`. + +`reset` receives the episode ID, complete seed history, validity mask, static +inputs, and random seed. It initializes the environment from this history and +must not execute a transition or replace the seed. `step` receives the current +observation, all named forecasts, static inputs, and zero-based step ID. It +returns external fields, one finite scalar reward, and separate Python boolean +`terminated` and `truncated` flags. Reset and step requests contain independent +copies of coordinator data. Adapters must return the final pre-reset successor +and must not autoreset. + +Every prediction executes exactly once before collection can end. A boundary +stops further prediction and stepping. `generation_steps` is a collection +budget: exhausting it leaves `collection_cutoff: true` when the episode is +unfinished, without fabricating termination or truncation. Optional +`rollout.max_episode_steps` is an episode time limit; reaching it records +`truncated: true`. An adapter can also report its own truncation. Both boundary +flags may be true together; termination takes precedence for bootstrap. + +## Trajectory format and Python API + +The version-1 JSON artifact contains the domain seed history and mask, static +values, input bindings, prediction names, episode ID, and ordered transitions. +Each transition records the observation, predictions (the executed forecast +actions), external fields, reward, successor, step ID, and separate boundary +flags. `bootstrap_allowed` is false on termination and true otherwise, including +time-limit truncation with its recorded final successor. The artifact also +contains `total_return`, `collection_cutoff`, and inference provenance, including +the exact preprocessing statistics and seed origin. + +Seed history plus successors reconstruct every later history and attention mask; +they are not duplicated in each record. Artifacts record one episode fragment +and do not promise simulator snapshots or exact resumability. + +```python +from sequifier.rollout_api import load_trajectory, validate_trajectory + +trajectory = load_trajectory("outputs/trajectory.json") +validated = validate_trajectory(trajectory) +``` + +Saving and loading validate format/version, finite scalar data, seed shape and +mask, producer coverage, successor bindings, observation continuity, contiguous +episode/step IDs, total return, bootstrap semantics, and absence of transitions +after an episode boundary. The same validation is available for in-memory data. + +For direct Python collection, a predictor is a callable receiving a copied +`SeedHistory` in original units and returning a dictionary of scalar forecasts. +The collector owns adapter closure. `history_validator`, if provided, validates +the initial history and every successor (including a terminal successor) for +the predictor's representation; the inference integration uses the fixed codec +and existing `ExecutionSchema` here. + +```python +from sequifier.rollout_api import ( + RolloutConfig, SeedHistory, collect_rollout, save_trajectory, +) +from sequifier.rollout.dependent import LinearDependentAdapter + +config = RolloutConfig( + adapter={"factory": "sequifier.rollout.dependent:LinearDependentAdapter"}, + bindings=[ + {"input": "x", "source": "prediction", "key": "x"}, + {"input": "d", "source": "environment", "key": "d"}, + ], + seed_index=0, + trajectory_path="outputs/trajectory.json", +) +seed = SeedHistory( + observations={"x": [0.0, 0.0], "d": [0.0, 0.0]}, + attention_valid_mask=[False, True], + static_inputs={}, +) + +def predict(history): + return {"x": history.observations["x"][-1] + history.observations["d"][-1] + 1.0} + +trajectory = collect_rollout( + predict, LinearDependentAdapter(), seed, config, ["x"], max_steps=3, +) +save_trajectory(trajectory, config.trajectory_path) +``` + +The hand-calculated trajectory for that predictor is: + +| Step ID | Observation | Prediction | Successor | Reward | Terminated | +| --- | --- | --- | --- | --- | --- | +| 0 | `(0, 0)` | `x=1` | `(1, 2)` | 1 | false | +| 1 | `(1, 2)` | `x=4` | `(4, 8)` | 4 | false | +| 2 | `(4, 8)` | `x=13` | `(13, 26)` | 13 | true | + +The return is 18, with no collection cutoff. The first mask is `[False, True]`; +the next and subsequent masks are `[True, True]`. No fourth call occurs. + +Multimodal and categorical observations, reset-only initialization, vectorized +environments, remote transport, learning algorithms, and new export support are +outside this slice. diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 5003fd6e..084fe91c 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -23,6 +23,7 @@ extract_inline_metadata, load_dataset_metadata, ) +from sequifier.config.rollout_config import RolloutConfig from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -692,7 +693,7 @@ def resolve_inference_config( ] if missing_columns: raise ValueError( - "Inference metadata is missing selected columns: " f"{missing_columns}" + f"Inference metadata is missing selected columns: {missing_columns}" ) temporal_columns = [ column @@ -930,6 +931,7 @@ class _InferenceConfigBase( sample_from_distribution_columns: Optional[list[str]] = Field(default=None) infer_with_dropout: bool = Field(default=False) + rollout: Optional[RolloutConfig] = None autoregressive: bool = Field(default=False) generation_steps: Optional[int] = Field(default=None) @@ -1007,6 +1009,56 @@ class ResolvedInferenceConfig(_InferenceConfigBase[str, list[str], dict[str, str dataset_metadata: Optional[DatasetMetadata] = Field(default=None, exclude=True) _image_store: Any = PrivateAttr(default=None) + @model_validator(mode="after") + def validate_numeric_rollout(self): + if self.rollout is None: + return self + if not self.autoregressive or self.generation_steps is None: + raise ValueError( + "rollout requires autoregressive: true and generation_steps" + ) + if self.device != "cpu" or self.model_type != "generative": + raise ValueError("numeric rollout requires a CPU generative model") + if ( + self.training_objective not in {"causal", "final_value"} + or self.window_view.target_offset != 1 + or self.prediction_length not in (None, 1) + ): + raise ValueError( + "forecast rollout requires a one-step causal/final_value forecast with target_offset=1" + ) + if self.output_probabilities or self.sample_from_distribution_columns: + raise ValueError( + "numeric rollout does not support categorical probabilities or sampling" + ) + paths = ( + self.model_path if isinstance(self.model_path, list) else [self.model_path] + ) + if len(paths) != 1: + raise ValueError("rollout requires exactly one model_path") + fields = self.input_columns + self.static_columns + self.target_columns + if not self.input_columns or any( + "float" not in self.column_data_types[column].lower() for column in fields + ): + raise ValueError( + "numeric rollout supports real scalar inputs, static inputs, and targets only" + ) + if set(self.input_columns) & set(self.static_columns): + raise ValueError("rollout static and changing inputs must be disjoint") + if any(kind != "real" for kind in self.target_column_types.values()): + raise ValueError("numeric rollout requires real forecast targets") + metadata = self.dataset_metadata + if metadata is None: + raise ValueError("rollout requires fixed training preprocessing metadata") + if ( + metadata.image_sources + or metadata.preconditioning_image is not None + or metadata.depth_layouts.relevant_layouts(fields) + ): + raise ValueError("numeric rollout does not support image or depth inputs") + self.rollout.validate_bindings(self.input_columns, self.target_columns) + return self + def image_store(self): if self._image_store is None and self.dataset_metadata is not None: from sequifier.io.images import ImageStore @@ -1159,7 +1211,7 @@ def validate_generation_steps(cls, v: Optional[int], info: Any) -> Optional[int] f"'generation_steps' can only be larger than 1 if 'autoregressive' is true: {info.data.get('autoregressive')}" ) - if not np.all( + if info.data.get("rollout") is None and not np.all( np.array(sorted(info.data.get("input_columns"))) == np.array(sorted(info.data.get("target_columns"))) ): @@ -1185,9 +1237,13 @@ def validate_autoregressive(cls, v: bool, info: Any): raise ValueError( "Autoregressive inference is not possible for models with prediction_length > 1" ) - if v and not np.all( - np.array(sorted(info.data.get("input_columns"))) - == np.array(sorted(info.data.get("target_columns"))) + if ( + v + and info.data.get("rollout") is None + and not np.all( + np.array(sorted(info.data.get("input_columns"))) + == np.array(sorted(info.data.get("target_columns"))) + ) ): raise ValueError( "Autoregressive inference with non-identical 'input_columns' and 'target_columns' is possible but should not be performed" diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py new file mode 100644 index 00000000..9d9941ca --- /dev/null +++ b/src/sequifier/config/rollout_config.py @@ -0,0 +1,68 @@ +"""Opt-in configuration for local, single-environment forecast rollouts.""" + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class InputBinding(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + input: str = Field(min_length=1) + source: Literal["prediction", "environment"] + key: str = Field(min_length=1) + + +class AdapterConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + factory: str + config: dict[str, Any] = Field(default_factory=dict) + protocol_version: Literal[1] = 1 + + @field_validator("factory") + @classmethod + def validate_factory(cls, value: str) -> str: + module, separator, attribute = value.partition(":") + if not separator or not module or not attribute.isidentifier(): + raise ValueError("adapter.factory must be a 'module:factory' import path") + return value + + +class RolloutConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + mode: Literal["forecast"] = "forecast" + adapter: AdapterConfig + bindings: list[InputBinding] + trajectory_path: str = Field(min_length=1) + # Index over prepared windows in dataset/chunk order, before legacy seed filtering. + seed_index: int = Field(ge=0) + max_episode_steps: int | None = Field(default=None, gt=0) + + @model_validator(mode="after") + def reject_duplicate_producers(self): + names = [binding.input for binding in self.bindings] + if len(names) != len(set(names)): + raise ValueError("rollout bindings contain duplicate input producers") + return self + + def validate_bindings(self, inputs: list[str], targets: list[str]) -> None: + if len(inputs) != len(set(inputs)) or len(targets) != len(set(targets)): + raise ValueError("rollout input and target columns must be unique") + bound = {binding.input for binding in self.bindings} + if bound != set(inputs): + raise ValueError( + "rollout requires exactly one producer per changing input; " + f"missing={sorted(set(inputs) - bound)}, " + f"unknown/static={sorted(bound - set(inputs))}" + ) + unknown = { + binding.key + for binding in self.bindings + if binding.source == "prediction" and binding.key not in targets + } + if unknown: + raise ValueError( + f"rollout bindings reference unknown predictions: {sorted(unknown)}" + ) diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 19c32a37..dad283dc 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -1703,6 +1703,11 @@ gradients are absent. # Infer Command Guide +For CPU numeric inference with externally calculated next-step inputs, see +[local forecast rollouts](../rollout.md). An explicit `rollout` configuration +selects a single seed window and saves a versioned trajectory through the +existing inference command. + Models with `static_conditioning` read the processed static feature table named in the preprocessing metadata. Each inference window receives the configured static values for its `sequenceId`. diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index c23dc58a..92f1d584 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -1041,6 +1041,9 @@ def infer_generative( column_data_types: dict[str, torch.dtype], ): """Write generative predictions/probabilities for each dataset chunk.""" + if getattr(config, "rollout", None) is not None: + infer_numeric_rollout(config, inferer, model_id, dataset, column_data_types) + return data_path = config.data_path if data_path is None: raise ValueError("data_path must be provided or resolved from metadata") @@ -1233,6 +1236,124 @@ def infer_generative( ) +def infer_numeric_rollout(config, inferer, model_id, dataset, column_data_types): + """Run the opt-in single-seed path through the shared forecast coordinator.""" + from sequifier.model.execution_schema import ExecutionSchema + from sequifier.rollout.coordinator import collect_rollout + from sequifier.rollout.factories import load_adapter + from sequifier.rollout.preprocessing import FixedNumericPreprocessing + from sequifier.trajectory.store import save_trajectory + + rollout = config.rollout + remaining = rollout.seed_index + selected = None + for data_id, data in enumerate(dataset): + windowed = _windowed_inference_batch(config, data, column_data_types) + count = windowed.sequence_ids.numel() + if remaining >= count: + remaining -= count + continue + selected = (data_id, windowed, remaining) + break + if selected is None: + raise ValueError( + f"rollout seed_index {rollout.seed_index} is outside the prepared dataset windows" + ) + data_id, windowed, row = selected + metadata = config.dataset_metadata + columns = list( + dict.fromkeys( + config.input_columns + config.static_columns + config.target_columns + ) + ) + codec = FixedNumericPreprocessing( + columns, + metadata.selected_columns_statistics, + metadata.normalize_real_columns, + { + column: torch.empty((), dtype=column_data_types[column]).numpy().dtype + for column in columns + }, + ) + seed_inputs = { + key: value[row : row + 1].numpy() for key, value in windowed.inputs.items() + } + seed_mask = windowed.metadata["attention_valid_mask"][row : row + 1].numpy() + seed = codec.seed_from_model( + seed_inputs, seed_mask, config.input_columns, config.static_columns + ) + schema = ( + ExecutionSchema.from_interface( + inferer.inference_model.interface, config.window_view.context_length + ) + if inferer.inference_model_type == "pt" + else inferer.execution_schema + ) + if schema is None: + raise ValueError("numeric rollout requires a model execution schema") + + def validate_history(history): + features, masks = codec.model_inputs(history) + schema.validate( + {key: torch.from_numpy(value) for key, value in features.items()}, + {key: torch.from_numpy(value) for key, value in masks.items()}, + ) + + def predict(history): + inputs, masks = codec.model_inputs(history) + predictions = inferer.infer_generative( + inputs, masks, column_data_types=column_data_types, return_probs=False + ) + decoded = {} + for column in config.target_columns: + value = np.asarray(predictions[column]) + if value.size != 1: + raise ValueError( + f"rollout predictor must produce one scalar for {column!r}" + ) + decoded[column] = float(codec.decode(column, value).reshape(-1)[0]) + return decoded + + # Instantiate only after validating the seed; collect_rollout owns close(), + # including reset, model, adapter, successor validation, and collection failures. + adapter = load_adapter(rollout.adapter) + trajectory = collect_rollout( + predict, + adapter, + seed, + rollout, + config.target_columns, + config.generation_steps, + random_seed=config.seed, + history_validator=validate_history, + metadata={ + "model_id": model_id, + "model_path": config.model_path, + "model_interface": config.model_interface, + "training_objective": config.training_objective, + "target_offset": config.window_view.target_offset, + "context_length": config.window_view.context_length, + "preprocessing": codec.to_dict(), + "rollout": rollout.model_dump(mode="json"), + "collection_budget": config.generation_steps, + "seed": config.seed, + "seed_origin": { + "data_id": data_id, + "seed_index": rollout.seed_index, + "sequence_id": int(windowed.sequence_ids[row]), + "subsequence_id": int(windowed.subsequence_ids[row]), + "model_start_position": int(windowed.model_start_positions[row]), + "window_start_offset": int(windowed.window_start_offsets[row]), + }, + }, + ) + path = normalize_path(rollout.trajectory_path, config.project_root) + save_trajectory(trajectory, path) + logger.info( + f"Wrote {len(trajectory.transitions)} rollout transitions to '{path}'; return={trajectory.total_return}" + ) + + @beartype def get_probs_preds_from_dict( config: Any, @@ -1263,8 +1384,11 @@ def get_probs_preds_from_dict( all_probs_list = {col: [] for col in target_cols} all_preds_list = {col: [] for col in target_cols} + from sequifier.rollout.coordinator import coordinate_steps + metadata_for_step = metadata_np - for i in range(total_steps): + + def predict(): if config.output_probabilities: probs_for_step = inferer.infer_generative( X, @@ -1284,12 +1408,15 @@ def get_probs_preds_from_dict( column_data_types=column_data_types, return_probs=False, ) + return preds_for_step + def execute(preds_for_step, i): + nonlocal X for col in target_cols: all_preds_list[col].append(preds_for_step[col]) if i == (total_steps - 1): - break + return False X_next = {} for col in X.keys(): @@ -1314,6 +1441,9 @@ def get_probs_preds_from_dict( metadata_for_step["attention_valid_mask"] = np.concatenate( [shifted_mask, appended_valid], axis=1 ) + return False + + coordinate_steps(predict, execute, total_steps) final_preds = { col: np.array(preds_list).T.reshape(-1, 1).flatten() diff --git a/src/sequifier/rollout/__init__.py b/src/sequifier/rollout/__init__.py new file mode 100644 index 00000000..67c28b70 --- /dev/null +++ b/src/sequifier/rollout/__init__.py @@ -0,0 +1 @@ +"""Local numeric forecast collection.""" diff --git a/src/sequifier/rollout/contracts.py b/src/sequifier/rollout/contracts.py new file mode 100644 index 00000000..8fb42ea1 --- /dev/null +++ b/src/sequifier/rollout/contracts.py @@ -0,0 +1,42 @@ +"""Adapter values are finite numeric scalars in original (unscaled) units.""" + +from dataclasses import dataclass +from typing import Mapping, Protocol + + +@dataclass(frozen=True) +class ResetRequest: + episode_id: str + seed_history: Mapping[str, tuple[float, ...]] + attention_valid_mask: tuple[bool, ...] + static_inputs: Mapping[str, float] + seed: int + + +@dataclass(frozen=True) +class StepRequest: + episode_id: str + step_id: int + observation: Mapping[str, float] + predictions: Mapping[str, float] + static_inputs: Mapping[str, float] + + +@dataclass(frozen=True) +class StepResult: + external_inputs: Mapping[str, float] + reward: float + terminated: bool = False + truncated: bool = False + + +class EnvironmentAdapter(Protocol): + protocol_version: int + + def reset(self, request: ResetRequest) -> None: + """Initialize from the supplied history; reset must not replace the seed.""" + ... + + def step(self, request: StepRequest) -> StepResult: ... + + def close(self) -> None: ... diff --git a/src/sequifier/rollout/coordinator.py b/src/sequifier/rollout/coordinator.py new file mode 100644 index 00000000..91b81eb3 --- /dev/null +++ b/src/sequifier/rollout/coordinator.py @@ -0,0 +1,166 @@ +"""Shared prediction/execution ordering and single-environment collection.""" + +import math +import warnings +from collections.abc import Callable +from uuid import uuid4 + +from sequifier.config.rollout_config import RolloutConfig +from sequifier.rollout.contracts import ( + EnvironmentAdapter, + ResetRequest, + StepRequest, + StepResult, +) +from sequifier.rollout.factories import validate_adapter +from sequifier.rollout.history import ( + advance_history, + numeric_fields, + numeric_scalar, + validate_seed, +) +from sequifier.trajectory.schema import SeedHistory, Trajectory, Transition + + +def coordinate_steps(predict: Callable, execute: Callable, max_steps: int) -> None: + """Execute every prediction, including the final budgeted one; stop at a boundary.""" + if not isinstance(max_steps, int) or isinstance(max_steps, bool) or max_steps < 1: + raise ValueError("rollout collection budget must be a positive integer") + for step_id in range(max_steps): + prediction = predict() + if execute(prediction, step_id): + break + + +def collect_rollout( + predictor: Callable, + adapter: EnvironmentAdapter, + seed: SeedHistory, + config: RolloutConfig, + prediction_columns: list[str], + max_steps: int, + *, + random_seed: int = 1010, + metadata: dict | None = None, + history_validator: Callable | None = None, +) -> Trajectory: + """Collect one episode fragment. This call owns and always closes the adapter. + + Predictor receives a copy of the current SeedHistory in original units and + returns named scalar forecasts. Conversion to model units belongs in that + predictor wrapper, using fixed training metadata. + """ + failed = False + try: + validate_adapter(adapter) + history = validate_seed(seed) + initial_history = history.model_copy(deep=True) + if history_validator is not None: + history_validator(history.model_copy(deep=True)) + if not prediction_columns or len(prediction_columns) != len( + set(prediction_columns) + ): + raise ValueError("prediction_columns must be nonempty and unique") + config.validate_bindings(list(history.observations), prediction_columns) + episode_id = str(uuid4()) + adapter.reset( + ResetRequest( + episode_id, + {key: tuple(values) for key, values in history.observations.items()}, + tuple(history.attention_valid_mask), + dict(history.static_inputs), + random_seed, + ) + ) + transitions = [] + external_keys = { + binding.key + for binding in config.bindings + if binding.source == "environment" + } + + def predict(): + return numeric_fields( + predictor(history.model_copy(deep=True)), + set(prediction_columns), + "predictions", + ) + + def execute(predictions, step_id): + nonlocal history + observation = { + key: values[-1] for key, values in history.observations.items() + } + result = adapter.step( + StepRequest( + episode_id, + step_id, + dict(observation), + dict(predictions), + dict(history.static_inputs), + ) + ) + if not isinstance(result, StepResult): + raise ValueError("adapter.step must return StepResult") + if not isinstance(result.terminated, bool) or not isinstance( + result.truncated, bool + ): + raise ValueError( + "adapter termination and truncation flags must be separate booleans" + ) + external = numeric_fields( + result.external_inputs, external_keys, "adapter.external_inputs" + ) + reward = numeric_scalar(result.reward, "adapter.reward") + successor = { + binding.input: ( + predictions if binding.source == "prediction" else external + )[binding.key] + for binding in config.bindings + } + truncated = result.truncated or ( + config.max_episode_steps is not None + and step_id + 1 >= config.max_episode_steps + ) + next_history = advance_history(history, successor) + if history_validator is not None: + history_validator(next_history.model_copy(deep=True)) + transitions.append( + Transition( + episode_id=episode_id, + step_id=step_id, + observation=observation, + predictions=predictions, + external_inputs=external, + reward=reward, + successor_observation=successor, + terminated=result.terminated, + truncated=truncated, + bootstrap_allowed=not result.terminated, + ) + ) + history = next_history + return result.terminated or truncated + + coordinate_steps(predict, execute, max_steps) + last = transitions[-1] + return Trajectory( + episode_id=episode_id, + seed=initial_history, + bindings=config.bindings, + prediction_columns=prediction_columns, + transitions=transitions, + collection_cutoff=not (last.terminated or last.truncated), + total_return=math.fsum(transition.reward for transition in transitions), + metadata=metadata or {}, + ) + except BaseException: + failed = True + raise + finally: + try: + adapter.close() + except Exception as exc: + if not failed: + raise + warnings.warn(f"Adapter close also failed: {exc}", RuntimeWarning) diff --git a/src/sequifier/rollout/dependent.py b/src/sequifier/rollout/dependent.py new file mode 100644 index 00000000..84e3225d --- /dev/null +++ b/src/sequifier/rollout/dependent.py @@ -0,0 +1,50 @@ +"""Small local dependent-feature adapter, also usable as an integration example.""" + +from sequifier.rollout.contracts import ResetRequest, StepRequest, StepResult + + +class LinearDependentAdapter: + """Supply a scaled prediction and use that prediction as the reward.""" + + protocol_version = 1 + + def __init__( + self, prediction="x", external_input="d", multiplier=2.0, terminate_after=3 + ): + if ( + not isinstance(terminate_after, int) + or isinstance(terminate_after, bool) + or terminate_after < 1 + ): + raise ValueError("terminate_after must be a positive integer") + self.prediction = prediction + self.external_input = external_input + self.multiplier = multiplier + self.terminate_after = terminate_after + self.closed = False + self.steps = 0 + self.episode_id = None + + def reset(self, request: ResetRequest) -> None: + if self.closed: + raise ValueError("adapter is closed") + self.steps = 0 + self.episode_id = request.episode_id + + def step(self, request: StepRequest) -> StepResult: + if ( + self.closed + or self.episode_id != request.episode_id + or self.steps >= self.terminate_after + ): + raise ValueError("adapter must be reset before stepping an active episode") + self.steps += 1 + prediction = request.predictions[self.prediction] + return StepResult( + {self.external_input: self.multiplier * prediction}, + reward=prediction, + terminated=self.steps == self.terminate_after, + ) + + def close(self) -> None: + self.closed = True diff --git a/src/sequifier/rollout/factories.py b/src/sequifier/rollout/factories.py new file mode 100644 index 00000000..4bd812ec --- /dev/null +++ b/src/sequifier/rollout/factories.py @@ -0,0 +1,29 @@ +"""Persistent adapter loading; factories receive their config as keyword arguments.""" + +from importlib import import_module +from typing import cast + +from sequifier.config.rollout_config import AdapterConfig +from sequifier.rollout.contracts import EnvironmentAdapter + + +def validate_adapter(adapter): + protocol_version = getattr(adapter, "protocol_version", None) + if ( + not isinstance(protocol_version, int) + or isinstance(protocol_version, bool) + or protocol_version != 1 + ): + raise ValueError("rollout adapter must declare protocol_version = 1") + for method in ("reset", "step", "close"): + if not callable(getattr(adapter, method, None)): + raise ValueError(f"rollout adapter requires a callable {method} method") + + +def load_adapter(spec: AdapterConfig) -> EnvironmentAdapter: + module, attribute = spec.factory.split(":") + factory = getattr(import_module(module), attribute) + if not callable(factory): + raise ValueError(f"Adapter factory {spec.factory!r} is not callable") + # Runtime validation happens inside collect_rollout's cleanup boundary. + return cast(EnvironmentAdapter, factory(**spec.config)) diff --git a/src/sequifier/rollout/history.py b/src/sequifier/rollout/history.py new file mode 100644 index 00000000..55f461bd --- /dev/null +++ b/src/sequifier/rollout/history.py @@ -0,0 +1,44 @@ +"""Validated seed windows and atomic numeric feature/mask advancement.""" + +import math +from collections.abc import Mapping +from numbers import Real + +from sequifier.trajectory.schema import SeedHistory + + +def numeric_scalar(value, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{label} must be a finite numeric scalar, got {value!r}") + try: + number = float(value) + except (ValueError, OverflowError) as exc: + raise ValueError(f"{label} is outside the supported numeric range") from exc + if not math.isfinite(number): + raise ValueError(f"{label} must be finite, got {value!r}") + return number + + +def numeric_fields(values, expected: set[str], label: str) -> dict[str, float]: + if not isinstance(values, Mapping) or set(values) != expected: + raise ValueError( + f"{label} must supply exactly these fields: {sorted(expected)}" + ) + return { + key: numeric_scalar(values[key], f"{label}.{key}") for key in sorted(expected) + } + + +def validate_seed(seed: SeedHistory) -> SeedHistory: + return SeedHistory.model_validate(seed.model_dump()) + + +def advance_history(history: SeedHistory, successor: dict[str, float]) -> SeedHistory: + return SeedHistory( + observations={ + key: values[1:] + [successor[key]] + for key, values in history.observations.items() + }, + attention_valid_mask=history.attention_valid_mask[1:] + [True], + static_inputs=dict(history.static_inputs), + ) diff --git a/src/sequifier/rollout/preprocessing.py b/src/sequifier/rollout/preprocessing.py new file mode 100644 index 00000000..e241c8b2 --- /dev/null +++ b/src/sequifier/rollout/preprocessing.py @@ -0,0 +1,114 @@ +"""Fixed numeric training transforms; never fit on seeds or adapter values.""" + +import math + +import numpy as np + +from sequifier.rollout.history import validate_seed +from sequifier.trajectory.schema import SeedHistory + + +class FixedNumericPreprocessing: + def __init__(self, columns, statistics, normalize, dtypes): + self.normalize = normalize + self.statistics = {} + self.dtypes = {column: np.dtype(dtypes[column]) for column in columns} + for column in columns: + if self.dtypes[column].kind != "f": + raise ValueError( + f"rollout field {column!r} must have a floating numeric dtype" + ) + if normalize: + stats = statistics.get(column, {}) + mean, std = stats.get("mean"), stats.get("std") + if ( + isinstance(mean, bool) + or isinstance(std, bool) + or not isinstance(mean, (int, float)) + or not isinstance(std, (int, float)) + or not math.isfinite(mean) + or not math.isfinite(std) + or std < 0 + ): + raise ValueError( + f"rollout requires finite training mean/std for {column!r}" + ) + self.statistics[column] = {"mean": float(mean), "std": float(std)} + + def decode(self, column, values): + values = np.asarray(values, dtype=np.float64) + if self.normalize: + stats = self.statistics[column] + values = values * (stats["std"] + 1e-9) + stats["mean"] + if not np.isfinite(values).all(): + raise ValueError(f"rollout field {column!r} decoded to non-finite values") + return values + + def encode(self, column, values): + values = np.asarray(values, dtype=np.float64) + if self.normalize: + stats = self.statistics[column] + values = (values - stats["mean"]) / (stats["std"] + 1e-9) + with np.errstate(over="ignore", invalid="ignore"): + values = values.astype(self.dtypes[column]) + if not np.isfinite(values).all(): + raise ValueError( + f"rollout field {column!r} is incompatible with its model dtype/scale" + ) + return values + + def seed_from_model(self, inputs, mask, input_columns, static_columns): + if mask.ndim != 2 or mask.shape[0] != 1 or mask.dtype != np.bool_: + raise ValueError( + "rollout seed requires a boolean attention mask for one window" + ) + observations, static = {}, {} + for column in input_columns: + values = inputs[column] + if values.shape != mask.shape: + raise ValueError( + f"rollout seed {column!r} must be a scalar feature window matching its mask" + ) + # Padding is represented by neutral domain values, never decoded as observations. + decoded = np.zeros(values.shape, dtype=np.float64) + decoded[mask] = self.decode(column, values[mask]) + observations[column] = decoded[0].tolist() + for column in static_columns: + if inputs[column].shape != (1,): + raise ValueError( + f"rollout static input {column!r} must have shape (1,)" + ) + static[column] = float(self.decode(column, inputs[column])[0]) + return validate_seed( + SeedHistory( + observations=observations, + attention_valid_mask=mask[0].tolist(), + static_inputs=static, + ) + ) + + def model_inputs(self, history: SeedHistory): + mask = np.asarray([history.attention_valid_mask], dtype=np.bool_) + features = {} + for column, values in history.observations.items(): + encoded = np.zeros(mask.shape, dtype=self.dtypes[column]) + # False-mask padding is not a physical value to normalize. + encoded[mask] = self.encode(column, np.asarray([values])[mask]) + features[column] = encoded + features.update( + { + column: self.encode(column, [value]) + for column, value in history.static_inputs.items() + } + ) + return features, {"attention_valid_mask": mask} + + def to_dict(self): + return { + "normalize_real_columns": self.normalize, + "selected_columns_statistics": self.statistics, + "model_dtypes": { + column: dtype.name for column, dtype in self.dtypes.items() + }, + "epsilon": 1e-9, + } diff --git a/src/sequifier/rollout_api.py b/src/sequifier/rollout_api.py new file mode 100644 index 00000000..0c2ec045 --- /dev/null +++ b/src/sequifier/rollout_api.py @@ -0,0 +1,33 @@ +"""Public API for local numeric forecast adapters and trajectory collection.""" + +from sequifier.config.rollout_config import AdapterConfig, InputBinding, RolloutConfig +from sequifier.rollout.contracts import ( + EnvironmentAdapter, + ResetRequest, + StepRequest, + StepResult, +) +from sequifier.rollout.coordinator import collect_rollout +from sequifier.trajectory import ( + Trajectory, + load_trajectory, + save_trajectory, + validate_trajectory, +) +from sequifier.trajectory.schema import SeedHistory + +__all__ = [ + "AdapterConfig", + "InputBinding", + "RolloutConfig", + "EnvironmentAdapter", + "ResetRequest", + "StepRequest", + "StepResult", + "SeedHistory", + "Trajectory", + "collect_rollout", + "load_trajectory", + "save_trajectory", + "validate_trajectory", +] diff --git a/src/sequifier/trajectory/__init__.py b/src/sequifier/trajectory/__init__.py new file mode 100644 index 00000000..e1d14b8e --- /dev/null +++ b/src/sequifier/trajectory/__init__.py @@ -0,0 +1,6 @@ +"""Versioned numeric trajectory artifacts.""" + +from sequifier.trajectory.schema import Trajectory, validate_trajectory +from sequifier.trajectory.store import load_trajectory, save_trajectory + +__all__ = ["Trajectory", "validate_trajectory", "load_trajectory", "save_trajectory"] diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py new file mode 100644 index 00000000..f340a89b --- /dev/null +++ b/src/sequifier/trajectory/schema.py @@ -0,0 +1,147 @@ +"""Inspectable version-1 single-episode forecast trajectory.""" + +import json +import math +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, FiniteFloat, model_validator + +from sequifier.config.rollout_config import InputBinding + + +class Record(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class SeedHistory(Record): + observations: dict[str, list[FiniteFloat]] + attention_valid_mask: list[bool] + static_inputs: dict[str, FiniteFloat] + + @model_validator(mode="after") + def validate_window(self): + mask = self.attention_valid_mask + if ( + not mask + or not mask[-1] + or any(mask[i] and not mask[i + 1] for i in range(len(mask) - 1)) + ): + raise ValueError("seed mask must be left padding followed by valid history") + if not self.observations or any( + len(values) != len(mask) for values in self.observations.values() + ): + raise ValueError( + "seed feature histories must be nonempty and match the attention mask length" + ) + if set(self.observations) & set(self.static_inputs): + raise ValueError("static inputs must be separate from changing inputs") + return self + + +class Transition(Record): + episode_id: str + step_id: int = Field(ge=0) + observation: dict[str, FiniteFloat] + predictions: dict[str, FiniteFloat] + external_inputs: dict[str, FiniteFloat] + reward: FiniteFloat + successor_observation: dict[str, FiniteFloat] + terminated: bool + truncated: bool + bootstrap_allowed: bool + + +class Trajectory(Record): + format: Literal["sequifier.numeric-forecast-trajectory"] = ( + "sequifier.numeric-forecast-trajectory" + ) + version: Literal[1] = 1 + episode_id: str = Field(min_length=1) + seed: SeedHistory + bindings: list[InputBinding] + prediction_columns: list[str] + transitions: list[Transition] = Field(min_length=1) + collection_cutoff: bool + total_return: FiniteFloat + metadata: dict[str, Any] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_temporal_contract(self): + # Metadata must round-trip through the same portable JSON representation. + json.dumps(self.metadata, allow_nan=False) + inputs = set(self.seed.observations) + if not self.prediction_columns or len(self.prediction_columns) != len( + set(self.prediction_columns) + ): + raise ValueError("prediction columns must be nonempty and unique") + bound = [binding.input for binding in self.bindings] + if len(bound) != len(set(bound)) or set(bound) != inputs: + raise ValueError( + "trajectory requires exactly one producer per changing input" + ) + external_keys = { + binding.key for binding in self.bindings if binding.source == "environment" + } + observation = { + key: values[-1] for key, values in self.seed.observations.items() + } + for index, transition in enumerate(self.transitions): + if transition.episode_id != self.episode_id or transition.step_id != index: + raise ValueError("transition episode/step IDs are not contiguous") + if transition.observation != observation: + raise ValueError( + f"transition {index} observation breaks trajectory continuity" + ) + if set(transition.predictions) != set(self.prediction_columns): + raise ValueError( + f"transition {index} prediction fields do not match the contract" + ) + if set(transition.external_inputs) != external_keys: + raise ValueError( + f"transition {index} external input fields do not match bindings" + ) + if set(transition.successor_observation) != inputs: + raise ValueError( + f"transition {index} successor fields do not match inputs" + ) + for binding in self.bindings: + producer = ( + transition.predictions + if binding.source == "prediction" + else transition.external_inputs + ) + if ( + binding.key not in producer + or transition.successor_observation[binding.input] + != producer[binding.key] + ): + raise ValueError( + f"transition {index} successor violates binding for {binding.input!r}" + ) + if transition.bootstrap_allowed != (not transition.terminated): + raise ValueError( + "termination must disable bootstrap; truncation preserves the final observation" + ) + if (transition.terminated or transition.truncated) and index != len( + self.transitions + ) - 1: + raise ValueError("trajectory contains a step after an episode boundary") + observation = transition.successor_observation + last = self.transitions[-1] + if self.collection_cutoff != (not (last.terminated or last.truncated)): + raise ValueError("collection cutoff must leave the episode unfinished") + expected_return = math.fsum( + transition.reward for transition in self.transitions + ) + if not math.isclose( + self.total_return, expected_return, rel_tol=1e-12, abs_tol=1e-12 + ): + raise ValueError("total_return does not equal the sum of rewards") + return self + + +def validate_trajectory(value: Trajectory | dict) -> Trajectory: + """Revalidate even an existing (possibly mutated) trajectory object.""" + return Trajectory.model_validate( + value.model_dump() if isinstance(value, Trajectory) else value + ) diff --git a/src/sequifier/trajectory/store.py b/src/sequifier/trajectory/store.py new file mode 100644 index 00000000..681d4af6 --- /dev/null +++ b/src/sequifier/trajectory/store.py @@ -0,0 +1,33 @@ +"""Atomic JSON storage, with validation at both read and write boundaries.""" + +import json +import os +import tempfile +from pathlib import Path + +from sequifier.trajectory.schema import Trajectory, validate_trajectory + + +def save_trajectory(trajectory: Trajectory, path: str | Path) -> None: + validated = validate_trajectory(trajectory) + # Reject non-JSON/non-finite metadata as well as invalid numeric records. + payload = json.dumps(validated.model_dump(mode="json"), indent=2, allow_nan=False) + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=destination.parent, delete=False + ) as stream: + temporary = stream.name + stream.write(payload + "\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, destination) + finally: + if temporary is not None and os.path.exists(temporary): + os.unlink(temporary) + + +def load_trajectory(path: str | Path) -> Trajectory: + return validate_trajectory(json.loads(Path(path).read_text(encoding="utf-8"))) From 255010de1809204bf2d47e2a5d8a6e066c86c574 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 10:56:01 +0100 Subject: [PATCH 02/71] policy increment WIP --- .pre-commit-config.yaml | 2 +- documentation/configs/train.md | 4 + .../configs/policy-control.yaml | 17 + ...nforcement-learning-implementation-plan.md | 9 + documentation/policy-learning.md | 196 ++++++++++++ documentation/rollout.md | 3 + hooks/consolidate_docs.py | 1 + src/sequifier/config/infer_config.py | 4 + src/sequifier/config/policy_config.py | 85 +++++ src/sequifier/config/rollout_config.py | 2 +- src/sequifier/consolidated-docs.md | 202 ++++++++++++ src/sequifier/docs.py | 1 + src/sequifier/rl/__init__.py | 1 + src/sequifier/rl/checkpoint.py | 66 ++++ src/sequifier/rl/distributions.py | 51 +++ src/sequifier/rl/experiment.py | 294 ++++++++++++++++++ src/sequifier/rl/history.py | 107 +++++++ src/sequifier/rl/policy.py | 139 +++++++++ src/sequifier/rl/runtime.py | 192 ++++++++++++ src/sequifier/rl/training.py | 144 +++++++++ src/sequifier/rl_api.py | 50 +++ src/sequifier/rollout/contracts.py | 15 +- src/sequifier/rollout/control.py | 64 ++++ src/sequifier/rollout/coordinator.py | 27 +- src/sequifier/rollout_api.py | 2 + src/sequifier/sequifier.py | 12 +- src/sequifier/trajectory/dataset.py | 218 +++++++++++++ src/sequifier/trajectory/returns.py | 50 +++ src/sequifier/trajectory/schema.py | 94 +++++- 29 files changed, 2035 insertions(+), 17 deletions(-) create mode 100644 documentation/example_inputs/configs/policy-control.yaml create mode 100644 documentation/policy-learning.md create mode 100644 src/sequifier/config/policy_config.py create mode 100644 src/sequifier/rl/__init__.py create mode 100644 src/sequifier/rl/checkpoint.py create mode 100644 src/sequifier/rl/distributions.py create mode 100644 src/sequifier/rl/experiment.py create mode 100644 src/sequifier/rl/history.py create mode 100644 src/sequifier/rl/policy.py create mode 100644 src/sequifier/rl/runtime.py create mode 100644 src/sequifier/rl/training.py create mode 100644 src/sequifier/rl_api.py create mode 100644 src/sequifier/rollout/control.py create mode 100644 src/sequifier/trajectory/dataset.py create mode 100644 src/sequifier/trajectory/returns.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c49d2de..ccabc291 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,5 +64,5 @@ repos: name: Consolidate configuration documentation entry: python hooks/consolidate_docs.py language: python - files: ^(README\.md|documentation/configs/(preprocess|train|infer|hyperparameter-search)\.md|documentation/commands/visualize-training\.md|documentation/training/multi-gpu-training\.md|src/sequifier/consolidated-docs\.md)$ + files: ^(README\.md|documentation/configs/(preprocess|train|infer|hyperparameter-search)\.md|documentation/policy-learning\.md|documentation/commands/visualize-training\.md|documentation/training/multi-gpu-training\.md|src/sequifier/consolidated-docs\.md)$ pass_filenames: false diff --git a/documentation/configs/train.md b/documentation/configs/train.md index c7d15577..5893fbfb 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -1,5 +1,9 @@ # Train Command Guide +For trajectory-backed categorical control learning and the local RC/BC +comparison, see [policy learning](../policy-learning.md) and the separate +`sequifier policy` command. + `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated adapter, and a decoder. Datasets own data and training policy; `model` diff --git a/documentation/example_inputs/configs/policy-control.yaml b/documentation/example_inputs/configs/policy-control.yaml new file mode 100644 index 00000000..a3b1432f --- /dev/null +++ b/documentation/example_inputs/configs/policy-control.yaml @@ -0,0 +1,17 @@ +project_root: . +output_dir: outputs/numeric-policy +horizon: 3 +repetitions: 16 +context_length: 4 +hidden_size: 32 +returns: + gamma: 1.0 + scale: 1.0 +training: + epochs: 80 + batch_size: 64 + learning_rate: 0.005 + seed: 1010 + validation_fraction: 0.2 +requested_returns: [0.0, 3.0] +sampling_episodes: 8 diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md index 069b00cd..c522c5b3 100644 --- a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md +++ b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md @@ -9,6 +9,15 @@ See [local forecast rollouts](../rollout.md) for configuration and public APIs. The broader stages below remain planned. No tests or acceptance fixtures were written or run for this slice, as requested. +Second implementation slice: episode-split trajectory datasets, causal +current-observation/previous-action windows, discounted return labels, a single +categorical actor, return-conditioned supervised training and BC, local actor +artifacts, and execution through the same coordinator. The tiny binary-control +workflow records achieved returns after reloading both actors. See +[policy learning](../policy-learning.md). This slice has not been executed or +acceptance-tested; no tests were written or run, as requested. IQL and the +remaining broader stages are still planned; RC is an independent route. + Deliver this as one coordinated update, built and verified in the three dependency-ordered stages below. Completion means all three stages work together; stage boundaries are engineering checkpoints, not separate proposed releases. This document specifies new behavior and proposed API names, not functionality already present. ## 1. Outcome and scope diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md new file mode 100644 index 00000000..a2139df2 --- /dev/null +++ b/documentation/policy-learning.md @@ -0,0 +1,196 @@ +# Policy Learning Command Guide + +`sequifier policy` runs a local, CPU-only categorical control experiment: + +```console +sequifier policy --config-path documentation/example_inputs/configs/policy-control.yaml +``` + +It collects trajectories, computes return-to-go labels, splits by episode, +trains return-conditioned and behavior-cloning actors, saves/reloads both, and +executes them through the existing rollout coordinator. `report.json` records +the achieved returns. This command is separate from forecast model training and +inference; it does not change `target_offset` or convert forecast checkpoints +into policies. `sequifier docs policy` prints this guide. + +## Task and collected data + +The deterministic binary control task starts at `position=0` with three decisions +remaining. Action `0` holds position; action `1` increments it. Each action earns +its numeric value as reward. Observations contain current `position` and +`remaining_steps`. The environment terminates after its last decision and returns +the actual final successor. The horizon is configurable from 1 through 8. + +Collection enumerates every binary action sequence, repeating each 16 times by +default. For horizon 3 that gives 128 episodes and 384 executed decisions from +the same starting conditions, with returns from 0 through 3 and a behavior mean +of 1.5. Repetition gives conflicting action labels for identical observations +while retaining examples at each return level. The dataset manifest lists the +exact collected files and their episode IDs; stale files are not added to the +current experiment. + +The example configuration writes these files beneath `outputs/numeric-policy`: + +- `dataset/sequence-*.json`: recorded episodes in original numeric units. +- `dataset/manifest.json`: exact source files, episode IDs, and raw returns. +- `dataset/return_labels.json`: the return definition and raw per-decision labels. +- `return_conditioned.pt` and `behavior_cloning.pt`: local actor artifacts. +- `evaluation/*.json`: trajectories executed using the reloaded actors. +- `report.json`: episode partitions, per-epoch action likelihood, checkpoint + selection, greedy/sampled actions, achieved returns, and requested-return errors. + +The report distinguishes raw episode reward sums from discounted achieved +returns. It compares both actors at the same requested returns, initial states, +and sampling seeds. The BC actor does not receive the requested return. This is +a controlled learning comparison; performance depends on training configuration +and is recorded rather than assumed. + +## Policy alignment and causal windows + +At decision position `t`, the policy receives `o_t`, `a_(t-1)`, and, for return +conditioning, `G_t`. The current action `a_t` is a separate target. The first +previous action is a dedicated start-token ID equal to the number of executable +actions. It cannot be selected by the action head. + +For the sequence `[1, 0, 1]`, gamma 1, and the default task: + +| Decision | Current observation | Previous action ID | Desired return | Action target | +| --- | --- | --- | --- | --- | +| 0 | `(position=0, remaining_steps=3)` | 2 (start) | 2 | 1 | +| 1 | `(position=1, remaining_steps=2)` | 1 | 1 | 0 | +| 2 | `(position=1, remaining_steps=1)` | 0 | 1 | 1 | + +Offline and online execution use `build_policy_inputs` to construct the same +left-padded causal windows. Observations, previous actions, and returns are +trimmed together. Padding has a false attention mask. Only the last position +of each training example has a true action-likelihood mask, so every recorded +decision contributes once even when context windows overlap. + +The actor uses numeric projection, previous-action embedding, a small causal GRU, +and one categorical head. Padded positions do not advance its hidden state. +Forward calls take `PolicyInputs`, which contain no action labels or observed +current rewards. A change to a current-action label cannot affect that position's +inputs; future observations are excluded from its window. Return-to-go is the +intentional reward-derived training condition, with requested budgets replacing +it online. Changing rewards can therefore change the return condition. + +Observation/static normalization is fit on the training episode partition only. +Its fixed means/scales are model buffers, saved with the actor. The RC and BC +actors use the same architecture, initialization seed, episode split, batch +order, and optimization configuration. BC disables the return channel by +feeding zero; it retains the same parameter count. + +## Return definitions and episode boundaries + +Raw labels use `G_t = r_t + gamma * G_(t+1)` and terminal `G_T = 0`. +For rewards `[1, 2, 3]`, gamma 1 produces `[6, 5, 3]`; gamma 0.5 produces +`[2.75, 3.5, 3]`. `returns.gamma` is explicit in `(0, 1]`, defaulting to 1. +`returns.scale` divides returns only at the numeric encoder boundary and defaults +to 1. Artifacts and execution traces preserve raw reward units. + +By default, only trajectories with true termination have exact return labels. +Collection cutoffs and truncation without termination are excluded and their +episode IDs are exposed by the dataset. An explicit `returns.finite_horizon: H` +allows labeling an observed prefix of exactly H decisions, including a cutoff +at H. A fragment shorter than H is still excluded unless it truly terminates. +No unknown tail is silently assigned zero. For online use of this objective, +`rollout.max_episode_steps` must equal H. + +Execution starts from the caller's requested return and updates the raw budget +after each executed reward: + +```text +G_next = (G_current - reward) / gamma +``` + +The request is not clamped to an inferred feasible range. Each episode reset +restores the requested budget, clears previous-action/history state, and uses +the start token again. A collection cutoff preserves the unfinished episode +flag; it does not fabricate a reset. Each call collects one episode/fragment, +so collecting another episode uses a fresh adapter and an explicit reset. + +Policy trajectories use `format: sequifier.numeric-policy-trajectory`, version 1, +with the same seed/transition schema and storage as numeric forecast trajectories. +`mode: policy` makes this alignment explicit. Forecast artifacts retain their +existing format and require an explicit declaration before their executed +predictions can be used as demonstration actions. +Policy episodes start with one valid observation because historical actions +before reset are unknown. The actor builds subsequent history from executed +transitions. + +Execution metadata stores the full policy/return configuration, initial request, +remaining budgets (including the final successor budget), previous-action IDs, +and selected-action log probabilities. The trajectory validator checks action +vocabulary, previous-action alignment, budget counts, and reward/gamma arithmetic +in addition to ordinary continuity, reward, and boundary validation. Both flags +may be true; termination takes precedence for bootstrap. Adapter closure remains +owned by the shared coordinator on success and failure. + +## Reusable Python interfaces + +The public entry points live in `sequifier.rl_api`. To train on compatible saved +policy episodes: + +```python +from sequifier.rl_api import ( + CategoricalPolicyConfig, PolicyTrainingConfig, TrajectoryDataset, + load_policy, save_policy, split_by_episode, train_policy, +) + +spec = CategoricalPolicyConfig( + observation_columns=["position", "remaining_steps"], + action_column="action", + action_values=[0.0, 1.0], +) +dataset = TrajectoryDataset(episode_paths, spec) +training, validation = split_by_episode(dataset, validation_fraction=0.2, seed=1010) +result = train_policy(training, validation, PolicyTrainingConfig()) +save_policy(result.policy, "actor.pt") +actor = load_policy("actor.pt") +``` + +First-increment forecast trajectories can also supply demonstrations when their +executed numeric predictions have a declared finite categorical vocabulary: +pass `action_semantics="executed_predictions"` to `TrajectoryDataset` and set +the appropriate `action_column` and `action_values` in the policy specification. +This is an explicit data interpretation, not a conversion of forecast model +alignment or weights. It uses the latest seed observation as decision zero, +discards earlier seed observations whose actions were never recorded, and +starts previous-action history with the start token. Episode splitting preserves +this declaration. Observation and action names may coincide because their +values are read from separate observation/target records; current labels are +never copied into current inputs. + +Action values are original numeric domain values; the distribution uses their +ordered categorical IDs internally. `actor.distribution(inputs)` returns a +`CategoricalActionDistribution` supporting `mode()`, `sample(generator)`, +`log_prob(recorded_action_ids)`, and `entropy()`. `masked_action_likelihood` +uses the dataset's independent action-valid mask. Training calls the existing +`OptimizationRuntime` for backward, clipping, optimizer, scaler, and scheduler +steps. The validation partition selects the saved best actor by action NLL. + +```python +from sequifier.rl_api import collect_policy_rollout +from sequifier.rollout.control import BinaryControlAdapter +from sequifier.rl.experiment import control_rollout_config, control_seed +from sequifier.rollout_api import save_trajectory + +rollout = control_rollout_config("execution.json", horizon=3) +trajectory = collect_policy_rollout( + actor, BinaryControlAdapter(horizon=3), control_seed(horizon=3), + rollout, max_steps=3, desired_return=3.0, deterministic=True, +) +save_trajectory(trajectory, rollout.trajectory_path) +``` + +Set `return_conditioned=False` in the policy specification to train BC, and omit +`desired_return` when executing it. The actor artifacts contain validated CPU +weights, normalization buffers, action vocabulary, policy alignment, and return +semantics. Loading uses strict state-dict matching. These local inference +artifacts do not contain optimizer/RNG state for exact training resumption and +do not use forecast export or inference formats. + +This increment has no critics, Q networks, PPO buffers, distributed execution, +or policy export additions. Return conditioning and unconditioned behavior +cloning are independent actor routes. Future offline IQL can reuse the +trajectory windows and distribution interface without requiring RC pretraining. diff --git a/documentation/rollout.md b/documentation/rollout.md index b66e0544..dccde7de 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -1,5 +1,8 @@ # Local numeric forecast rollouts +For trajectory-backed categorical policy training and return-conditioned +execution through the same coordinator, see [policy learning](policy-learning.md). + The first rollout slice collects one seed-initialized episode or episode fragment on CPU. It supports real scalar features and real static conditioning, with one-step `causal` or `final_value` forecast models (`target_offset: 1`). Model diff --git a/hooks/consolidate_docs.py b/hooks/consolidate_docs.py index abd9a867..ff918528 100644 --- a/hooks/consolidate_docs.py +++ b/hooks/consolidate_docs.py @@ -10,6 +10,7 @@ "documentation/commands/visualize-training.md", "documentation/configs/hyperparameter-search.md", "documentation/training/multi-gpu-training.md", + "documentation/policy-learning.md", ] OUTPUT_FILE = "src/sequifier/consolidated-docs.md" diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 084fe91c..dadc6531 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -1013,6 +1013,10 @@ class ResolvedInferenceConfig(_InferenceConfigBase[str, list[str], dict[str, str def validate_numeric_rollout(self): if self.rollout is None: return self + if self.rollout.mode != "forecast": + raise ValueError( + "policy alignment uses the policy API, not forecast inference" + ) if not self.autoregressive or self.generation_steps is None: raise ValueError( "rollout requires autoregressive: true and generation_steps" diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py new file mode 100644 index 00000000..97825d02 --- /dev/null +++ b/src/sequifier/config/policy_config.py @@ -0,0 +1,85 @@ +"""Standalone, CPU-only categorical policy learning configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, FiniteFloat, model_validator + +from sequifier.config.components import ComponentSpec + + +class PolicyConfigBase(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class ReturnDefinition(PolicyConfigBase): + gamma: float = Field(default=1.0, gt=0, le=1, allow_inf_nan=False) + scale: float = Field(default=1.0, gt=0, allow_inf_nan=False) + finite_horizon: int | None = Field(default=None, gt=0) + + +class CategoricalPolicyConfig(PolicyConfigBase): + alignment: Literal["current_observation_previous_action"] = ( + "current_observation_previous_action" + ) + observation_columns: list[str] = Field(min_length=1) + static_columns: list[str] = Field(default_factory=list) + action_column: str = Field(default="action", min_length=1) + action_values: list[FiniteFloat] = Field( + default_factory=lambda: [0.0, 1.0], min_length=2 + ) + context_length: int = Field(default=4, gt=0) + hidden_size: int = Field(default=32, gt=0) + return_conditioned: bool = True + returns: ReturnDefinition = Field(default_factory=ReturnDefinition) + + @model_validator(mode="after") + def validate_names(self): + names = self.observation_columns + self.static_columns + if any(not name for name in names) or len(names) != len(set(names)): + raise ValueError( + "policy observation/static columns must be nonempty and disjoint" + ) + if len(self.action_values) != len(set(self.action_values)): + raise ValueError("policy action_values must be unique") + return self + + +class PolicyTrainingConfig(PolicyConfigBase): + epochs: int = Field(default=80, gt=0) + batch_size: int = Field(default=64, gt=0) + seed: int = Field(default=1010, ge=0) + validation_fraction: float = Field(default=0.2, gt=0, lt=1) + learning_rate: float = Field(default=0.005, gt=0, allow_inf_nan=False) + gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) + optimizer: ComponentSpec = Field(default_factory=lambda: ComponentSpec(name="Adam")) + scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + scheduler_step_on: Literal["epoch"] = "epoch" + accumulation_steps: Literal[1] = 1 + layer_type_dtypes: None = None + + +class ControlExperimentConfig(PolicyConfigBase): + project_root: str = "." + output_dir: str = "outputs/numeric-policy" + horizon: int = Field(default=3, ge=1, le=8) + repetitions: int = Field(default=16, ge=2) + context_length: int = Field(default=4, gt=0) + hidden_size: int = Field(default=32, gt=0) + returns: ReturnDefinition = Field(default_factory=ReturnDefinition) + training: PolicyTrainingConfig = Field(default_factory=PolicyTrainingConfig) + requested_returns: list[FiniteFloat] | None = None + sampling_episodes: int = Field(default=8, gt=0) + + @model_validator(mode="after") + def validate_horizon(self): + if self.returns.finite_horizon not in (None, self.horizon): + raise ValueError( + "control experiment finite_horizon must match its task horizon" + ) + if self.requested_returns == []: + raise ValueError("requested_returns must be nonempty when supplied") + return self diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index 9d9941ca..cb60c27c 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -32,7 +32,7 @@ def validate_factory(cls, value: str) -> str: class RolloutConfig(BaseModel): model_config = ConfigDict(extra="forbid") - mode: Literal["forecast"] = "forecast" + mode: Literal["forecast", "policy"] = "forecast" adapter: AdapterConfig bindings: list[InputBinding] trajectory_path: str = Field(min_length=1) diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index dad283dc..93bfeda8 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -936,6 +936,10 @@ features from it. See [the training guide](train.md#depth-encoders-and-nested-co # Train Command Guide +For trajectory-backed categorical control learning and the local RC/BC +comparison, see [policy learning](../policy-learning.md) and the separate +`sequifier policy` command. + `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated adapter, and a decoder. Datasets own data and training policy; `model` @@ -2284,3 +2288,201 @@ srun torchrun \ * **Batch Size:** The `batch_size` in your `train.yaml` is the **per-process** batch size. If `batch_size` is 100 and `world_size` is 32, each synchronized backward pass covers 3,200 samples. With gradient accumulation, the samples per optimizer update are `batch_size * world_size * accumulation_steps` (apart from a final partial accumulation window). * **Learning Rate:** You may need to scale your `learning_rate` up if you drastically increase your global batch size via distributed training. * **Data Access:** All nodes must have access to the same shared filesystem (e.g., NFS, GPFS) where the `project_root` and the sharded preprocessing output are stored. + + +# Policy Learning Command Guide + +`sequifier policy` runs a local, CPU-only categorical control experiment: + +```console +sequifier policy --config-path documentation/example_inputs/configs/policy-control.yaml +``` + +It collects trajectories, computes return-to-go labels, splits by episode, +trains return-conditioned and behavior-cloning actors, saves/reloads both, and +executes them through the existing rollout coordinator. `report.json` records +the achieved returns. This command is separate from forecast model training and +inference; it does not change `target_offset` or convert forecast checkpoints +into policies. `sequifier docs policy` prints this guide. + +## Task and collected data + +The deterministic binary control task starts at `position=0` with three decisions +remaining. Action `0` holds position; action `1` increments it. Each action earns +its numeric value as reward. Observations contain current `position` and +`remaining_steps`. The environment terminates after its last decision and returns +the actual final successor. The horizon is configurable from 1 through 8. + +Collection enumerates every binary action sequence, repeating each 16 times by +default. For horizon 3 that gives 128 episodes and 384 executed decisions from +the same starting conditions, with returns from 0 through 3 and a behavior mean +of 1.5. Repetition gives conflicting action labels for identical observations +while retaining examples at each return level. The dataset manifest lists the +exact collected files and their episode IDs; stale files are not added to the +current experiment. + +The example configuration writes these files beneath `outputs/numeric-policy`: + +- `dataset/sequence-*.json`: recorded episodes in original numeric units. +- `dataset/manifest.json`: exact source files, episode IDs, and raw returns. +- `dataset/return_labels.json`: the return definition and raw per-decision labels. +- `return_conditioned.pt` and `behavior_cloning.pt`: local actor artifacts. +- `evaluation/*.json`: trajectories executed using the reloaded actors. +- `report.json`: episode partitions, per-epoch action likelihood, checkpoint + selection, greedy/sampled actions, achieved returns, and requested-return errors. + +The report distinguishes raw episode reward sums from discounted achieved +returns. It compares both actors at the same requested returns, initial states, +and sampling seeds. The BC actor does not receive the requested return. This is +a controlled learning comparison; performance depends on training configuration +and is recorded rather than assumed. + +## Policy alignment and causal windows + +At decision position `t`, the policy receives `o_t`, `a_(t-1)`, and, for return +conditioning, `G_t`. The current action `a_t` is a separate target. The first +previous action is a dedicated start-token ID equal to the number of executable +actions. It cannot be selected by the action head. + +For the sequence `[1, 0, 1]`, gamma 1, and the default task: + +| Decision | Current observation | Previous action ID | Desired return | Action target | +| --- | --- | --- | --- | --- | +| 0 | `(position=0, remaining_steps=3)` | 2 (start) | 2 | 1 | +| 1 | `(position=1, remaining_steps=2)` | 1 | 1 | 0 | +| 2 | `(position=1, remaining_steps=1)` | 0 | 1 | 1 | + +Offline and online execution use `build_policy_inputs` to construct the same +left-padded causal windows. Observations, previous actions, and returns are +trimmed together. Padding has a false attention mask. Only the last position +of each training example has a true action-likelihood mask, so every recorded +decision contributes once even when context windows overlap. + +The actor uses numeric projection, previous-action embedding, a small causal GRU, +and one categorical head. Padded positions do not advance its hidden state. +Forward calls take `PolicyInputs`, which contain no action labels or observed +current rewards. A change to a current-action label cannot affect that position's +inputs; future observations are excluded from its window. Return-to-go is the +intentional reward-derived training condition, with requested budgets replacing +it online. Changing rewards can therefore change the return condition. + +Observation/static normalization is fit on the training episode partition only. +Its fixed means/scales are model buffers, saved with the actor. The RC and BC +actors use the same architecture, initialization seed, episode split, batch +order, and optimization configuration. BC disables the return channel by +feeding zero; it retains the same parameter count. + +## Return definitions and episode boundaries + +Raw labels use `G_t = r_t + gamma * G_(t+1)` and terminal `G_T = 0`. +For rewards `[1, 2, 3]`, gamma 1 produces `[6, 5, 3]`; gamma 0.5 produces +`[2.75, 3.5, 3]`. `returns.gamma` is explicit in `(0, 1]`, defaulting to 1. +`returns.scale` divides returns only at the numeric encoder boundary and defaults +to 1. Artifacts and execution traces preserve raw reward units. + +By default, only trajectories with true termination have exact return labels. +Collection cutoffs and truncation without termination are excluded and their +episode IDs are exposed by the dataset. An explicit `returns.finite_horizon: H` +allows labeling an observed prefix of exactly H decisions, including a cutoff +at H. A fragment shorter than H is still excluded unless it truly terminates. +No unknown tail is silently assigned zero. For online use of this objective, +`rollout.max_episode_steps` must equal H. + +Execution starts from the caller's requested return and updates the raw budget +after each executed reward: + +```text +G_next = (G_current - reward) / gamma +``` + +The request is not clamped to an inferred feasible range. Each episode reset +restores the requested budget, clears previous-action/history state, and uses +the start token again. A collection cutoff preserves the unfinished episode +flag; it does not fabricate a reset. Each call collects one episode/fragment, +so collecting another episode uses a fresh adapter and an explicit reset. + +Policy trajectories use `format: sequifier.numeric-policy-trajectory`, version 1, +with the same seed/transition schema and storage as numeric forecast trajectories. +`mode: policy` makes this alignment explicit. Forecast artifacts retain their +existing format and require an explicit declaration before their executed +predictions can be used as demonstration actions. +Policy episodes start with one valid observation because historical actions +before reset are unknown. The actor builds subsequent history from executed +transitions. + +Execution metadata stores the full policy/return configuration, initial request, +remaining budgets (including the final successor budget), previous-action IDs, +and selected-action log probabilities. The trajectory validator checks action +vocabulary, previous-action alignment, budget counts, and reward/gamma arithmetic +in addition to ordinary continuity, reward, and boundary validation. Both flags +may be true; termination takes precedence for bootstrap. Adapter closure remains +owned by the shared coordinator on success and failure. + +## Reusable Python interfaces + +The public entry points live in `sequifier.rl_api`. To train on compatible saved +policy episodes: + +```python +from sequifier.rl_api import ( + CategoricalPolicyConfig, PolicyTrainingConfig, TrajectoryDataset, + load_policy, save_policy, split_by_episode, train_policy, +) + +spec = CategoricalPolicyConfig( + observation_columns=["position", "remaining_steps"], + action_column="action", + action_values=[0.0, 1.0], +) +dataset = TrajectoryDataset(episode_paths, spec) +training, validation = split_by_episode(dataset, validation_fraction=0.2, seed=1010) +result = train_policy(training, validation, PolicyTrainingConfig()) +save_policy(result.policy, "actor.pt") +actor = load_policy("actor.pt") +``` + +First-increment forecast trajectories can also supply demonstrations when their +executed numeric predictions have a declared finite categorical vocabulary: +pass `action_semantics="executed_predictions"` to `TrajectoryDataset` and set +the appropriate `action_column` and `action_values` in the policy specification. +This is an explicit data interpretation, not a conversion of forecast model +alignment or weights. It uses the latest seed observation as decision zero, +discards earlier seed observations whose actions were never recorded, and +starts previous-action history with the start token. Episode splitting preserves +this declaration. Observation and action names may coincide because their +values are read from separate observation/target records; current labels are +never copied into current inputs. + +Action values are original numeric domain values; the distribution uses their +ordered categorical IDs internally. `actor.distribution(inputs)` returns a +`CategoricalActionDistribution` supporting `mode()`, `sample(generator)`, +`log_prob(recorded_action_ids)`, and `entropy()`. `masked_action_likelihood` +uses the dataset's independent action-valid mask. Training calls the existing +`OptimizationRuntime` for backward, clipping, optimizer, scaler, and scheduler +steps. The validation partition selects the saved best actor by action NLL. + +```python +from sequifier.rl_api import collect_policy_rollout +from sequifier.rollout.control import BinaryControlAdapter +from sequifier.rl.experiment import control_rollout_config, control_seed +from sequifier.rollout_api import save_trajectory + +rollout = control_rollout_config("execution.json", horizon=3) +trajectory = collect_policy_rollout( + actor, BinaryControlAdapter(horizon=3), control_seed(horizon=3), + rollout, max_steps=3, desired_return=3.0, deterministic=True, +) +save_trajectory(trajectory, rollout.trajectory_path) +``` + +Set `return_conditioned=False` in the policy specification to train BC, and omit +`desired_return` when executing it. The actor artifacts contain validated CPU +weights, normalization buffers, action vocabulary, policy alignment, and return +semantics. Loading uses strict state-dict matching. These local inference +artifacts do not contain optimizer/RNG state for exact training resumption and +do not use forecast export or inference formats. + +This increment has no critics, Q networks, PPO buffers, distributed execution, +or policy export additions. Return conditioning and unconditioned behavior +cloning are independent actor routes. Future offline IQL can reuse the +trajectory windows and distribution interface without requiring RC pretraining. diff --git a/src/sequifier/docs.py b/src/sequifier/docs.py index 0f16853e..514d0dfd 100644 --- a/src/sequifier/docs.py +++ b/src/sequifier/docs.py @@ -6,6 +6,7 @@ "preprocess": "# Preprocess Command Guide", "train": "# Train Command Guide", "infer": "# Infer Command Guide", + "policy": "# Policy Learning Command Guide", } diff --git a/src/sequifier/rl/__init__.py b/src/sequifier/rl/__init__.py new file mode 100644 index 00000000..bb67d4b9 --- /dev/null +++ b/src/sequifier/rl/__init__.py @@ -0,0 +1 @@ +"""Local categorical policy learning, independent of forecast alignment.""" diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py new file mode 100644 index 00000000..508b2951 --- /dev/null +++ b/src/sequifier/rl/checkpoint.py @@ -0,0 +1,66 @@ +"""Versioned local actor weights, input transforms, and policy/return semantics.""" + +import json +import os +import tempfile +from pathlib import Path + +import torch + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.rl.policy import CategoricalPolicy + + +def validate_policy_weights(policy: CategoricalPolicy) -> None: + for name, value in policy.state_dict().items(): + if value.device.type != "cpu" or not torch.isfinite(value).all(): + raise ValueError(f"policy tensor {name!r} must be finite and on CPU") + if (policy.observation_scale <= 0).any() or (policy.static_scale <= 0).any(): + raise ValueError("policy normalization scales must be positive") + + +def save_policy( + policy: CategoricalPolicy, path: str | Path, *, metadata: dict | None = None +) -> None: + validate_policy_weights(policy) + json.dumps(metadata or {}, allow_nan=False) + payload = { + "format": "sequifier.numeric-categorical-policy", + "version": 1, + "config": policy.config.model_dump(mode="json"), + "state_dict": policy.state_dict(), + "metadata": metadata or {}, + } + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile( + dir=destination.parent, delete=False + ) as stream: + temporary = stream.name + torch.save(payload, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, destination) + finally: + if temporary is not None and os.path.exists(temporary): + os.unlink(temporary) + + +def load_policy(path: str | Path) -> CategoricalPolicy: + payload = torch.load(path, map_location="cpu", weights_only=True) + if ( + not isinstance(payload, dict) + or payload.get("format") != "sequifier.numeric-categorical-policy" + or payload.get("version") != 1 + ): + raise ValueError("unsupported numeric categorical policy artifact") + config = CategoricalPolicyConfig.model_validate(payload["config"]) + json.dumps(payload.get("metadata", {}), allow_nan=False) + with torch.random.fork_rng(devices=[]): + policy = CategoricalPolicy(config) + policy.load_state_dict(payload["state_dict"], strict=True) + validate_policy_weights(policy) + policy.eval() + return policy diff --git a/src/sequifier/rl/distributions.py b/src/sequifier/rl/distributions.py new file mode 100644 index 00000000..62a9641a --- /dev/null +++ b/src/sequifier/rl/distributions.py @@ -0,0 +1,51 @@ +"""The categorical action distribution used by learning, scoring, and execution.""" + +import torch +from torch import Tensor + + +class CategoricalActionDistribution: + def __init__(self, logits: Tensor): + if logits.ndim < 2 or logits.shape[-1] < 2 or not torch.isfinite(logits).all(): + raise ValueError( + "categorical policy logits must be finite with at least two actions" + ) + self.logits = logits + self.distribution = torch.distributions.Categorical(logits=logits) + + def mode(self) -> Tensor: + return self.logits.argmax(dim=-1) + + def sample(self, generator: torch.Generator | None = None) -> Tensor: + probabilities = self.distribution.probs + sampled = torch.multinomial( + probabilities.reshape(-1, probabilities.shape[-1]), 1, generator=generator + ) + return sampled.reshape(probabilities.shape[:-1]) + + def log_prob(self, actions: Tensor) -> Tensor: + if actions.dtype != torch.int64 or actions.shape != self.logits.shape[:-1]: + raise ValueError( + "categorical action IDs must be int64 and match the logits batch shape" + ) + if (actions < 0).any() or (actions >= self.logits.shape[-1]).any(): + raise ValueError("categorical action ID is out of range") + return self.distribution.log_prob(actions) + + def entropy(self) -> Tensor: + return self.distribution.entropy() + + +def masked_action_likelihood( + distribution: CategoricalActionDistribution, actions: Tensor, valid_mask: Tensor +) -> Tensor: + if ( + valid_mask.dtype != torch.bool + or valid_mask.shape != actions.shape + or actions.shape != distribution.logits.shape[:-1] + ): + raise ValueError("action loss masks must be boolean and match actions/logits") + if not valid_mask.any(): + raise ValueError("action likelihood requires at least one labeled decision") + selected = CategoricalActionDistribution(distribution.logits[valid_mask]) + return -selected.log_prob(actions[valid_mask]).mean() diff --git a/src/sequifier/rl/experiment.py b/src/sequifier/rl/experiment.py new file mode 100644 index 00000000..a33955f8 --- /dev/null +++ b/src/sequifier/rl/experiment.py @@ -0,0 +1,294 @@ +"""Collect, train RC/BC actors, reload, and report achieved control-task returns.""" + +import json +from itertools import product +from pathlib import Path + +import yaml +from loguru import logger + +from sequifier.config.policy_config import ( + CategoricalPolicyConfig, + ControlExperimentConfig, +) +from sequifier.config.rollout_config import AdapterConfig, InputBinding, RolloutConfig +from sequifier.rl.checkpoint import load_policy, save_policy +from sequifier.rl.runtime import collect_policy_rollout +from sequifier.rl.training import train_policy +from sequifier.rollout.control import BinaryControlAdapter +from sequifier.rollout.coordinator import collect_rollout +from sequifier.trajectory.dataset import TrajectoryDataset, split_by_episode +from sequifier.trajectory.returns import return_to_go +from sequifier.trajectory.schema import SeedHistory +from sequifier.trajectory.store import load_trajectory, save_trajectory + + +def control_seed(horizon: int = 3, context_length: int = 4) -> SeedHistory: + if horizon < 1 or context_length < 1: + raise ValueError("control horizon and context_length must be positive") + return SeedHistory( + observations={ + "position": [0.0] * context_length, + "remaining_steps": [0.0] * (context_length - 1) + [float(horizon)], + }, + attention_valid_mask=[False] * (context_length - 1) + [True], + static_inputs={}, + ) + + +def control_rollout_config( + path: str | Path, horizon: int = 3, *, finite_horizon: int | None = None +) -> RolloutConfig: + return RolloutConfig( + mode="policy", + adapter=AdapterConfig( + factory="sequifier.rollout.control:BinaryControlAdapter", + config={"horizon": horizon}, + ), + bindings=[ + InputBinding(input=column, source="environment", key=column) + for column in ("position", "remaining_steps") + ], + seed_index=0, + trajectory_path=str(path), + max_episode_steps=finite_horizon, + ) + + +def collect_control_dataset( + directory: str | Path, + *, + horizon: int = 3, + repetitions: int = 16, + context_length: int = 4, + random_seed: int = 1010, +) -> list[Path]: + """Enumerate every binary action sequence, repeatedly from identical conditions.""" + if ( + isinstance(horizon, bool) + or not isinstance(horizon, int) + or not 1 <= horizon <= 8 + ): + raise ValueError("dataset control horizon must be an integer in [1, 8]") + if ( + isinstance(repetitions, bool) + or not isinstance(repetitions, int) + or repetitions < 1 + ): + raise ValueError("dataset repetitions must be a positive integer") + destination = Path(directory) + seed = control_seed(horizon, context_length) + paths = [] + manifest = [] + for sequence_index, actions in enumerate(product((0.0, 1.0), repeat=horizon)): + for repetition in range(repetitions): + path = ( + destination + / f"sequence-{sequence_index:03d}-episode-{repetition:03d}.json" + ) + rollout = control_rollout_config(path, horizon) + action_sequence = iter(actions) + + def predict(_history): + return {"action": next(action_sequence)} + + trajectory = collect_rollout( + predict, + BinaryControlAdapter(horizon), + seed, + rollout, + ["action"], + horizon, + random_seed=random_seed, + metadata={ + "policy_alignment": "current_observation_previous_action", + "behavior": "enumerated_action_sequence", + "action_values": [0.0, 1.0], + "task": {"name": "binary_control", "horizon": horizon}, + }, + ) + save_trajectory(trajectory, path) + paths.append(path) + manifest.append( + { + "path": path.name, + "episode_id": trajectory.episode_id, + "total_return": trajectory.total_return, + } + ) + # Consumers use these exact paths rather than accidentally including stale files. + (destination / "manifest.json").write_text( + json.dumps({"version": 1, "episodes": manifest}, indent=2) + "\n", + encoding="utf-8", + ) + return paths + + +def run_control_experiment(config: ControlExperimentConfig) -> dict: + root = Path(config.project_root).resolve() + output = Path(config.output_dir) + if not output.is_absolute(): + output = root / output + output.mkdir(parents=True, exist_ok=True) + paths = collect_control_dataset( + output / "dataset", + horizon=config.horizon, + repetitions=config.repetitions, + context_length=config.context_length, + random_seed=config.training.seed, + ) + spec = CategoricalPolicyConfig( + observation_columns=["position", "remaining_steps"], + context_length=config.context_length, + hidden_size=config.hidden_size, + returns=config.returns, + ) + corpus = TrajectoryDataset(paths, spec) + training, validation = split_by_episode( + corpus, config.training.validation_fraction, config.training.seed + ) + (output / "dataset" / "return_labels.json").write_text( + json.dumps( + { + "version": 1, + "definition": spec.returns.model_dump(mode="json"), + "episodes": [ + {"episode_id": episode.episode_id, "return_to_go": labels} + for episode, labels in zip(corpus.episodes, corpus.returns) + ], + }, + indent=2, + allow_nan=False, + ) + + "\n", + encoding="utf-8", + ) + targets = config.requested_returns + if targets is None: + targets = [0.0, return_to_go([1.0] * config.horizon, config.returns.gamma)[0]] + report = { + "format": "sequifier.numeric-policy-experiment", + "version": 1, + "config": config.model_dump(mode="json"), + "dataset": { + "episodes": len(corpus.episodes), + "decisions": len(corpus), + "excluded_episode_ids": corpus.excluded_episode_ids, + "training_episode_ids": [ + episode.episode_id for episode in training.episodes + ], + "validation_episode_ids": [ + episode.episode_id for episode in validation.episodes + ], + "behavior_mean_return": sum( + episode.total_return for episode in corpus.episodes + ) + / len(corpus.episodes), + }, + "training": {}, + "evaluation": [], + } + for algorithm, conditioned in ( + ("return_conditioned", True), + ("behavior_cloning", False), + ): + actor_spec = spec.model_copy(update={"return_conditioned": conditioned}) + train_dataset = TrajectoryDataset(training.episodes, actor_spec) + validation_dataset = TrajectoryDataset(validation.episodes, actor_spec) + logger.info(f"Training {algorithm} on {len(train_dataset)} decisions") + result = train_policy(train_dataset, validation_dataset, config.training) + policy_path = output / f"{algorithm}.pt" + save_policy( + result.policy, + policy_path, + metadata={ + "algorithm": algorithm, + "training_config": config.training.model_dump(mode="json"), + "training_episode_ids": list(result.training_episode_ids), + "validation_episode_ids": list(result.validation_episode_ids), + "best_epoch": result.best_epoch, + }, + ) + # All reported execution uses weights/transforms reloaded from disk. + policy = load_policy(policy_path) + report["training"][algorithm] = { + "policy_path": str(policy_path), + "best_epoch": result.best_epoch, + "optimizer_steps": result.optimizer_steps, + "epochs": list(result.metrics), + } + for target_index, requested_return in enumerate(targets): + for selection, count in ( + ("deterministic", 1), + ("sample", config.sampling_episodes), + ): + episodes = [] + for episode_index in range(count): + path = ( + output + / "evaluation" + / f"{algorithm}-return-{target_index}-{selection}-{episode_index}.json" + ) + rollout = control_rollout_config( + path, config.horizon, finite_horizon=spec.returns.finite_horizon + ) + trajectory = collect_policy_rollout( + policy, + BinaryControlAdapter(config.horizon), + control_seed(config.horizon, config.context_length), + rollout, + config.horizon, + desired_return=requested_return if conditioned else None, + deterministic=selection == "deterministic", + random_seed=config.training.seed + episode_index, + ) + save_trajectory(trajectory, path) + recorded = load_trajectory(path) + achieved = return_to_go( + [step.reward for step in recorded.transitions], + spec.returns.gamma, + )[0] + episodes.append( + { + "episode_id": recorded.episode_id, + "trajectory_path": str(path), + "actions": [ + step.predictions[spec.action_column] + for step in recorded.transitions + ], + "achieved_return": achieved, + "total_return": recorded.total_return, + "remaining_returns": recorded.metadata["policy_execution"][ + "remaining_returns" + ], + } + ) + report["evaluation"].append( + { + "algorithm": algorithm, + "requested_return": requested_return, + "selection": selection, + "episodes": episodes, + "mean_achieved_return": sum( + episode["achieved_return"] for episode in episodes + ) + / count, + "mean_absolute_return_error": sum( + abs(episode["achieved_return"] - requested_return) + for episode in episodes + ) + / count, + } + ) + report_path = output / "report.json" + report_path.write_text( + json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + logger.info(f"Saved policy learning comparison to '{report_path}'") + return report + + +def run_policy_command(config_path: str) -> None: + with open(config_path, encoding="utf-8") as stream: + config = ControlExperimentConfig.model_validate(yaml.safe_load(stream)) + run_control_experiment(config) diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py new file mode 100644 index 00000000..270a4274 --- /dev/null +++ b/src/sequifier/rl/history.py @@ -0,0 +1,107 @@ +"""One causal policy window builder shared by offline training and execution.""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +import torch +from torch import Tensor + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.rollout.history import numeric_fields, numeric_scalar + + +@dataclass(frozen=True) +class PolicyInputs: + observations: Tensor + previous_actions: Tensor + attention_valid_mask: Tensor + static_inputs: Tensor + desired_returns: Tensor | None + + +def action_index(value: float, config: CategoricalPolicyConfig) -> int: + value = numeric_scalar(value, "recorded action") + try: + return config.action_values.index(value) + except ValueError as exc: + raise ValueError( + f"action {value!r} is outside action_values {config.action_values!r}" + ) from exc + + +def build_policy_inputs( + config: CategoricalPolicyConfig, + observations: Sequence[Mapping[str, float]], + previous_actions: Sequence[int], + desired_returns: Sequence[float] | None, + static_inputs: Mapping[str, float], +) -> PolicyInputs: + """Only supplied decision prefixes are visible; targets are never an argument. + + At position t, observations[t] is o_t and previous_actions[t] is a_(t-1). + The action-count ID is the explicit start token, never an executable action. + """ + length = len(observations) + if not length or len(previous_actions) != length: + raise ValueError( + "policy observations and previous actions must have matching nonempty lengths" + ) + if config.return_conditioned and ( + desired_returns is None or len(desired_returns) != length + ): + raise ValueError( + "return-conditioned windows require one desired return per decision position" + ) + static = numeric_fields( + static_inputs, set(config.static_columns), "policy static inputs" + ) + context = config.context_length + start = max(0, length - context) + count = length - start + padding = context - count + feature_rows = [[0.0] * len(config.observation_columns) for _ in range(padding)] + for observation in observations[start:]: + values = numeric_fields( + observation, set(config.observation_columns), "policy observation" + ) + feature_rows.append([values[column] for column in config.observation_columns]) + action_ids = [len(config.action_values)] * padding + for action in previous_actions[start:]: + if ( + isinstance(action, bool) + or not isinstance(action, int) + or not 0 <= action <= len(config.action_values) + ): + raise ValueError( + "previous actions must be categorical IDs or the start token" + ) + action_ids.append(action) + returns = None + if config.return_conditioned: + assert desired_returns is not None + returns = torch.tensor( + [ + [0.0] * padding + + [ + numeric_scalar(value, "desired_return") + for value in desired_returns[start:] + ] + ], + dtype=torch.float32, + ) + inputs = PolicyInputs( + observations=torch.tensor([feature_rows], dtype=torch.float32), + previous_actions=torch.tensor([action_ids], dtype=torch.int64), + attention_valid_mask=torch.tensor([[False] * padding + [True] * count]), + static_inputs=torch.tensor( + [[static[column] for column in config.static_columns]], dtype=torch.float32 + ), + desired_returns=returns, + ) + if ( + not torch.isfinite(inputs.observations).all() + or not torch.isfinite(inputs.static_inputs).all() + or (returns is not None and not torch.isfinite(returns).all()) + ): + raise ValueError("policy inputs exceed the finite float32 range") + return inputs diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py new file mode 100644 index 00000000..fce79f81 --- /dev/null +++ b/src/sequifier/rl/policy.py @@ -0,0 +1,139 @@ +"""A small causal numeric actor with one categorical head and no value network.""" + +import torch +from torch import Tensor, nn + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.rl.history import PolicyInputs + + +class CategoricalPolicy(nn.Module): + def __init__(self, config: CategoricalPolicyConfig): + super().__init__() + self.config = config + width = config.hidden_size + numeric_width = len(config.observation_columns) + len(config.static_columns) + 1 + self.ingestion = nn.Linear(numeric_width, width) + self.previous_action_embedding = nn.Embedding( + len(config.action_values) + 1, width + ) + self.backbone = nn.GRUCell(width, width) + self.decoder = nn.Linear(width, len(config.action_values)) + self.register_buffer( + "observation_mean", torch.zeros(len(config.observation_columns)) + ) + self.register_buffer( + "observation_scale", torch.ones(len(config.observation_columns)) + ) + self.register_buffer("static_mean", torch.zeros(len(config.static_columns))) + self.register_buffer("static_scale", torch.ones(len(config.static_columns))) + + def set_normalization(self, observations: Tensor, static_inputs: Tensor) -> None: + """Fit once on the training episode partition; persist as model buffers.""" + for values, mean, scale in ( + (observations, self.observation_mean, self.observation_scale), + (static_inputs, self.static_mean, self.static_scale), + ): + if ( + values.ndim != 2 + or values.shape[1:] != mean.shape + or not len(values) + or not torch.isfinite(values).all() + ): + raise ValueError( + "normalization requires finite training rows matching policy columns" + ) + fitted = values.to(torch.float64) + mean.copy_(fitted.mean(dim=0)) + scale.copy_(fitted.std(dim=0, unbiased=False).clamp_min(1e-6)) + if not torch.isfinite(mean).all() or not torch.isfinite(scale).all(): + raise ValueError( + "training normalization exceeds the model's numeric range" + ) + + def forward(self, inputs: PolicyInputs) -> Tensor: + self.validate_inputs(inputs) + mask = inputs.attention_valid_mask + batch, context = mask.shape + observations = ( + inputs.observations - self.observation_mean + ) / self.observation_scale + static = (inputs.static_inputs - self.static_mean) / self.static_scale + static = static[:, None, :].expand(batch, context, -1) + if self.config.return_conditioned: + assert inputs.desired_returns is not None + returns = inputs.desired_returns / self.config.returns.scale + else: + # Same parameters and architecture as RC; the return channel is disabled. + returns = observations.new_zeros((batch, context)) + numeric = torch.cat((observations, static, returns.unsqueeze(-1)), dim=-1) + encoded = torch.tanh( + self.ingestion(numeric) + + self.previous_action_embedding(inputs.previous_actions) + ) + hidden = encoded.new_zeros((batch, self.config.hidden_size)) + outputs = [] + for position in range(context): + candidate = self.backbone(encoded[:, position], hidden) + hidden = torch.where(mask[:, position, None], candidate, hidden) + outputs.append(self.decoder(hidden)) + return torch.stack(outputs, dim=1) + + def distribution(self, inputs: PolicyInputs) -> CategoricalActionDistribution: + return CategoricalActionDistribution(self(inputs)) + + def validate_inputs(self, inputs: PolicyInputs) -> None: + mask = inputs.attention_valid_mask + if ( + mask.ndim != 2 + or mask.dtype != torch.bool + or mask.shape[1] != self.config.context_length + or mask.shape[0] < 1 + ): + raise ValueError( + "policy attention masks must be bool [batch, context_length]" + ) + if not mask[:, -1].all() or (mask[:, :-1] & ~mask[:, 1:]).any(): + raise ValueError( + "policy histories require left padding followed by valid positions" + ) + if inputs.observations.shape != ( + *mask.shape, + len(self.config.observation_columns), + ): + raise ValueError("policy observation shape does not match its schema") + if inputs.static_inputs.shape != ( + mask.shape[0], + len(self.config.static_columns), + ): + raise ValueError("policy static input shape does not match its schema") + actions = inputs.previous_actions + if ( + actions.shape != mask.shape + or actions.dtype != torch.int64 + or (actions < 0).any() + or (actions > len(self.config.action_values)).any() + ): + raise ValueError( + "policy previous actions must be valid int64 IDs, including the start token" + ) + numeric = [inputs.observations, inputs.static_inputs] + if self.config.return_conditioned: + if ( + inputs.desired_returns is None + or inputs.desired_returns.shape != mask.shape + ): + raise ValueError( + "policy requires a desired return at every decision position" + ) + numeric.append(inputs.desired_returns) + if any( + value.device.type != "cpu" + or value.dtype != torch.float32 + or not torch.isfinite(value).all() + for value in numeric + ): + raise ValueError("numeric policy inputs must be finite CPU float32 tensors") + if mask.device.type != "cpu" or actions.device.type != "cpu": + raise ValueError("numeric policies currently support CPU execution only") diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py new file mode 100644 index 00000000..22325a15 --- /dev/null +++ b/src/sequifier/rl/runtime.py @@ -0,0 +1,192 @@ +"""Return-conditioned actors executed through the ordinary rollout coordinator.""" + +from collections import deque + +import torch + +from sequifier.config.rollout_config import RolloutConfig +from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.rl.history import build_policy_inputs +from sequifier.rl.policy import CategoricalPolicy +from sequifier.rollout.contracts import EnvironmentAdapter, ResetRequest +from sequifier.rollout.coordinator import collect_rollout +from sequifier.rollout.history import numeric_fields, numeric_scalar +from sequifier.trajectory.returns import remaining_return +from sequifier.trajectory.schema import ( + SeedHistory, + Trajectory, + Transition, + validate_trajectory, +) + + +class PolicyExecutor: + """Callable predictor/observer; current actions enter history only after step().""" + + def __init__( + self, + policy: CategoricalPolicy, + desired_return: float | None, + deterministic: bool = True, + ): + self.policy = policy + self.initial_return = desired_return + self.deterministic = deterministic + self.active = False + + def reset(self, request: ResetRequest) -> None: + spec = self.policy.config + if not isinstance(self.deterministic, bool): + raise ValueError("policy deterministic selection must be a boolean") + if sum(request.attention_valid_mask) != 1: + raise ValueError( + "policy reset requires one current observation; prior actions are unknown" + ) + if spec.return_conditioned: + self.budget = numeric_scalar(self.initial_return, "initial desired_return") + else: + if self.initial_return is not None: + raise ValueError("behavior cloning has no desired-return input") + self.budget = None + self.observations = deque(maxlen=spec.context_length) + self.previous_actions = deque(maxlen=spec.context_length) + self.budgets = deque(maxlen=spec.context_length) + self.previous_action = len(spec.action_values) + self.return_trace = [self.budget] if spec.return_conditioned else [] + self.previous_action_trace = [] + self.log_probabilities = [] + self.generator = torch.Generator().manual_seed(request.seed) + self.pending = None + self.ended = False + self.active = True + self.episode_id = request.episode_id + self.step_id = 0 + + def __call__(self, history: SeedHistory) -> dict[str, float]: + if not self.active or self.pending is not None or self.ended: + raise ValueError( + "policy predictions require an active episode and an executed previous prediction" + ) + spec = self.policy.config + observation = { + column: values[-1] for column, values in history.observations.items() + } + inputs = build_policy_inputs( + spec, + list(self.observations) + [observation], + list(self.previous_actions) + [self.previous_action], + list(self.budgets) + [self.budget] if spec.return_conditioned else None, + history.static_inputs, + ) + with torch.no_grad(): + distribution = CategoricalActionDistribution(self.policy(inputs)[:, -1, :]) + action = ( + distribution.mode() + if self.deterministic + else distribution.sample(self.generator) + ) + log_probability = float(distribution.log_prob(action)[0]) + index = int(action[0]) + self.pending = (observation, index, log_probability) + return {spec.action_column: spec.action_values[index]} + + def observe(self, transition: Transition) -> None: + if not self.active or self.pending is None: + raise ValueError("policy observed a transition without a predicted action") + observation, index, log_probability = self.pending + spec = self.policy.config + if ( + transition.episode_id != self.episode_id + or transition.step_id != self.step_id + ): + raise ValueError("policy transition episode/step IDs are out of sync") + if transition.observation != observation or transition.predictions != { + spec.action_column: spec.action_values[index] + }: + raise ValueError("executed transition does not match the policy decision") + self.observations.append(observation) + self.previous_actions.append(self.previous_action) + self.previous_action_trace.append(self.previous_action) + self.log_probabilities.append(log_probability) + if spec.return_conditioned: + assert self.budget is not None + self.budgets.append(self.budget) + self.budget = remaining_return( + self.budget, transition.reward, spec.returns.gamma + ) + self.return_trace.append(self.budget) + self.previous_action = index + self.pending = None + self.ended = transition.terminated or transition.truncated + self.step_id += 1 + + +def collect_policy_rollout( + policy: CategoricalPolicy, + adapter: EnvironmentAdapter, + seed: SeedHistory, + config: RolloutConfig, + max_steps: int, + *, + desired_return: float | None = None, + deterministic: bool = True, + random_seed: int = 1010, +) -> Trajectory: + """Collect one reset-initialized policy episode/fragment and always close adapter. + + Each call/reset starts the requested budget again. Time limits and collection + cutoffs retain the coordinator's distinct boundary semantics. + """ + executor = PolicyExecutor(policy, desired_return, deterministic) + spec = policy.config + + def validate_history(history): + if config.mode != "policy": + raise ValueError("categorical execution requires rollout mode: policy") + horizon = spec.returns.finite_horizon + if horizon is not None and config.max_episode_steps != horizon: + raise ValueError( + "finite-horizon policy execution requires max_episode_steps equal to its return horizon" + ) + observation = { + column: values[-1] for column, values in history.observations.items() + } + numeric_fields(observation, set(spec.observation_columns), "policy observation") + # Validate the final successor too, even when no subsequent action is needed. + inputs = build_policy_inputs( + spec, + [observation], + [len(spec.action_values)], + [numeric_scalar(desired_return, "desired_return")] + if spec.return_conditioned + else None, + history.static_inputs, + ) + policy.validate_inputs(inputs) + + was_training = policy.training + policy.eval() + try: + trajectory = collect_rollout( + executor, + adapter, + seed, + config, + [spec.action_column], + max_steps, + random_seed=random_seed, + history_validator=validate_history, + observer=executor, + metadata={"policy_alignment": spec.alignment}, + ) + finally: + policy.train(was_training) + trajectory.metadata["policy_execution"] = { + "config": spec.model_dump(mode="json"), + "initial_desired_return": desired_return, + "remaining_returns": executor.return_trace, + "previous_action_ids": executor.previous_action_trace, + "action_log_probabilities": executor.log_probabilities, + "deterministic": deterministic, + } + return validate_trajectory(trajectory) diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py new file mode 100644 index 00000000..684e4219 --- /dev/null +++ b/src/sequifier/rl/training.py @@ -0,0 +1,144 @@ +"""Supervised masked action likelihood using Sequifier's optimizer transaction.""" + +from copy import deepcopy +from dataclasses import dataclass + +import torch +from torch.utils.data import DataLoader + +from sequifier.config.policy_config import PolicyTrainingConfig +from sequifier.integration.callbacks import IntegrationManager +from sequifier.integration.contexts import StepIdentity +from sequifier.rl.distributions import masked_action_likelihood +from sequifier.rl.policy import CategoricalPolicy +from sequifier.training.optimization import OptimizationRuntime, UpdatePolicy +from sequifier.trajectory.dataset import TrajectoryDataset, collate_policy_examples + + +@dataclass(frozen=True) +class PolicyTrainingResult: + policy: CategoricalPolicy + metrics: tuple[dict[str, float], ...] + best_epoch: int + optimizer_steps: int + training_episode_ids: tuple[str, ...] + validation_episode_ids: tuple[str, ...] + + +def train_policy( + training: TrajectoryDataset, + validation: TrajectoryDataset, + config: PolicyTrainingConfig, +) -> PolicyTrainingResult: + if not len(training) or not len(validation): + raise ValueError("policy training and validation partitions must be nonempty") + if ( + training.config != validation.config + or training.action_semantics != validation.action_semantics + ): + raise ValueError( + "training and validation must share the policy/return definition" + ) + train_ids = tuple(episode.episode_id for episode in training.episodes) + validation_ids = tuple(episode.episode_id for episode in validation.episodes) + if set(train_ids) & set(validation_ids): + raise ValueError( + "training and validation partitions must not share episode IDs" + ) + # Keep initialization and batch order equal across the RC and BC comparison. + with torch.random.fork_rng(devices=[]): + torch.manual_seed(config.seed) + policy = CategoricalPolicy(training.config) + policy.set_normalization(*training.normalization_rows()) + optimization = OptimizationRuntime.create(config, "cpu", policy.parameters()) + integrations = IntegrationManager() + generator = torch.Generator().manual_seed(config.seed) + train_loader = DataLoader( + training, + batch_size=config.batch_size, + shuffle=True, + generator=generator, + collate_fn=collate_policy_examples, + ) + validation_loader = DataLoader( + validation, batch_size=config.batch_size, collate_fn=collate_policy_examples + ) + metrics = [] + best_loss = float("inf") + best_epoch = 0 + best_weights = None + global_batch = 0 + optimization.optimizer.zero_grad(set_to_none=True) + for epoch in range(config.epochs): + policy.train() + total_loss = 0.0 + decisions = 0 + for batch_index, batch in enumerate(train_loader): + distribution = policy.distribution(batch.inputs) + loss = masked_action_likelihood( + distribution, batch.actions, batch.action_valid_mask + ) + if not torch.isfinite(loss): + raise ValueError("policy action likelihood became non-finite") + count = int(batch.action_valid_mask.sum()) + total_loss += float(loss.detach()) * count + decisions += count + identity = StepIdentity( + epoch, + batch_index, + global_batch, + optimization.optimizer_step, + 0, + 1, + 0, + 1, + ) + optimization.accumulate(loss, identity, integrations, policy) + result = optimization.complete_step( + policy, identity, integrations, UpdatePolicy() + ) + if result.stop_requested or not result.applied: + raise RuntimeError( + "local policy optimizer did not apply its update" + ) + global_batch += 1 + policy.eval() + validation_loss = 0.0 + validation_count = 0 + with torch.no_grad(): + for batch in validation_loader: + loss = masked_action_likelihood( + policy.distribution(batch.inputs), + batch.actions, + batch.action_valid_mask, + ) + count = int(batch.action_valid_mask.sum()) + validation_loss += float(loss) * count + validation_count += count + validation_loss /= validation_count + metrics.append( + { + "epoch": float(epoch), + "training_nll": total_loss / decisions, + "validation_nll": validation_loss, + } + ) + if validation_loss < best_loss: + best_loss = validation_loss + best_epoch = epoch + best_weights = deepcopy(policy.state_dict()) + optimization.step_scheduler(validation_loss) + if best_weights is None: + raise ValueError( + "policy training did not produce finite validation likelihood" + ) + policy.load_state_dict(best_weights, strict=True) + policy.eval() + return PolicyTrainingResult( + policy, + tuple(metrics), + best_epoch, + optimization.optimizer_step, + train_ids, + validation_ids, + ) diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py new file mode 100644 index 00000000..b2f22d59 --- /dev/null +++ b/src/sequifier/rl_api.py @@ -0,0 +1,50 @@ +"""Local trajectory-backed categorical policies and return-conditioned learning.""" + +from sequifier.config.policy_config import ( + CategoricalPolicyConfig, + ControlExperimentConfig, + PolicyTrainingConfig, + ReturnDefinition, +) +from sequifier.rl.checkpoint import load_policy, save_policy +from sequifier.rl.distributions import ( + CategoricalActionDistribution, + masked_action_likelihood, +) +from sequifier.rl.experiment import collect_control_dataset, run_control_experiment +from sequifier.rl.history import PolicyInputs, action_index, build_policy_inputs +from sequifier.rl.policy import CategoricalPolicy +from sequifier.rl.runtime import PolicyExecutor, collect_policy_rollout +from sequifier.rl.training import PolicyTrainingResult, train_policy +from sequifier.trajectory.dataset import ( + TrajectoryDataset, + collate_policy_examples, + split_by_episode, +) +from sequifier.trajectory.returns import remaining_return, return_to_go + +__all__ = [ + "CategoricalPolicyConfig", + "ControlExperimentConfig", + "PolicyTrainingConfig", + "ReturnDefinition", + "CategoricalActionDistribution", + "CategoricalPolicy", + "PolicyInputs", + "PolicyExecutor", + "PolicyTrainingResult", + "TrajectoryDataset", + "action_index", + "build_policy_inputs", + "collate_policy_examples", + "collect_control_dataset", + "collect_policy_rollout", + "load_policy", + "masked_action_likelihood", + "remaining_return", + "return_to_go", + "run_control_experiment", + "save_policy", + "split_by_episode", + "train_policy", +] diff --git a/src/sequifier/rollout/contracts.py b/src/sequifier/rollout/contracts.py index 8fb42ea1..e8686f61 100644 --- a/src/sequifier/rollout/contracts.py +++ b/src/sequifier/rollout/contracts.py @@ -1,7 +1,12 @@ """Adapter values are finite numeric scalars in original (unscaled) units.""" +from __future__ import annotations + from dataclasses import dataclass -from typing import Mapping, Protocol +from typing import TYPE_CHECKING, Mapping, Protocol + +if TYPE_CHECKING: + from sequifier.trajectory.schema import Transition @dataclass(frozen=True) @@ -40,3 +45,11 @@ def reset(self, request: ResetRequest) -> None: def step(self, request: StepRequest) -> StepResult: ... def close(self) -> None: ... + + +class RolloutObserver(Protocol): + """Per-episode state updated only after an executed, recorded transition.""" + + def reset(self, request: ResetRequest) -> None: ... + + def observe(self, transition: Transition) -> None: ... diff --git a/src/sequifier/rollout/control.py b/src/sequifier/rollout/control.py new file mode 100644 index 00000000..60af40a7 --- /dev/null +++ b/src/sequifier/rollout/control.py @@ -0,0 +1,64 @@ +"""Deterministic finite-horizon binary numeric control task.""" + +from sequifier.rollout.contracts import ResetRequest, StepRequest, StepResult +from sequifier.rollout.history import numeric_fields, numeric_scalar + + +class BinaryControlAdapter: + """Action 0 holds position, action 1 increments it; reward equals the action.""" + + protocol_version = 1 + + def __init__(self, horizon: int = 3): + if isinstance(horizon, bool) or not isinstance(horizon, int) or horizon < 1: + raise ValueError("control horizon must be a positive integer") + self.horizon = horizon + self.closed = False + self.episode_id = None + self.steps = 0 + self.position = 0.0 + + def reset(self, request: ResetRequest) -> None: + if self.closed: + raise ValueError("control adapter is closed") + observation = numeric_fields( + {key: values[-1] for key, values in request.seed_history.items()}, + {"position", "remaining_steps"}, + "control seed", + ) + if observation != {"position": 0.0, "remaining_steps": float(self.horizon)}: + raise ValueError( + "control episodes start at position=0 with the full horizon remaining" + ) + self.episode_id = request.episode_id + self.steps = 0 + self.position = 0.0 + + def step(self, request: StepRequest) -> StepResult: + if ( + self.closed + or self.episode_id != request.episode_id + or self.steps >= self.horizon + ): + raise ValueError("control step requires an active reset episode") + if request.step_id != self.steps or request.observation != { + "position": self.position, + "remaining_steps": float(self.horizon - self.steps), + }: + raise ValueError("control request observation/step is out of sync") + action = numeric_scalar(request.predictions.get("action"), "control action") + if action not in (0.0, 1.0) or set(request.predictions) != {"action"}: + raise ValueError("binary control actions must be 0 or 1") + self.position += action + self.steps += 1 + return StepResult( + { + "position": self.position, + "remaining_steps": float(self.horizon - self.steps), + }, + reward=action, + terminated=self.steps == self.horizon, + ) + + def close(self) -> None: + self.closed = True diff --git a/src/sequifier/rollout/coordinator.py b/src/sequifier/rollout/coordinator.py index 91b81eb3..2878a6c7 100644 --- a/src/sequifier/rollout/coordinator.py +++ b/src/sequifier/rollout/coordinator.py @@ -3,12 +3,14 @@ import math import warnings from collections.abc import Callable +from copy import deepcopy from uuid import uuid4 from sequifier.config.rollout_config import RolloutConfig from sequifier.rollout.contracts import ( EnvironmentAdapter, ResetRequest, + RolloutObserver, StepRequest, StepResult, ) @@ -43,6 +45,7 @@ def collect_rollout( random_seed: int = 1010, metadata: dict | None = None, history_validator: Callable | None = None, + observer: RolloutObserver | None = None, ) -> Trajectory: """Collect one episode fragment. This call owns and always closes the adapter. @@ -63,15 +66,16 @@ def collect_rollout( raise ValueError("prediction_columns must be nonempty and unique") config.validate_bindings(list(history.observations), prediction_columns) episode_id = str(uuid4()) - adapter.reset( - ResetRequest( - episode_id, - {key: tuple(values) for key, values in history.observations.items()}, - tuple(history.attention_valid_mask), - dict(history.static_inputs), - random_seed, - ) + reset_request = ResetRequest( + episode_id, + {key: tuple(values) for key, values in history.observations.items()}, + tuple(history.attention_valid_mask), + dict(history.static_inputs), + random_seed, ) + adapter.reset(deepcopy(reset_request)) + if observer is not None: + observer.reset(deepcopy(reset_request)) transitions = [] external_keys = { binding.key @@ -139,12 +143,19 @@ def execute(predictions, step_id): bootstrap_allowed=not result.terminated, ) ) + if observer is not None: + observer.observe(transitions[-1].model_copy(deep=True)) history = next_history return result.terminated or truncated coordinate_steps(predict, execute, max_steps) last = transitions[-1] return Trajectory( + format=( + "sequifier.numeric-policy-trajectory" + if config.mode == "policy" + else "sequifier.numeric-forecast-trajectory" + ), episode_id=episode_id, seed=initial_history, bindings=config.bindings, diff --git a/src/sequifier/rollout_api.py b/src/sequifier/rollout_api.py index 0c2ec045..83ac4586 100644 --- a/src/sequifier/rollout_api.py +++ b/src/sequifier/rollout_api.py @@ -4,6 +4,7 @@ from sequifier.rollout.contracts import ( EnvironmentAdapter, ResetRequest, + RolloutObserver, StepRequest, StepResult, ) @@ -22,6 +23,7 @@ "RolloutConfig", "EnvironmentAdapter", "ResetRequest", + "RolloutObserver", "StepRequest", "StepResult", "SeedHistory", diff --git a/src/sequifier/sequifier.py b/src/sequifier/sequifier.py index e7ae9204..28f70504 100644 --- a/src/sequifier/sequifier.py +++ b/src/sequifier/sequifier.py @@ -58,7 +58,7 @@ def setup_parser() -> ArgumentParser: parser_docs.add_argument( "topic", nargs="?", - choices=("train", "preprocess", "infer"), + choices=("train", "preprocess", "infer", "policy"), help="Show documentation for one command (default: all documentation)", ) parser_preprocess = subparsers.add_parser( @@ -66,6 +66,10 @@ def setup_parser() -> ArgumentParser: ) parser_train = subparsers.add_parser("train", help="Run the training step") parser_infer = subparsers.add_parser("infer", help="Run the inference step") + parser_policy = subparsers.add_parser( + "policy", help="Collect/train/reload numeric return-conditioned and BC policies" + ) + parser_policy.add_argument("--config-path", type=str, required=True) parser_hyperparameter_search = subparsers.add_parser( "hyperparameter-search", help="Run hyperparamter search" @@ -165,6 +169,12 @@ def main() -> None: docs(args.topic) return + if args.command == "policy": + from sequifier.rl.experiment import run_policy_command + + run_policy_command(args.config_path) + return + if args.command != "hyperparameter-search": args_config = build_args_config(args) diff --git a/src/sequifier/trajectory/dataset.py b/src/sequifier/trajectory/dataset.py new file mode 100644 index 00000000..075634e4 --- /dev/null +++ b/src/sequifier/trajectory/dataset.py @@ -0,0 +1,218 @@ +"""Episode-split training windows with explicit current-action policy alignment.""" + +import random +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import torch +from torch.utils.data import Dataset + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.rl.history import PolicyInputs, action_index, build_policy_inputs +from sequifier.trajectory.returns import labeled_episode_length, return_to_go +from sequifier.trajectory.schema import Trajectory, validate_trajectory +from sequifier.trajectory.store import load_trajectory + + +@dataclass(frozen=True) +class PolicyExample: + inputs: PolicyInputs + actions: torch.Tensor + action_valid_mask: torch.Tensor + episode_ids: tuple[str, ...] + step_ids: tuple[int, ...] + + +def collate_policy_examples(examples: Sequence[PolicyExample]) -> PolicyExample: + if not examples: + raise ValueError("cannot collate an empty policy batch") + conditioned = examples[0].inputs.desired_returns is not None + if any( + (example.inputs.desired_returns is not None) != conditioned + for example in examples + ): + raise ValueError("policy batches cannot mix conditioning modes") + returns = ( + torch.cat( + [ + example.inputs.desired_returns + for example in examples + if example.inputs.desired_returns is not None + ] + ) + if conditioned + else None + ) + return PolicyExample( + inputs=PolicyInputs( + observations=torch.cat( + [example.inputs.observations for example in examples] + ), + previous_actions=torch.cat( + [example.inputs.previous_actions for example in examples] + ), + attention_valid_mask=torch.cat( + [example.inputs.attention_valid_mask for example in examples] + ), + static_inputs=torch.cat( + [example.inputs.static_inputs for example in examples] + ), + desired_returns=returns, + ), + actions=torch.cat([example.actions for example in examples]), + action_valid_mask=torch.cat( + [example.action_valid_mask for example in examples] + ), + episode_ids=tuple( + identity for example in examples for identity in example.episode_ids + ), + step_ids=tuple(step for example in examples for step in example.step_ids), + ) + + +class TrajectoryDataset(Dataset): + """One labeled decision per example; context contains only its causal prefix. + + Context positions have observations and previous actions, while only the + current (last) position contributes action likelihood. No future observations, + current actions, or current rewards are arguments to the history builder. + Return-to-go is an intentional conditioning label computed from rewards. + """ + + def __init__( + self, + sources: Sequence[str | Path | Trajectory], + config: CategoricalPolicyConfig, + *, + action_semantics: Literal["policy", "executed_predictions"] = "policy", + ): + if action_semantics not in {"policy", "executed_predictions"}: + raise ValueError( + "trajectory action_semantics must be policy or executed_predictions" + ) + self.config = config + self.action_semantics: Literal["policy", "executed_predictions"] = ( + action_semantics + ) + self.episodes: list[Trajectory] = [] + self.excluded_episode_ids: list[str] = [] + self.lengths: list[int] = [] + self.returns: list[list[float]] = [] + self.actions: list[list[int]] = [] + self.index: list[tuple[int, int]] = [] + seen = set() + for source in sources: + episode = ( + validate_trajectory(source) + if isinstance(source, Trajectory) + else load_trajectory(source) + ) + if episode.episode_id in seen: + raise ValueError(f"duplicate episode ID {episode.episode_id!r}") + seen.add(episode.episode_id) + if ( + episode.format != "sequifier.numeric-policy-trajectory" + and action_semantics != "executed_predictions" + ): + raise ValueError( + "forecast sources require explicit action_semantics='executed_predictions'; model forecast alignment is separate" + ) + if set(episode.seed.observations) != set(config.observation_columns) or set( + episode.seed.static_inputs + ) != set(config.static_columns): + raise ValueError( + "trajectory observations/static inputs do not match the policy schema" + ) + if ( + action_semantics == "policy" + and sum(episode.seed.attention_valid_mask) != 1 + ): + raise ValueError( + "policy episodes must start with one observation and an explicit previous-action start token" + ) + if episode.prediction_columns != [config.action_column]: + raise ValueError( + "policy trajectories must contain the configured single action head" + ) + length = labeled_episode_length(episode, config.returns) + if length is None: + self.excluded_episode_ids.append(episode.episode_id) + continue + actions = [ + action_index(transition.predictions[config.action_column], config) + for transition in episode.transitions[:length] + ] + returns = return_to_go( + [transition.reward for transition in episode.transitions[:length]], + config.returns.gamma, + ) + episode_index = len(self.episodes) + self.episodes.append(episode) + self.lengths.append(length) + self.actions.append(actions) + self.returns.append(returns) + self.index.extend((episode_index, step) for step in range(length)) + + def __len__(self): + return len(self.index) + + def __getitem__(self, index: int) -> PolicyExample: + episode_index, step = self.index[index] + episode = self.episodes[episode_index] + inputs = build_policy_inputs( + self.config, + [transition.observation for transition in episode.transitions[: step + 1]], + [len(self.config.action_values)] + self.actions[episode_index][:step], + self.returns[episode_index][: step + 1] + if self.config.return_conditioned + else None, + episode.seed.static_inputs, + ) + # Labels have their own mask; they are never passed to policy.forward(). + actions = torch.zeros((1, self.config.context_length), dtype=torch.int64) + actions[0, -1] = self.actions[episode_index][step] + valid = torch.zeros_like(actions, dtype=torch.bool) + valid[0, -1] = True + return PolicyExample(inputs, actions, valid, (episode.episode_id,), (step,)) + + def normalization_rows(self) -> tuple[torch.Tensor, torch.Tensor]: + observations = [ + [ + transition.observation[column] + for column in self.config.observation_columns + ] + for episode, length in zip(self.episodes, self.lengths) + for transition in episode.transitions[:length] + ] + static = [ + [ + episode.seed.static_inputs[column] + for column in self.config.static_columns + ] + for episode in self.episodes + ] + return torch.tensor(observations, dtype=torch.float32), torch.tensor( + static, dtype=torch.float32 + ) + + +def split_by_episode( + dataset: TrajectoryDataset, validation_fraction: float = 0.2, seed: int = 1010 +) -> tuple[TrajectoryDataset, TrajectoryDataset]: + if not 0 < validation_fraction < 1 or len(dataset.episodes) < 2: + raise ValueError( + "episode splitting requires at least two eligible episodes and validation_fraction in (0,1)" + ) + episodes = list(dataset.episodes) + random.Random(seed).shuffle(episodes) + count = min(len(episodes) - 1, max(1, int(len(episodes) * validation_fraction))) + return ( + TrajectoryDataset( + episodes[count:], dataset.config, action_semantics=dataset.action_semantics + ), + TrajectoryDataset( + episodes[:count], dataset.config, action_semantics=dataset.action_semantics + ), + ) diff --git a/src/sequifier/trajectory/returns.py b/src/sequifier/trajectory/returns.py new file mode 100644 index 00000000..b07843b9 --- /dev/null +++ b/src/sequifier/trajectory/returns.py @@ -0,0 +1,50 @@ +"""Exact recorded-reward returns and matching online return-budget arithmetic.""" + +from collections.abc import Sequence + +from sequifier.config.policy_config import ReturnDefinition +from sequifier.rollout.history import numeric_scalar +from sequifier.trajectory.schema import Trajectory + + +def return_to_go(rewards: Sequence[float], gamma: float = 1.0) -> list[float]: + gamma = numeric_scalar(gamma, "gamma") + if not 0 < gamma <= 1: + raise ValueError("gamma must be in (0, 1]") + result = [0.0] * len(rewards) + remaining = 0.0 + for index in range(len(rewards) - 1, -1, -1): + remaining = numeric_scalar( + numeric_scalar(rewards[index], "reward") + gamma * remaining, "return-to-go" + ) + result[index] = remaining + return result + + +def remaining_return(desired_return: float, reward: float, gamma: float = 1.0) -> float: + """If G_t = r_t + gamma G_(t+1), then G_(t+1) = (G_t-r_t)/gamma.""" + gamma = numeric_scalar(gamma, "gamma") + if not 0 < gamma <= 1: + raise ValueError("gamma must be in (0, 1]") + return numeric_scalar( + ( + numeric_scalar(desired_return, "desired_return") + - numeric_scalar(reward, "reward") + ) + / gamma, + "successor desired_return", + ) + + +def labeled_episode_length( + trajectory: Trajectory, definition: ReturnDefinition +) -> int | None: + """Exclude unknown tails; an explicit horizon labels only a fully observed prefix.""" + length = len(trajectory.transitions) + horizon = definition.finite_horizon + if horizon is not None and length >= horizon: + return horizon + if trajectory.transitions[-1].terminated: + return length + # A time limit is not a known terminal return without a finite-horizon objective. + return None diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index f340a89b..b73c9c97 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -1,10 +1,18 @@ -"""Inspectable version-1 single-episode forecast trajectory.""" +"""Inspectable version-1 numeric forecast and policy episode records.""" import json import math from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, FiniteFloat, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + FiniteFloat, + StrictInt, + TypeAdapter, + model_validator, +) from sequifier.config.rollout_config import InputBinding @@ -52,9 +60,9 @@ class Transition(Record): class Trajectory(Record): - format: Literal["sequifier.numeric-forecast-trajectory"] = ( - "sequifier.numeric-forecast-trajectory" - ) + format: Literal[ + "sequifier.numeric-forecast-trajectory", "sequifier.numeric-policy-trajectory" + ] = "sequifier.numeric-forecast-trajectory" version: Literal[1] = 1 episode_id: str = Field(min_length=1) seed: SeedHistory @@ -137,9 +145,85 @@ def validate_temporal_contract(self): self.total_return, expected_return, rel_tol=1e-12, abs_tol=1e-12 ): raise ValueError("total_return does not equal the sum of rewards") + if self.format == "sequifier.numeric-policy-trajectory": + if ( + sum(self.seed.attention_valid_mask) != 1 + or len(self.prediction_columns) != 1 + ): + raise ValueError( + "policy episodes require one initial observation and one action head" + ) + if "policy_execution" in self.metadata: + _validate_policy_execution(self) return self +def _validate_policy_execution(trajectory: Trajectory) -> None: + from sequifier.config.policy_config import CategoricalPolicyConfig + + execution = trajectory.metadata["policy_execution"] + if not isinstance(execution, dict): + raise ValueError("policy_execution must be an object") + if not isinstance(execution.get("deterministic"), bool): + raise ValueError( + "policy execution selection must declare a deterministic boolean" + ) + spec = CategoricalPolicyConfig.model_validate(execution.get("config")) + if ( + trajectory.prediction_columns != [spec.action_column] + or set(trajectory.seed.observations) != set(spec.observation_columns) + or set(trajectory.seed.static_inputs) != set(spec.static_columns) + ): + raise ValueError("policy execution schema does not match trajectory fields") + budgets = TypeAdapter(list[FiniteFloat]).validate_python( + execution.get("remaining_returns"), strict=True + ) + previous_actions = TypeAdapter(list[StrictInt]).validate_python( + execution.get("previous_action_ids"), strict=True + ) + log_probabilities = TypeAdapter(list[FiniteFloat]).validate_python( + execution.get("action_log_probabilities"), strict=True + ) + count = len(trajectory.transitions) + if ( + len(previous_actions) != count + or len(log_probabilities) != count + or any(value > 1e-6 for value in log_probabilities) + ): + raise ValueError( + "policy execution action IDs/log probabilities must match decision count" + ) + expected_previous = len(spec.action_values) + for index, step in enumerate(trajectory.transitions): + action = step.predictions[spec.action_column] + if ( + action not in spec.action_values + or previous_actions[index] != expected_previous + ): + raise ValueError( + "policy previous actions must lag executed categorical actions by one decision" + ) + expected_previous = spec.action_values.index(action) + if spec.return_conditioned: + initial = TypeAdapter(FiniteFloat).validate_python( + execution.get("initial_desired_return"), strict=True + ) + if len(budgets) != count + 1 or budgets[0] != initial: + raise ValueError( + "policy return budgets must include the initial and every successor budget" + ) + for index, step in enumerate(trajectory.transitions): + expected = (budgets[index] - step.reward) / spec.returns.gamma + if not math.isclose( + budgets[index + 1], expected, rel_tol=1e-12, abs_tol=1e-12 + ): + raise ValueError( + "policy return budgets violate the recorded reward/gamma arithmetic" + ) + elif budgets or execution.get("initial_desired_return") is not None: + raise ValueError("behavior cloning must not consume desired-return inputs") + + def validate_trajectory(value: Trajectory | dict) -> Trajectory: """Revalidate even an existing (possibly mutated) trajectory object.""" return Trajectory.model_validate( From 04090e37797eb15ae2276a4307251246a483ceb0 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 11:30:33 +0100 Subject: [PATCH 03/71] move policy to transformer --- .../configs/policy-control.yaml | 18 ++++++- documentation/policy-learning.md | 40 ++++++++++++++- src/sequifier/config/policy_config.py | 51 ++++++++++++++++--- src/sequifier/consolidated-docs.md | 40 ++++++++++++++- src/sequifier/rl/checkpoint.py | 9 ++-- src/sequifier/rl/experiment.py | 2 +- src/sequifier/rl/policy.py | 32 ++++++++---- 7 files changed, 165 insertions(+), 27 deletions(-) diff --git a/documentation/example_inputs/configs/policy-control.yaml b/documentation/example_inputs/configs/policy-control.yaml index a3b1432f..bb0303a4 100644 --- a/documentation/example_inputs/configs/policy-control.yaml +++ b/documentation/example_inputs/configs/policy-control.yaml @@ -3,7 +3,23 @@ output_dir: outputs/numeric-policy horizon: 3 repetitions: 16 context_length: 4 -hidden_size: 32 +backbone: + dim_model: 32 + num_layers: 2 + max_context_length: 4 + attention: + type: mha + n_heads: 4 + feed_forward: + dim: 128 + activation: swiglu + normalization: + type: rmsnorm + norm_first: true + position_encoding: + type: learned + positional_encoding_scope: global + dropout: 0.0 returns: gamma: 1.0 scale: 1.0 diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index a2139df2..78cc6499 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -66,14 +66,50 @@ trimmed together. Padding has a false attention mask. Only the last position of each training example has a true action-likelihood mask, so every recorded decision contributes once even when context windows overlap. -The actor uses numeric projection, previous-action embedding, a small causal GRU, -and one categorical head. Padded positions do not advance its hidden state. +The actor uses numeric projection, previous-action embedding, the same +`TransformerBackbone` and encoder layers as `sequifier train`, and one categorical +head. Attention is causal and excludes padded keys; padded logits are zero. +The model recomputes each finite context window rather than retaining recurrent +state across windows. Forward calls take `PolicyInputs`, which contain no action labels or observed current rewards. A change to a current-action label cannot affect that position's inputs; future observations are excluded from its window. Return-to-go is the intentional reward-derived training condition, with requested budgets replacing it online. Changing rewards can therefore change the return condition. +Both the command configuration and `CategoricalPolicyConfig` accept a `backbone` +section using the same architecture fields as training's +`model.backbone.architecture`: + +```yaml +context_length: 4 +backbone: + dim_model: 32 + num_layers: 2 + max_context_length: 4 + attention: {type: mha, n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} + normalization: {type: rmsnorm, norm_first: true} + position_encoding: {type: learned} + positional_encoding_scope: global + dropout: 0.0 + shared_layer_groups: [] +``` + +This includes MHA/MQA/GQA, feed-forward activation, normalization, positional +encoding (learned, RoPE, sinusoidal, range or range-concat), dropout and layer +sharing. `context_length` must fit `max_context_length`. Learned positions require +`global` scope because policy inputs use a combined numeric projection rather +than per-feature position embeddings. Scope defaults to `global` for policies. +Without `backbone`, defaults are two layers, width 32, one attention head and +feed-forward width 128. Configure width with `backbone.dim_model`; the experimental +`hidden_size` option has been removed. + +Transformer actor artifacts use version 2. Version 1 GRU artifacts cannot be +loaded into this architecture and must be retrained. Sharing backbone primitives +does not yet provide forecast-checkpoint transfer. The numeric policy input +projection, action head and CPU-only execution remain policy-specific. + Observation/static normalization is fit on the training episode partition only. Its fixed means/scales are model buffers, saved with the actor. The RC and BC actors use the same architecture, initialization seed, episode split, batch diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 97825d02..6794cbbb 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ConfigDict, Field, FiniteFloat, model_validator -from sequifier.config.components import ComponentSpec +from sequifier.config.components import BackboneArchitectureConfig, ComponentSpec class PolicyConfigBase(BaseModel): @@ -17,7 +17,48 @@ class ReturnDefinition(PolicyConfigBase): finite_horizon: int | None = Field(default=None, gt=0) -class CategoricalPolicyConfig(PolicyConfigBase): +class PolicyModelConfig(PolicyConfigBase): + context_length: int = Field(default=4, gt=0) + backbone: BackboneArchitectureConfig + + @model_validator(mode="before") + @classmethod + def resolve_backbone(cls, values): + if not isinstance(values, dict): + return values + values = dict(values) + if "backbone" not in values: + width = 32 + values["backbone"] = { + "dim_model": width, + "num_layers": 2, + "max_context_length": values.get("context_length", 4), + "attention": {"n_heads": 1}, + "feed_forward": {"dim": width * 4}, + "positional_encoding_scope": "global", + } + elif isinstance(values["backbone"], dict): + values["backbone"] = dict(values["backbone"]) + values["backbone"].setdefault("positional_encoding_scope", "global") + return values + + @model_validator(mode="after") + def validate_backbone(self): + if self.context_length > self.backbone.max_context_length: + raise ValueError( + "policy context_length exceeds backbone max_context_length" + ) + if ( + self.backbone.position_encoding.type == "learned" + and self.backbone.positional_encoding_scope != "global" + ): + raise ValueError( + "policy learned positions require positional_encoding_scope: global" + ) + return self + + +class CategoricalPolicyConfig(PolicyModelConfig): alignment: Literal["current_observation_previous_action"] = ( "current_observation_previous_action" ) @@ -27,8 +68,6 @@ class CategoricalPolicyConfig(PolicyConfigBase): action_values: list[FiniteFloat] = Field( default_factory=lambda: [0.0, 1.0], min_length=2 ) - context_length: int = Field(default=4, gt=0) - hidden_size: int = Field(default=32, gt=0) return_conditioned: bool = True returns: ReturnDefinition = Field(default_factory=ReturnDefinition) @@ -62,13 +101,11 @@ class PolicyTrainingConfig(PolicyConfigBase): layer_type_dtypes: None = None -class ControlExperimentConfig(PolicyConfigBase): +class ControlExperimentConfig(PolicyModelConfig): project_root: str = "." output_dir: str = "outputs/numeric-policy" horizon: int = Field(default=3, ge=1, le=8) repetitions: int = Field(default=16, ge=2) - context_length: int = Field(default=4, gt=0) - hidden_size: int = Field(default=32, gt=0) returns: ReturnDefinition = Field(default_factory=ReturnDefinition) training: PolicyTrainingConfig = Field(default_factory=PolicyTrainingConfig) requested_returns: list[FiniteFloat] | None = None diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 93bfeda8..f9cd3a58 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -2358,14 +2358,50 @@ trimmed together. Padding has a false attention mask. Only the last position of each training example has a true action-likelihood mask, so every recorded decision contributes once even when context windows overlap. -The actor uses numeric projection, previous-action embedding, a small causal GRU, -and one categorical head. Padded positions do not advance its hidden state. +The actor uses numeric projection, previous-action embedding, the same +`TransformerBackbone` and encoder layers as `sequifier train`, and one categorical +head. Attention is causal and excludes padded keys; padded logits are zero. +The model recomputes each finite context window rather than retaining recurrent +state across windows. Forward calls take `PolicyInputs`, which contain no action labels or observed current rewards. A change to a current-action label cannot affect that position's inputs; future observations are excluded from its window. Return-to-go is the intentional reward-derived training condition, with requested budgets replacing it online. Changing rewards can therefore change the return condition. +Both the command configuration and `CategoricalPolicyConfig` accept a `backbone` +section using the same architecture fields as training's +`model.backbone.architecture`: + +```yaml +context_length: 4 +backbone: + dim_model: 32 + num_layers: 2 + max_context_length: 4 + attention: {type: mha, n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} + normalization: {type: rmsnorm, norm_first: true} + position_encoding: {type: learned} + positional_encoding_scope: global + dropout: 0.0 + shared_layer_groups: [] +``` + +This includes MHA/MQA/GQA, feed-forward activation, normalization, positional +encoding (learned, RoPE, sinusoidal, range or range-concat), dropout and layer +sharing. `context_length` must fit `max_context_length`. Learned positions require +`global` scope because policy inputs use a combined numeric projection rather +than per-feature position embeddings. Scope defaults to `global` for policies. +Without `backbone`, defaults are two layers, width 32, one attention head and +feed-forward width 128. Configure width with `backbone.dim_model`; the experimental +`hidden_size` option has been removed. + +Transformer actor artifacts use version 2. Version 1 GRU artifacts cannot be +loaded into this architecture and must be retrained. Sharing backbone primitives +does not yet provide forecast-checkpoint transfer. The numeric policy input +projection, action head and CPU-only execution remain policy-specific. + Observation/static normalization is fit on the training episode partition only. Its fixed means/scales are model buffers, saved with the actor. The RC and BC actors use the same architecture, initialization seed, episode split, batch diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 508b2951..24f7c1fe 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -26,7 +26,7 @@ def save_policy( json.dumps(metadata or {}, allow_nan=False) payload = { "format": "sequifier.numeric-categorical-policy", - "version": 1, + "version": 2, "config": policy.config.model_dump(mode="json"), "state_dict": policy.state_dict(), "metadata": metadata or {}, @@ -53,9 +53,12 @@ def load_policy(path: str | Path) -> CategoricalPolicy: if ( not isinstance(payload, dict) or payload.get("format") != "sequifier.numeric-categorical-policy" - or payload.get("version") != 1 + or payload.get("version") != 2 ): - raise ValueError("unsupported numeric categorical policy artifact") + raise ValueError( + "unsupported numeric categorical policy artifact; transformer actors require " + "version 2 (version 1 GRU weights cannot be loaded)" + ) config = CategoricalPolicyConfig.model_validate(payload["config"]) json.dumps(payload.get("metadata", {}), allow_nan=False) with torch.random.fork_rng(devices=[]): diff --git a/src/sequifier/rl/experiment.py b/src/sequifier/rl/experiment.py index a33955f8..f26f2ef4 100644 --- a/src/sequifier/rl/experiment.py +++ b/src/sequifier/rl/experiment.py @@ -140,7 +140,7 @@ def run_control_experiment(config: ControlExperimentConfig) -> dict: spec = CategoricalPolicyConfig( observation_columns=["position", "remaining_steps"], context_length=config.context_length, - hidden_size=config.hidden_size, + backbone=config.backbone, returns=config.returns, ) corpus = TrajectoryDataset(paths, spec) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index fce79f81..3b03050f 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -1,9 +1,11 @@ -"""A small causal numeric actor with one categorical head and no value network.""" +"""A causal numeric transformer actor with one categorical head and no value network.""" import torch from torch import Tensor, nn from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.model.backbone import TransformerBackbone +from sequifier.objectives import CausalObjective from sequifier.rl.distributions import CategoricalActionDistribution from sequifier.rl.history import PolicyInputs @@ -12,14 +14,19 @@ class CategoricalPolicy(nn.Module): def __init__(self, config: CategoricalPolicyConfig): super().__init__() self.config = config - width = config.hidden_size + self.backbone = TransformerBackbone(config.backbone) + width = self.backbone.input_dim numeric_width = len(config.observation_columns) + len(config.static_columns) + 1 self.ingestion = nn.Linear(numeric_width, width) self.previous_action_embedding = nn.Embedding( len(config.action_values) + 1, width ) - self.backbone = nn.GRUCell(width, width) - self.decoder = nn.Linear(width, len(config.action_values)) + self.decoder = nn.Linear(config.backbone.dim_model, len(config.action_values)) + self.register_buffer( + "causal_attention_mask", + CausalObjective.build_attention_mask_policy(config.context_length), + persistent=False, + ) self.register_buffer( "observation_mean", torch.zeros(len(config.observation_columns)) ) @@ -44,6 +51,8 @@ def set_normalization(self, observations: Tensor, static_inputs: Tensor) -> None raise ValueError( "normalization requires finite training rows matching policy columns" ) + if not mean.numel(): + continue fitted = values.to(torch.float64) mean.copy_(fitted.mean(dim=0)) scale.copy_(fitted.std(dim=0, unbiased=False).clamp_min(1e-6)) @@ -72,13 +81,14 @@ def forward(self, inputs: PolicyInputs) -> Tensor: self.ingestion(numeric) + self.previous_action_embedding(inputs.previous_actions) ) - hidden = encoded.new_zeros((batch, self.config.hidden_size)) - outputs = [] - for position in range(context): - candidate = self.backbone(encoded[:, position], hidden) - hidden = torch.where(mask[:, position, None], candidate, hidden) - outputs.append(self.decoder(hidden)) - return torch.stack(outputs, dim=1) + # As in ModelInterfaceModule, combine the objective's causal mask with + # finite padding penalties so left-padded query rows remain well-defined. + attention_mask = self.causal_attention_mask[None, None, :, :] + torch.zeros( + batch, 1, 1, context, dtype=encoded.dtype, device=encoded.device + ).masked_fill(~mask[:, None, None, :], torch.finfo(encoded.dtype).min) + hidden = self.backbone(encoded, attention_mask) + logits = self.decoder(hidden) + return logits.masked_fill(~mask[:, :, None], 0.0) def distribution(self, inputs: PolicyInputs) -> CategoricalActionDistribution: return CategoricalActionDistribution(self(inputs)) From 43e44a937e3bde1ae3da8730849721a45e3151be Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 11:53:15 +0100 Subject: [PATCH 04/71] unify --- README.md | 3 +- documentation/configs/train.md | 12 + .../configs/policy-control.yaml | 33 -- .../configs/train-return-conditioned.yaml | 47 +++ ...nforcement-learning-implementation-plan.md | 13 +- documentation/policy-learning.md | 150 +++++---- documentation/rollout.md | 48 +-- release-notes/unreleased-policy-training.md | 21 ++ .../canonical_hyperparameter_search_config.py | 5 + src/sequifier/config/infer_config.py | 6 + src/sequifier/config/policy_config.py | 28 +- src/sequifier/config/policy_train_config.py | 133 ++++++++ src/sequifier/config/train_config.py | 41 ++- src/sequifier/consolidated-docs.md | 165 +++++----- src/sequifier/make.py | 29 ++ src/sequifier/rl/experiment.py | 294 ------------------ src/sequifier/rl/learner.py | 184 +++++++++++ src/sequifier/rl/training.py | 14 +- src/sequifier/rl_api.py | 27 +- src/sequifier/rollout/control.py | 64 ---- src/sequifier/rollout/dependent.py | 50 --- src/sequifier/sequifier.py | 11 - src/sequifier/train.py | 12 +- src/sequifier/trajectory/dataset.py | 36 ++- 24 files changed, 736 insertions(+), 690 deletions(-) delete mode 100644 documentation/example_inputs/configs/policy-control.yaml create mode 100644 documentation/example_inputs/configs/train-return-conditioned.yaml create mode 100644 release-notes/unreleased-policy-training.md create mode 100644 src/sequifier/config/policy_train_config.py delete mode 100644 src/sequifier/rl/experiment.py create mode 100644 src/sequifier/rl/learner.py delete mode 100644 src/sequifier/rollout/control.py delete mode 100644 src/sequifier/rollout/dependent.py diff --git a/README.md b/README.md index 82e876b9..17f8066a 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ There are seven standalone commands within sequifier: `make`, `preprocess`, `tra | --- | --- | | `make` | Create a new sequifier project with config templates. | | `preprocess` | Convert input data into fixed-length subsequences. | -| `train` | Train a model on preprocessed data. | +| `train` | Train a supervised model or a trajectory-backed categorical policy. | | `infer` | Generate predictions, probabilities, or embeddings. | | `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | | `visualize-training` | Generate interactive HTML plots from structured training metrics. | @@ -116,6 +116,7 @@ The processing and visualization commands have dedicated documentation pages: - [preprocess documentation](./documentation/configs/preprocess.md) - [train documentation](./documentation/configs/train.md) +- [policy training](./documentation/policy-learning.md) - [infer documentation](./documentation/configs/infer.md) - [hyperparameter-search documentation](./documentation/configs/hyperparameter-search.md) - [visualize-training documentation](./documentation/commands/visualize-training.md) diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 5893fbfb..0752cb44 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -767,3 +767,15 @@ frozen modules. At optimizer boundaries, frozen gradients are removed **before** AMP unscaling/overflow detection, so a frozen-only overflow cannot suppress a healthy active update. Momentum and weight decay cannot update parameters whose gradients are absent. + + +## Trajectory-backed policy training + +`sequifier train` also accepts one `return_conditioned` phase with named +trajectory sources and a categorical policy registry. This CPU workflow supports +return conditioning or BC, with no supervised dataset placeholders. See the +[policy training guide](../policy-learning.md) and +[complete example](../example_inputs/configs/train-return-conditioned.yaml). +The phase owns `epochs`; the run owns `seed`; `optimization` configures updates. +Existing phases default to `algorithm: supervised`. Mixed algorithm plans and +IQL/PPO remain planned. diff --git a/documentation/example_inputs/configs/policy-control.yaml b/documentation/example_inputs/configs/policy-control.yaml deleted file mode 100644 index bb0303a4..00000000 --- a/documentation/example_inputs/configs/policy-control.yaml +++ /dev/null @@ -1,33 +0,0 @@ -project_root: . -output_dir: outputs/numeric-policy -horizon: 3 -repetitions: 16 -context_length: 4 -backbone: - dim_model: 32 - num_layers: 2 - max_context_length: 4 - attention: - type: mha - n_heads: 4 - feed_forward: - dim: 128 - activation: swiglu - normalization: - type: rmsnorm - norm_first: true - position_encoding: - type: learned - positional_encoding_scope: global - dropout: 0.0 -returns: - gamma: 1.0 - scale: 1.0 -training: - epochs: 80 - batch_size: 64 - learning_rate: 0.005 - seed: 1010 - validation_fraction: 0.2 -requested_returns: [0.0, 3.0] -sampling_episodes: 8 diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml new file mode 100644 index 00000000..b6c5636e --- /dev/null +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -0,0 +1,47 @@ +project_root: . +model_name: policy +device: cpu +seed: 1010 + +trajectory_sources: + demonstrations: + # Paths inside the manifest resolve relative to the manifest directory. + manifest: data/trajectories/manifest.json + action_semantics: policy + +# Set observation columns and action values to match your recorded trajectories. +policies: + actor: + observation_columns: [OBSERVATION_COLUMN] + action_column: action + action_values: [0.0, 1.0] + context_length: 4 + return_conditioned: true + returns: {gamma: 1.0, scale: 1.0} + backbone: + dim_model: 32 + num_layers: 2 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} + positional_encoding_scope: global + dropout: 0.0 + +training_plan: + phases: + - name: offline_policy + algorithm: return_conditioned + epochs: 80 + trajectory_sources: [demonstrations] + policy: actor + +optimization: + batch_size: 64 + learning_rate: 0.005 + gradient_clip: 1.0 + validation_fraction: 0.2 + optimizer: {name: Adam} + scheduler: {name: ConstantLR, factor: 1.0, total_iters: 1} + +actor_path: models/policy/actor.pt +report_path: outputs/policy/training-report.json diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md index c522c5b3..8d0ca600 100644 --- a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md +++ b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md @@ -12,12 +12,21 @@ written or run for this slice, as requested. Second implementation slice: episode-split trajectory datasets, causal current-observation/previous-action windows, discounted return labels, a single categorical actor, return-conditioned supervised training and BC, local actor -artifacts, and execution through the same coordinator. The tiny binary-control -workflow records achieved returns after reloading both actors. See +artifacts, and execution through the same coordinator. See [policy learning](../policy-learning.md). This slice has not been executed or acceptance-tested; no tests were written or run, as requested. IQL and the remaining broader stages are still planned; RC is an independent route. +Third implementation slice: `sequifier train` resolves a single +`return_conditioned` phase with named trajectory sources and policies, an +algorithm runtime boundary, grouped episode splitting, best-action-NLL selection, +actor-only artifacts and training reports. Synthetic demo code, demo-specific +configurations/public exports and the experimental `policy` command are removed. +Applications supply their own recorded trajectories and environment adapters. JSON episodes and the CPU categorical actor remain interim contracts. +Mixed phases, phase transfer, sharded/multimodal storage, general rollout +infrastructure, IQL/PPO, distributed RL, exact resume and exports remain planned. +No tests were written or run for this increment, as requested. + Deliver this as one coordinated update, built and verified in the three dependency-ordered stages below. Completion means all three stages work together; stage boundaries are engineering checkpoints, not separate proposed releases. This document specifies new behavior and proposed API names, not functionality already present. ## 1. Outcome and scope diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 78cc6499..3c39e197 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -1,49 +1,60 @@ -# Policy Learning Command Guide +# Policy training through `sequifier train` -`sequifier policy` runs a local, CPU-only categorical control experiment: +Train an actor on saved trajectories with: ```console -sequifier policy --config-path documentation/example_inputs/configs/policy-control.yaml +sequifier train --config-path documentation/example_inputs/configs/train-return-conditioned.yaml ``` -It collects trajectories, computes return-to-go labels, splits by episode, -trains return-conditioned and behavior-cloning actors, saves/reloads both, and -executes them through the existing rollout coordinator. `report.json` records -the achieved returns. This command is separate from forecast model training and -inference; it does not change `target_offset` or convert forecast checkpoints -into policies. `sequifier docs policy` prints this guide. - -## Task and collected data - -The deterministic binary control task starts at `position=0` with three decisions -remaining. Action `0` holds position; action `1` increments it. Each action earns -its numeric value as reward. Observations contain current `position` and -`remaining_steps`. The environment terminates after its last decision and returns -the actual final successor. The horizon is configurable from 1 through 8. - -Collection enumerates every binary action sequence, repeating each 16 times by -default. For horizon 3 that gives 128 episodes and 384 executed decisions from -the same starting conditions, with returns from 0 through 3 and a behavior mean -of 1.5. Repetition gives conflicting action labels for identical observations -while retaining examples at each return level. The dataset manifest lists the -exact collected files and their episode IDs; stale files are not added to the -current experiment. - -The example configuration writes these files beneath `outputs/numeric-policy`: - -- `dataset/sequence-*.json`: recorded episodes in original numeric units. -- `dataset/manifest.json`: exact source files, episode IDs, and raw returns. -- `dataset/return_labels.json`: the return definition and raw per-decision labels. -- `return_conditioned.pt` and `behavior_cloning.pt`: local actor artifacts. -- `evaluation/*.json`: trajectories executed using the reloaded actors. -- `report.json`: episode partitions, per-epoch action likelihood, checkpoint - selection, greedy/sampled actions, achieved returns, and requested-return errors. - -The report distinguishes raw episode reward sums from discounted achieved -returns. It compares both actors at the same requested returns, initial states, -and sampling seeds. The BC actor does not receive the requested return. This is -a controlled learning comparison; performance depends on training configuration -and is recorded rather than assumed. +The configuration declares named `trajectory_sources`, named `policies`, and one +`training_plan.phases` entry with `algorithm: return_conditioned`. Ordinary +training phases default to `algorithm: supervised`. Policy phases have their own +source and model contract and require no supervised dataset, criterion, or +preprocessing metadata. This increment supports CPU execution and one categorical +actor in one phase. Mixed algorithm plans, phase transfer, IQL, PPO, distributed +policy learning, custom training integrations, and policy export remain planned. +Unsupported run fields are rejected before learning. + +A trajectory source specifies either `paths: [episode-a.json, episode-b.json]` or +`manifest: dataset/manifest.json`. A version-1 manifest has an `episodes` list of +objects with a `path` for each file. Only listed files are loaded. Run paths resolve +relative to `project_root`; manifest entries resolve relative to the manifest's +directory. These JSON episodes are the interim numeric trajectory format, preceding +the plan's sharded multimodal storage. + +The phase references `trajectory_sources: [demonstrations]` and `policy: actor`. +Set `epochs` on the phase and `seed` on the run. Optimizer, scheduler, batch size, +gradient clipping and automatic `validation_fraction` belong in `optimization`. +The actor's `returns` declares gamma, scale, and optional finite horizon. A concise +`training_plan: {algorithm: return_conditioned, epochs: 80, trajectory_sources: +[demonstrations], policy: actor}` is also accepted. + +Without `validation_sources`, the runner splits whole episodes deterministically, +keeping identical recorded trajectories together. Repeated copies therefore do +not cross the partition boundary. At least two distinct trajectory groups are +required. For authored partitions, add `validation_sources: [held_out]` to the +phase and declare that separate source in the registry. Duplicate file paths, +episode IDs, or identical trajectory content across partitions are rejected. +Normalization is fit only on training episodes. Truncations and collection cutoffs +with unknown returns are excluded under the existing return-label contract; +excluded IDs are recorded in the report. + +The runtime uses the existing optimization transactions and selects the best +actor by held-out action NLL. `actor_path` defaults to +`models//actor.pt`; `report_path` defaults to +`outputs/training//report.json`. The report records resolved config, +selected files, partition IDs, decision counts, excluded episodes, per-epoch NLL, +optimizer steps and checkpoint selection. Actor files contain model weights, +normalization buffers, vocabulary, alignment and return settings. They are +actor-only inference artifacts; optimizer and RNG state for exact resume are not +included. Use `load_policy` and `collect_policy_rollout` to execute them. + +For behavior cloning, set the actor's `return_conditioned: false` in the same +workflow. Offline training requires only saved trajectories; it does not start an +environment. Python callers use `load_train_config` and `run_training` with the +same configuration as the CLI. + +`sequifier docs policy` prints this guide. ## Policy alignment and causal windows @@ -52,14 +63,6 @@ conditioning, `G_t`. The current action `a_t` is a separate target. The first previous action is a dedicated start-token ID equal to the number of executable actions. It cannot be selected by the action head. -For the sequence `[1, 0, 1]`, gamma 1, and the default task: - -| Decision | Current observation | Previous action ID | Desired return | Action target | -| --- | --- | --- | --- | --- | -| 0 | `(position=0, remaining_steps=3)` | 2 (start) | 2 | 1 | -| 1 | `(position=1, remaining_steps=2)` | 1 | 1 | 0 | -| 2 | `(position=1, remaining_steps=1)` | 0 | 1 | 1 | - Offline and online execution use `build_policy_inputs` to construct the same left-padded causal windows. Observations, previous actions, and returns are trimmed together. Padding has a false attention mask. Only the last position @@ -77,8 +80,8 @@ inputs; future observations are excluded from its window. Return-to-go is the intentional reward-derived training condition, with requested budgets replacing it online. Changing rewards can therefore change the return condition. -Both the command configuration and `CategoricalPolicyConfig` accept a `backbone` -section using the same architecture fields as training's +Each entry in the `policies` registry and `CategoricalPolicyConfig` accepts a +`backbone` section using the same architecture fields as training's `model.backbone.architecture`: ```yaml @@ -111,10 +114,8 @@ does not yet provide forecast-checkpoint transfer. The numeric policy input projection, action head and CPU-only execution remain policy-specific. Observation/static normalization is fit on the training episode partition only. -Its fixed means/scales are model buffers, saved with the actor. The RC and BC -actors use the same architecture, initialization seed, episode split, batch -order, and optimization configuration. BC disables the return channel by -feeding zero; it retains the same parameter count. +Its fixed means/scales are model buffers, saved with the actor. BC disables the return channel by feeding zero and retains the same parameter +count as the corresponding return-conditioned actor. ## Return definitions and episode boundaries @@ -164,25 +165,17 @@ owned by the shared coordinator on success and failure. ## Reusable Python interfaces -The public entry points live in `sequifier.rl_api`. To train on compatible saved -policy episodes: +The public policy entry points live in `sequifier.rl_api`. Python callers can +use the same configuration loader and training entry point as the CLI: ```python -from sequifier.rl_api import ( - CategoricalPolicyConfig, PolicyTrainingConfig, TrajectoryDataset, - load_policy, save_policy, split_by_episode, train_policy, -) +from sequifier.config.train_config import load_train_config +from sequifier.train import run_training +from sequifier.rl_api import load_policy -spec = CategoricalPolicyConfig( - observation_columns=["position", "remaining_steps"], - action_column="action", - action_values=[0.0, 1.0], -) -dataset = TrajectoryDataset(episode_paths, spec) -training, validation = split_by_episode(dataset, validation_fraction=0.2, seed=1010) -result = train_policy(training, validation, PolicyTrainingConfig()) -save_policy(result.policy, "actor.pt") -actor = load_policy("actor.pt") +config = load_train_config("configs/train-policy.yaml", {}, False) +run_training(config) +actor = load_policy(config.actor_path) ``` First-increment forecast trajectories can also supply demonstrations when their @@ -205,18 +198,21 @@ uses the dataset's independent action-valid mask. Training calls the existing `OptimizationRuntime` for backward, clipping, optimizer, scaler, and scheduler steps. The validation partition selects the saved best actor by action NLL. +For execution, supply an application-owned environment adapter, a validated +`SeedHistory`, and a `RolloutConfig` with `mode: policy`. Bind each changing input +to the appropriate environment field or executed action. The adapter follows the +[rollout protocol](rollout.md#adapter-protocol). + ```python from sequifier.rl_api import collect_policy_rollout -from sequifier.rollout.control import BinaryControlAdapter -from sequifier.rl.experiment import control_rollout_config, control_seed from sequifier.rollout_api import save_trajectory -rollout = control_rollout_config("execution.json", horizon=3) +# actor, adapter, seed, rollout_config and requested_return come from the application. trajectory = collect_policy_rollout( - actor, BinaryControlAdapter(horizon=3), control_seed(horizon=3), - rollout, max_steps=3, desired_return=3.0, deterministic=True, + actor, adapter, seed, rollout_config, + max_steps=1000, desired_return=requested_return, deterministic=True, ) -save_trajectory(trajectory, rollout.trajectory_path) +save_trajectory(trajectory, rollout_config.trajectory_path) ``` Set `return_conditioned=False` in the policy specification to train BC, and omit diff --git a/documentation/rollout.md b/documentation/rollout.md index dccde7de..1f0181b3 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -24,18 +24,17 @@ rollout: seed_index: 0 trajectory_path: outputs/trajectory.json adapter: - factory: sequifier.rollout.dependent:LinearDependentAdapter + factory: your_application.adapters:make_environment protocol_version: 1 - config: - prediction: x - external_input: d - multiplier: 2.0 - terminate_after: 3 + config: {} # keyword arguments accepted by your factory bindings: - {input: x, source: prediction, key: x} - {input: d, source: environment, key: d} ``` +Replace the adapter factory with your application's importable `module:factory`. +It must implement the protocol below and provide the bound environment fields. + Run the existing `sequifier infer --config-path configs/infer.yaml` command. PT and existing schema-bearing ONNX models use the same coordinator and their existing inference backends. This feature adds no export format. @@ -142,46 +141,15 @@ the predictor's representation; the inference integration uses the fixed codec and existing `ExecutionSchema` here. ```python -from sequifier.rollout_api import ( - RolloutConfig, SeedHistory, collect_rollout, save_trajectory, -) -from sequifier.rollout.dependent import LinearDependentAdapter - -config = RolloutConfig( - adapter={"factory": "sequifier.rollout.dependent:LinearDependentAdapter"}, - bindings=[ - {"input": "x", "source": "prediction", "key": "x"}, - {"input": "d", "source": "environment", "key": "d"}, - ], - seed_index=0, - trajectory_path="outputs/trajectory.json", -) -seed = SeedHistory( - observations={"x": [0.0, 0.0], "d": [0.0, 0.0]}, - attention_valid_mask=[False, True], - static_inputs={}, -) - -def predict(history): - return {"x": history.observations["x"][-1] + history.observations["d"][-1] + 1.0} +from sequifier.rollout_api import collect_rollout, save_trajectory +# predictor, adapter, seed and config are provided by your application. trajectory = collect_rollout( - predict, LinearDependentAdapter(), seed, config, ["x"], max_steps=3, + predictor, adapter, seed, config, prediction_columns=["x"], max_steps=1000, ) save_trajectory(trajectory, config.trajectory_path) ``` -The hand-calculated trajectory for that predictor is: - -| Step ID | Observation | Prediction | Successor | Reward | Terminated | -| --- | --- | --- | --- | --- | --- | -| 0 | `(0, 0)` | `x=1` | `(1, 2)` | 1 | false | -| 1 | `(1, 2)` | `x=4` | `(4, 8)` | 4 | false | -| 2 | `(4, 8)` | `x=13` | `(13, 26)` | 13 | true | - -The return is 18, with no collection cutoff. The first mask is `[False, True]`; -the next and subsequent masks are `[True, True]`. No fourth call occurs. - Multimodal and categorical observations, reset-only initialization, vectorized environments, remote transport, learning algorithms, and new export support are outside this slice. diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md new file mode 100644 index 00000000..c38646ed --- /dev/null +++ b/release-notes/unreleased-policy-training.md @@ -0,0 +1,21 @@ +# Trajectory-backed policy training (unreleased) + +`sequifier train` accepts a single `return_conditioned` phase with named trajectory +sources and categorical actors. Set `return_conditioned: false` on the actor to +train behavior cloning through the same workflow. Sources use explicit JSON file +lists or version-1 manifests, with automatic grouped episode splitting or authored +training/validation sources. Normalization fits the training partition only. + +Runs save a best-action-NLL actor and a training report with source paths, +partition IDs, exclusions, metrics and optimizer-step counts. These CPU actor +artifacts support `load_policy` and `collect_policy_rollout`; exact training resume +and policy exports remain planned. + +The experimental `sequifier policy` command and synthetic control/dependent-feature +demos have been removed, including their configuration types and public exports. +Applications supply their own trajectories and environment adapters. +`sequifier make` now includes `configs/train-policy.yaml`. + +Mixed algorithm phases, transfer, IQL/PPO, distributed RL, multimodal trajectories +and scalable sharded storage remain future increments. No tests or training runs +were executed for this increment, as requested. diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index ebff5c2c..ae023aec 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -872,6 +872,11 @@ def compile_canonical_hyperparameter_search_config( f"referenced by {config_path!r}: {error}" ) from error + if not isinstance(base_config, SequifierConfig): + raise ValueError( + "hyperparameter search for trajectory training is not yet supported" + ) + search_values = copy.deepcopy(config_values) search_values["base_config_path"] = base_config_path search_values.setdefault("project_root", base_config.project_root) diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index dadc6531..bc750bc1 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -432,6 +432,12 @@ def load_inferer_config( from sequifier.config.train_config import load_train_config training = load_train_config(training_config_path, {}, False) + from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig + + if isinstance(training, ResolvedPolicyTrainConfig): + raise ValueError( + "policy actors execute through collect_policy_rollout; forecast inference requires a supervised training config" + ) datasets = training.dataset_training if dataset_name is not None: if dataset_name not in datasets: diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 6794cbbb..5651970f 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -1,4 +1,4 @@ -"""Standalone, CPU-only categorical policy learning configuration.""" +"""CPU categorical policy model and optimization configuration.""" from typing import Literal @@ -83,10 +83,8 @@ def validate_names(self): return self -class PolicyTrainingConfig(PolicyConfigBase): - epochs: int = Field(default=80, gt=0) +class PolicyOptimizationConfig(PolicyConfigBase): batch_size: int = Field(default=64, gt=0) - seed: int = Field(default=1010, ge=0) validation_fraction: float = Field(default=0.2, gt=0, lt=1) learning_rate: float = Field(default=0.005, gt=0, allow_inf_nan=False) gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) @@ -101,22 +99,6 @@ class PolicyTrainingConfig(PolicyConfigBase): layer_type_dtypes: None = None -class ControlExperimentConfig(PolicyModelConfig): - project_root: str = "." - output_dir: str = "outputs/numeric-policy" - horizon: int = Field(default=3, ge=1, le=8) - repetitions: int = Field(default=16, ge=2) - returns: ReturnDefinition = Field(default_factory=ReturnDefinition) - training: PolicyTrainingConfig = Field(default_factory=PolicyTrainingConfig) - requested_returns: list[FiniteFloat] | None = None - sampling_episodes: int = Field(default=8, gt=0) - - @model_validator(mode="after") - def validate_horizon(self): - if self.returns.finite_horizon not in (None, self.horizon): - raise ValueError( - "control experiment finite_horizon must match its task horizon" - ) - if self.requested_returns == []: - raise ValueError("requested_returns must be nonempty when supplied") - return self +class PolicyTrainingConfig(PolicyOptimizationConfig): + epochs: int = Field(default=80, gt=0) + seed: int = Field(default=1010, ge=0) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py new file mode 100644 index 00000000..ceaf8279 --- /dev/null +++ b/src/sequifier/config/policy_train_config.py @@ -0,0 +1,133 @@ +"""Authored and resolved trajectory training contracts for the train command.""" + +from pathlib import Path +from typing import Literal + +from pydantic import Field, field_validator, model_validator + +from sequifier.config.policy_config import ( + CategoricalPolicyConfig, + PolicyConfigBase, + PolicyOptimizationConfig, +) + + +class TrajectorySourceConfig(PolicyConfigBase): + paths: list[str] = Field(default_factory=list) + manifest: str | None = None + action_semantics: Literal["policy", "executed_predictions"] = "policy" + + @model_validator(mode="after") + def validate_location(self): + if bool(self.paths) == (self.manifest is not None): + raise ValueError("trajectory source requires either paths or one manifest") + if any(not path for path in self.paths) or self.manifest == "": + raise ValueError("trajectory paths must be nonempty") + return self + + +class ReturnConditionedPhaseConfig(PolicyConfigBase): + algorithm: Literal["return_conditioned"] + name: str = Field(default="policy", min_length=1) + epochs: int = Field(gt=0) + trajectory_sources: list[str] = Field(min_length=1) + validation_sources: list[str] | None = Field(default=None, min_length=1) + policy: str = Field(min_length=1) + + +class PolicyTrainingPlanConfig(PolicyConfigBase): + phases: list[ReturnConditionedPhaseConfig] = Field(min_length=1, max_length=1) + + +class _PolicyTrainConfigBase(PolicyConfigBase): + project_root: str = "." + model_name: str = Field(min_length=1) + device: Literal["cpu"] = "cpu" + seed: int = Field(default=1010, ge=0) + trajectory_sources: dict[str, TrajectorySourceConfig] = Field(min_length=1) + policies: dict[str, CategoricalPolicyConfig] = Field(min_length=1) + training_plan: PolicyTrainingPlanConfig + # Epochs and seed belong to the phase and run respectively. + optimization: PolicyOptimizationConfig = Field( + default_factory=PolicyOptimizationConfig + ) + + @field_validator("model_name") + @classmethod + def validate_model_name(cls, value): + if value in {".", ".."} or "/" in value or "\\" in value: + raise ValueError("model_name must be a single path component") + return value + + @model_validator(mode="before") + @classmethod + def normalize_plan(cls, values): + if not isinstance(values, dict): + return values + values = dict(values) + plan = values.get("training_plan") + if isinstance(plan, dict) and "phases" not in plan: + values["training_plan"] = {"phases": [plan]} + optimization = values.get("optimization", {}) + if isinstance(optimization, dict) and {"epochs", "seed"} & optimization.keys(): + raise ValueError("set epochs on the phase and seed on the run") + return values + + @model_validator(mode="after") + def validate_references(self): + phase = self.training_plan.phases[0] + if phase.policy not in self.policies: + raise ValueError(f"unknown policy {phase.policy!r}") + for names in (phase.trajectory_sources, phase.validation_sources or []): + if len(names) != len(set(names)): + raise ValueError("trajectory source references must be unique") + unknown = set(names) - self.trajectory_sources.keys() + if unknown: + raise ValueError(f"unknown trajectory sources: {sorted(unknown)}") + if set(phase.trajectory_sources) & set(phase.validation_sources or []): + raise ValueError("training and validation sources must be disjoint") + return self + + @field_validator("actor_path", "report_path", check_fields=False) + @classmethod + def validate_output_path(cls, value): + if value == "": + raise ValueError("output paths must be nonempty") + return value + + +class PolicyTrainConfig(_PolicyTrainConfigBase): + """Authored paths may be omitted to select the standard output locations.""" + + actor_path: str | None = None + report_path: str | None = None + + +class ResolvedPolicyTrainConfig(_PolicyTrainConfigBase): + """Run paths are absolute and the authored phase has been validated.""" + + actor_path: str + report_path: str + + +def resolve_policy_train_config(config: PolicyTrainConfig) -> ResolvedPolicyTrainConfig: + root = Path(config.project_root).resolve() + + def absolute(path): + return str((root / path).resolve()) + + values = config.model_dump(mode="json") + values["project_root"] = str(root) + for source in values["trajectory_sources"].values(): + source["paths"] = [absolute(path) for path in source["paths"]] + if source["manifest"] is not None: + source["manifest"] = absolute(source["manifest"]) + values["actor_path"] = absolute( + config.actor_path or f"models/{config.model_name}/actor.pt" + ) + values["report_path"] = absolute( + config.report_path or f"outputs/training/{config.model_name}/report.json" + ) + if values["actor_path"] == values["report_path"]: + raise ValueError("actor_path and report_path must differ") + return ResolvedPolicyTrainConfig.model_validate(values) diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 5a4809e5..43a85caa 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -59,6 +59,11 @@ OptimizerPlan, resolve_plan_scheduler_arguments, ) +from sequifier.config.policy_train_config import ( + PolicyTrainConfig, + ResolvedPolicyTrainConfig, + resolve_policy_train_config, +) from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -123,7 +128,9 @@ def _unique_columns(value: list[str], usage: str) -> list[str]: _SINGLETON_CONFIG_NAME = "default" _SINGLE_PHASE_NAME = "train" -_PHASE_FIELD_NAMES = frozenset({"name", "epochs", "mode", "selection", "sources"}) +_PHASE_FIELD_NAMES = frozenset( + {"algorithm", "name", "epochs", "mode", "selection", "sources"} +) _TRAINING_PLAN_POLICY_FIELDS = frozenset({"curriculum_training", "curriculum_column"}) @@ -832,6 +839,8 @@ def validate_source(cls, value): class TrainingPhaseSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") + algorithm: Literal["supervised"] = "supervised" + name: str epochs: int = Field(gt=0) mode: Literal["sequential", "interleaved"] @@ -1231,6 +1240,8 @@ class ResolvedTrainingSource(BaseModel): class ResolvedTrainingPhase(BaseModel): model_config = ConfigDict(extra="forbid") + algorithm: Literal["supervised"] = "supervised" + name: str epochs: int mode: Literal["sequential", "interleaved"] @@ -1362,8 +1373,8 @@ def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: @dataclass(frozen=True) class LoadedTrainConfig: - config: SequifierConfig - resolved: ResolvedSequifierConfig + config: SequifierConfig | PolicyTrainConfig + resolved: ResolvedSequifierConfig | ResolvedPolicyTrainConfig metadata: dict[str, DatasetMetadata] @@ -2237,6 +2248,28 @@ def load_train_config_with_source( if args: raise ValueError(f"Unsupported training CLI overrides: {sorted(args)}") + plan = raw.get("training_plan", {}) + phases = plan.get("phases", [plan]) if isinstance(plan, dict) else [] + algorithms = { + phase.get("algorithm", "supervised") + for phase in phases + if isinstance(phase, dict) + } + if algorithms - {"supervised"}: + if algorithms != {"return_conditioned"} or len(phases) != 1: + raise ValueError( + "This increment supports one return_conditioned phase; mixed plans, " + "IQL and PPO are not yet implemented." + ) + if sensitive or metadata_inline_values or skip_metadata: + raise ValueError( + "trajectory training does not accept supervised metadata overrides" + ) + authored = PolicyTrainConfig.model_validate(raw) + return LoadedTrainConfig( + config=authored, resolved=resolve_policy_train_config(authored), metadata={} + ) + config = try_catch_excess_keys(config_path, SequifierConfig, raw) part_overrides: dict[str, dict[str, str]] = {} if sensitive: @@ -2299,7 +2332,7 @@ def load_train_config_with_source( @beartype def load_train_config( config_path: str, args_config: dict[str, Any], skip_metadata: bool -) -> ResolvedSequifierConfig: +) -> ResolvedSequifierConfig | ResolvedPolicyTrainConfig: return load_train_config_with_source( config_path, args_config, skip_metadata ).resolved diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index f9cd3a58..c3584064 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -104,7 +104,7 @@ There are seven standalone commands within sequifier: `make`, `preprocess`, `tra | --- | --- | | `make` | Create a new sequifier project with config templates. | | `preprocess` | Convert input data into fixed-length subsequences. | -| `train` | Train a model on preprocessed data. | +| `train` | Train a supervised model or a trajectory-backed categorical policy. | | `infer` | Generate predictions, probabilities, or embeddings. | | `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | | `visualize-training` | Generate interactive HTML plots from structured training metrics. | @@ -116,6 +116,7 @@ The processing and visualization commands have dedicated documentation pages: - [preprocess documentation](./documentation/configs/preprocess.md) - [train documentation](./documentation/configs/train.md) +- [policy training](./documentation/policy-learning.md) - [infer documentation](./documentation/configs/infer.md) - [hyperparameter-search documentation](./documentation/configs/hyperparameter-search.md) - [visualize-training documentation](./documentation/commands/visualize-training.md) @@ -1705,6 +1706,18 @@ healthy active update. Momentum and weight decay cannot update parameters whose gradients are absent. +## Trajectory-backed policy training + +`sequifier train` also accepts one `return_conditioned` phase with named +trajectory sources and a categorical policy registry. This CPU workflow supports +return conditioning or BC, with no supervised dataset placeholders. See the +[policy training guide](../policy-learning.md) and +[complete example](../example_inputs/configs/train-return-conditioned.yaml). +The phase owns `epochs`; the run owns `seed`; `optimization` configures updates. +Existing phases default to `algorithm: supervised`. Mixed algorithm plans and +IQL/PPO remain planned. + + # Infer Command Guide For CPU numeric inference with externally calculated next-step inputs, see @@ -2290,52 +2303,63 @@ srun torchrun \ * **Data Access:** All nodes must have access to the same shared filesystem (e.g., NFS, GPFS) where the `project_root` and the sharded preprocessing output are stored. -# Policy Learning Command Guide +# Policy training through `sequifier train` -`sequifier policy` runs a local, CPU-only categorical control experiment: +Train an actor on saved trajectories with: ```console -sequifier policy --config-path documentation/example_inputs/configs/policy-control.yaml +sequifier train --config-path documentation/example_inputs/configs/train-return-conditioned.yaml ``` -It collects trajectories, computes return-to-go labels, splits by episode, -trains return-conditioned and behavior-cloning actors, saves/reloads both, and -executes them through the existing rollout coordinator. `report.json` records -the achieved returns. This command is separate from forecast model training and -inference; it does not change `target_offset` or convert forecast checkpoints -into policies. `sequifier docs policy` prints this guide. - -## Task and collected data - -The deterministic binary control task starts at `position=0` with three decisions -remaining. Action `0` holds position; action `1` increments it. Each action earns -its numeric value as reward. Observations contain current `position` and -`remaining_steps`. The environment terminates after its last decision and returns -the actual final successor. The horizon is configurable from 1 through 8. - -Collection enumerates every binary action sequence, repeating each 16 times by -default. For horizon 3 that gives 128 episodes and 384 executed decisions from -the same starting conditions, with returns from 0 through 3 and a behavior mean -of 1.5. Repetition gives conflicting action labels for identical observations -while retaining examples at each return level. The dataset manifest lists the -exact collected files and their episode IDs; stale files are not added to the -current experiment. - -The example configuration writes these files beneath `outputs/numeric-policy`: - -- `dataset/sequence-*.json`: recorded episodes in original numeric units. -- `dataset/manifest.json`: exact source files, episode IDs, and raw returns. -- `dataset/return_labels.json`: the return definition and raw per-decision labels. -- `return_conditioned.pt` and `behavior_cloning.pt`: local actor artifacts. -- `evaluation/*.json`: trajectories executed using the reloaded actors. -- `report.json`: episode partitions, per-epoch action likelihood, checkpoint - selection, greedy/sampled actions, achieved returns, and requested-return errors. - -The report distinguishes raw episode reward sums from discounted achieved -returns. It compares both actors at the same requested returns, initial states, -and sampling seeds. The BC actor does not receive the requested return. This is -a controlled learning comparison; performance depends on training configuration -and is recorded rather than assumed. +The configuration declares named `trajectory_sources`, named `policies`, and one +`training_plan.phases` entry with `algorithm: return_conditioned`. Ordinary +training phases default to `algorithm: supervised`. Policy phases have their own +source and model contract and require no supervised dataset, criterion, or +preprocessing metadata. This increment supports CPU execution and one categorical +actor in one phase. Mixed algorithm plans, phase transfer, IQL, PPO, distributed +policy learning, custom training integrations, and policy export remain planned. +Unsupported run fields are rejected before learning. + +A trajectory source specifies either `paths: [episode-a.json, episode-b.json]` or +`manifest: dataset/manifest.json`. A version-1 manifest has an `episodes` list of +objects with a `path` for each file. Only listed files are loaded. Run paths resolve +relative to `project_root`; manifest entries resolve relative to the manifest's +directory. These JSON episodes are the interim numeric trajectory format, preceding +the plan's sharded multimodal storage. + +The phase references `trajectory_sources: [demonstrations]` and `policy: actor`. +Set `epochs` on the phase and `seed` on the run. Optimizer, scheduler, batch size, +gradient clipping and automatic `validation_fraction` belong in `optimization`. +The actor's `returns` declares gamma, scale, and optional finite horizon. A concise +`training_plan: {algorithm: return_conditioned, epochs: 80, trajectory_sources: +[demonstrations], policy: actor}` is also accepted. + +Without `validation_sources`, the runner splits whole episodes deterministically, +keeping identical recorded trajectories together. Repeated copies therefore do +not cross the partition boundary. At least two distinct trajectory groups are +required. For authored partitions, add `validation_sources: [held_out]` to the +phase and declare that separate source in the registry. Duplicate file paths, +episode IDs, or identical trajectory content across partitions are rejected. +Normalization is fit only on training episodes. Truncations and collection cutoffs +with unknown returns are excluded under the existing return-label contract; +excluded IDs are recorded in the report. + +The runtime uses the existing optimization transactions and selects the best +actor by held-out action NLL. `actor_path` defaults to +`models//actor.pt`; `report_path` defaults to +`outputs/training//report.json`. The report records resolved config, +selected files, partition IDs, decision counts, excluded episodes, per-epoch NLL, +optimizer steps and checkpoint selection. Actor files contain model weights, +normalization buffers, vocabulary, alignment and return settings. They are +actor-only inference artifacts; optimizer and RNG state for exact resume are not +included. Use `load_policy` and `collect_policy_rollout` to execute them. + +For behavior cloning, set the actor's `return_conditioned: false` in the same +workflow. Offline training requires only saved trajectories; it does not start an +environment. Python callers use `load_train_config` and `run_training` with the +same configuration as the CLI. + +`sequifier docs policy` prints this guide. ## Policy alignment and causal windows @@ -2344,14 +2368,6 @@ conditioning, `G_t`. The current action `a_t` is a separate target. The first previous action is a dedicated start-token ID equal to the number of executable actions. It cannot be selected by the action head. -For the sequence `[1, 0, 1]`, gamma 1, and the default task: - -| Decision | Current observation | Previous action ID | Desired return | Action target | -| --- | --- | --- | --- | --- | -| 0 | `(position=0, remaining_steps=3)` | 2 (start) | 2 | 1 | -| 1 | `(position=1, remaining_steps=2)` | 1 | 1 | 0 | -| 2 | `(position=1, remaining_steps=1)` | 0 | 1 | 1 | - Offline and online execution use `build_policy_inputs` to construct the same left-padded causal windows. Observations, previous actions, and returns are trimmed together. Padding has a false attention mask. Only the last position @@ -2369,8 +2385,8 @@ inputs; future observations are excluded from its window. Return-to-go is the intentional reward-derived training condition, with requested budgets replacing it online. Changing rewards can therefore change the return condition. -Both the command configuration and `CategoricalPolicyConfig` accept a `backbone` -section using the same architecture fields as training's +Each entry in the `policies` registry and `CategoricalPolicyConfig` accepts a +`backbone` section using the same architecture fields as training's `model.backbone.architecture`: ```yaml @@ -2403,10 +2419,8 @@ does not yet provide forecast-checkpoint transfer. The numeric policy input projection, action head and CPU-only execution remain policy-specific. Observation/static normalization is fit on the training episode partition only. -Its fixed means/scales are model buffers, saved with the actor. The RC and BC -actors use the same architecture, initialization seed, episode split, batch -order, and optimization configuration. BC disables the return channel by -feeding zero; it retains the same parameter count. +Its fixed means/scales are model buffers, saved with the actor. BC disables the return channel by feeding zero and retains the same parameter +count as the corresponding return-conditioned actor. ## Return definitions and episode boundaries @@ -2456,25 +2470,17 @@ owned by the shared coordinator on success and failure. ## Reusable Python interfaces -The public entry points live in `sequifier.rl_api`. To train on compatible saved -policy episodes: +The public policy entry points live in `sequifier.rl_api`. Python callers can +use the same configuration loader and training entry point as the CLI: ```python -from sequifier.rl_api import ( - CategoricalPolicyConfig, PolicyTrainingConfig, TrajectoryDataset, - load_policy, save_policy, split_by_episode, train_policy, -) +from sequifier.config.train_config import load_train_config +from sequifier.train import run_training +from sequifier.rl_api import load_policy -spec = CategoricalPolicyConfig( - observation_columns=["position", "remaining_steps"], - action_column="action", - action_values=[0.0, 1.0], -) -dataset = TrajectoryDataset(episode_paths, spec) -training, validation = split_by_episode(dataset, validation_fraction=0.2, seed=1010) -result = train_policy(training, validation, PolicyTrainingConfig()) -save_policy(result.policy, "actor.pt") -actor = load_policy("actor.pt") +config = load_train_config("configs/train-policy.yaml", {}, False) +run_training(config) +actor = load_policy(config.actor_path) ``` First-increment forecast trajectories can also supply demonstrations when their @@ -2497,18 +2503,21 @@ uses the dataset's independent action-valid mask. Training calls the existing `OptimizationRuntime` for backward, clipping, optimizer, scaler, and scheduler steps. The validation partition selects the saved best actor by action NLL. +For execution, supply an application-owned environment adapter, a validated +`SeedHistory`, and a `RolloutConfig` with `mode: policy`. Bind each changing input +to the appropriate environment field or executed action. The adapter follows the +[rollout protocol](rollout.md#adapter-protocol). + ```python from sequifier.rl_api import collect_policy_rollout -from sequifier.rollout.control import BinaryControlAdapter -from sequifier.rl.experiment import control_rollout_config, control_seed from sequifier.rollout_api import save_trajectory -rollout = control_rollout_config("execution.json", horizon=3) +# actor, adapter, seed, rollout_config and requested_return come from the application. trajectory = collect_policy_rollout( - actor, BinaryControlAdapter(horizon=3), control_seed(horizon=3), - rollout, max_steps=3, desired_return=3.0, deterministic=True, + actor, adapter, seed, rollout_config, + max_steps=1000, desired_return=requested_return, deterministic=True, ) -save_trajectory(trajectory, rollout.trajectory_path) +save_trajectory(trajectory, rollout_config.trajectory_path) ``` Set `return_conditioned=False` in the policy specification to train BC, and omit diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 7339d6ba..4c298e43 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -91,6 +91,32 @@ generation_steps: 5 """ +policy_train_config_string = """project_root: . +model_name: policy +device: cpu +seed: 1010 +trajectory_sources: + demonstrations: + manifest: PLEASE_FILL +policies: + actor: + observation_columns: [EXAMPLE_OBSERVATION] + action_column: action + action_values: [0.0, 1.0] + context_length: 4 + return_conditioned: true + returns: {gamma: 1.0, scale: 1.0} +training_plan: + algorithm: return_conditioned + epochs: 80 + trajectory_sources: [demonstrations] + policy: actor +optimization: + batch_size: 64 + learning_rate: 0.005 + validation_fraction: 0.2 +""" + gitignore_string = """models/ logs/ checkpoints/ @@ -122,3 +148,6 @@ def make(args): with open(f"{project_name}/configs/infer.yaml", "w") as f: f.write(infer_config_string) + + with open(f"{project_name}/configs/train-policy.yaml", "w") as f: + f.write(policy_train_config_string) diff --git a/src/sequifier/rl/experiment.py b/src/sequifier/rl/experiment.py deleted file mode 100644 index f26f2ef4..00000000 --- a/src/sequifier/rl/experiment.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Collect, train RC/BC actors, reload, and report achieved control-task returns.""" - -import json -from itertools import product -from pathlib import Path - -import yaml -from loguru import logger - -from sequifier.config.policy_config import ( - CategoricalPolicyConfig, - ControlExperimentConfig, -) -from sequifier.config.rollout_config import AdapterConfig, InputBinding, RolloutConfig -from sequifier.rl.checkpoint import load_policy, save_policy -from sequifier.rl.runtime import collect_policy_rollout -from sequifier.rl.training import train_policy -from sequifier.rollout.control import BinaryControlAdapter -from sequifier.rollout.coordinator import collect_rollout -from sequifier.trajectory.dataset import TrajectoryDataset, split_by_episode -from sequifier.trajectory.returns import return_to_go -from sequifier.trajectory.schema import SeedHistory -from sequifier.trajectory.store import load_trajectory, save_trajectory - - -def control_seed(horizon: int = 3, context_length: int = 4) -> SeedHistory: - if horizon < 1 or context_length < 1: - raise ValueError("control horizon and context_length must be positive") - return SeedHistory( - observations={ - "position": [0.0] * context_length, - "remaining_steps": [0.0] * (context_length - 1) + [float(horizon)], - }, - attention_valid_mask=[False] * (context_length - 1) + [True], - static_inputs={}, - ) - - -def control_rollout_config( - path: str | Path, horizon: int = 3, *, finite_horizon: int | None = None -) -> RolloutConfig: - return RolloutConfig( - mode="policy", - adapter=AdapterConfig( - factory="sequifier.rollout.control:BinaryControlAdapter", - config={"horizon": horizon}, - ), - bindings=[ - InputBinding(input=column, source="environment", key=column) - for column in ("position", "remaining_steps") - ], - seed_index=0, - trajectory_path=str(path), - max_episode_steps=finite_horizon, - ) - - -def collect_control_dataset( - directory: str | Path, - *, - horizon: int = 3, - repetitions: int = 16, - context_length: int = 4, - random_seed: int = 1010, -) -> list[Path]: - """Enumerate every binary action sequence, repeatedly from identical conditions.""" - if ( - isinstance(horizon, bool) - or not isinstance(horizon, int) - or not 1 <= horizon <= 8 - ): - raise ValueError("dataset control horizon must be an integer in [1, 8]") - if ( - isinstance(repetitions, bool) - or not isinstance(repetitions, int) - or repetitions < 1 - ): - raise ValueError("dataset repetitions must be a positive integer") - destination = Path(directory) - seed = control_seed(horizon, context_length) - paths = [] - manifest = [] - for sequence_index, actions in enumerate(product((0.0, 1.0), repeat=horizon)): - for repetition in range(repetitions): - path = ( - destination - / f"sequence-{sequence_index:03d}-episode-{repetition:03d}.json" - ) - rollout = control_rollout_config(path, horizon) - action_sequence = iter(actions) - - def predict(_history): - return {"action": next(action_sequence)} - - trajectory = collect_rollout( - predict, - BinaryControlAdapter(horizon), - seed, - rollout, - ["action"], - horizon, - random_seed=random_seed, - metadata={ - "policy_alignment": "current_observation_previous_action", - "behavior": "enumerated_action_sequence", - "action_values": [0.0, 1.0], - "task": {"name": "binary_control", "horizon": horizon}, - }, - ) - save_trajectory(trajectory, path) - paths.append(path) - manifest.append( - { - "path": path.name, - "episode_id": trajectory.episode_id, - "total_return": trajectory.total_return, - } - ) - # Consumers use these exact paths rather than accidentally including stale files. - (destination / "manifest.json").write_text( - json.dumps({"version": 1, "episodes": manifest}, indent=2) + "\n", - encoding="utf-8", - ) - return paths - - -def run_control_experiment(config: ControlExperimentConfig) -> dict: - root = Path(config.project_root).resolve() - output = Path(config.output_dir) - if not output.is_absolute(): - output = root / output - output.mkdir(parents=True, exist_ok=True) - paths = collect_control_dataset( - output / "dataset", - horizon=config.horizon, - repetitions=config.repetitions, - context_length=config.context_length, - random_seed=config.training.seed, - ) - spec = CategoricalPolicyConfig( - observation_columns=["position", "remaining_steps"], - context_length=config.context_length, - backbone=config.backbone, - returns=config.returns, - ) - corpus = TrajectoryDataset(paths, spec) - training, validation = split_by_episode( - corpus, config.training.validation_fraction, config.training.seed - ) - (output / "dataset" / "return_labels.json").write_text( - json.dumps( - { - "version": 1, - "definition": spec.returns.model_dump(mode="json"), - "episodes": [ - {"episode_id": episode.episode_id, "return_to_go": labels} - for episode, labels in zip(corpus.episodes, corpus.returns) - ], - }, - indent=2, - allow_nan=False, - ) - + "\n", - encoding="utf-8", - ) - targets = config.requested_returns - if targets is None: - targets = [0.0, return_to_go([1.0] * config.horizon, config.returns.gamma)[0]] - report = { - "format": "sequifier.numeric-policy-experiment", - "version": 1, - "config": config.model_dump(mode="json"), - "dataset": { - "episodes": len(corpus.episodes), - "decisions": len(corpus), - "excluded_episode_ids": corpus.excluded_episode_ids, - "training_episode_ids": [ - episode.episode_id for episode in training.episodes - ], - "validation_episode_ids": [ - episode.episode_id for episode in validation.episodes - ], - "behavior_mean_return": sum( - episode.total_return for episode in corpus.episodes - ) - / len(corpus.episodes), - }, - "training": {}, - "evaluation": [], - } - for algorithm, conditioned in ( - ("return_conditioned", True), - ("behavior_cloning", False), - ): - actor_spec = spec.model_copy(update={"return_conditioned": conditioned}) - train_dataset = TrajectoryDataset(training.episodes, actor_spec) - validation_dataset = TrajectoryDataset(validation.episodes, actor_spec) - logger.info(f"Training {algorithm} on {len(train_dataset)} decisions") - result = train_policy(train_dataset, validation_dataset, config.training) - policy_path = output / f"{algorithm}.pt" - save_policy( - result.policy, - policy_path, - metadata={ - "algorithm": algorithm, - "training_config": config.training.model_dump(mode="json"), - "training_episode_ids": list(result.training_episode_ids), - "validation_episode_ids": list(result.validation_episode_ids), - "best_epoch": result.best_epoch, - }, - ) - # All reported execution uses weights/transforms reloaded from disk. - policy = load_policy(policy_path) - report["training"][algorithm] = { - "policy_path": str(policy_path), - "best_epoch": result.best_epoch, - "optimizer_steps": result.optimizer_steps, - "epochs": list(result.metrics), - } - for target_index, requested_return in enumerate(targets): - for selection, count in ( - ("deterministic", 1), - ("sample", config.sampling_episodes), - ): - episodes = [] - for episode_index in range(count): - path = ( - output - / "evaluation" - / f"{algorithm}-return-{target_index}-{selection}-{episode_index}.json" - ) - rollout = control_rollout_config( - path, config.horizon, finite_horizon=spec.returns.finite_horizon - ) - trajectory = collect_policy_rollout( - policy, - BinaryControlAdapter(config.horizon), - control_seed(config.horizon, config.context_length), - rollout, - config.horizon, - desired_return=requested_return if conditioned else None, - deterministic=selection == "deterministic", - random_seed=config.training.seed + episode_index, - ) - save_trajectory(trajectory, path) - recorded = load_trajectory(path) - achieved = return_to_go( - [step.reward for step in recorded.transitions], - spec.returns.gamma, - )[0] - episodes.append( - { - "episode_id": recorded.episode_id, - "trajectory_path": str(path), - "actions": [ - step.predictions[spec.action_column] - for step in recorded.transitions - ], - "achieved_return": achieved, - "total_return": recorded.total_return, - "remaining_returns": recorded.metadata["policy_execution"][ - "remaining_returns" - ], - } - ) - report["evaluation"].append( - { - "algorithm": algorithm, - "requested_return": requested_return, - "selection": selection, - "episodes": episodes, - "mean_achieved_return": sum( - episode["achieved_return"] for episode in episodes - ) - / count, - "mean_absolute_return_error": sum( - abs(episode["achieved_return"] - requested_return) - for episode in episodes - ) - / count, - } - ) - report_path = output / "report.json" - report_path.write_text( - json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8" - ) - logger.info(f"Saved policy learning comparison to '{report_path}'") - return report - - -def run_policy_command(config_path: str) -> None: - with open(config_path, encoding="utf-8") as stream: - config = ControlExperimentConfig.model_validate(yaml.safe_load(stream)) - run_control_experiment(config) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py new file mode 100644 index 00000000..661e087e --- /dev/null +++ b/src/sequifier/rl/learner.py @@ -0,0 +1,184 @@ +"""Algorithm boundary used by trajectory-backed train runs and Python callers.""" + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol + +from sequifier.config.policy_config import PolicyTrainingConfig +from sequifier.config.policy_train_config import ( + ResolvedPolicyTrainConfig, + TrajectorySourceConfig, +) +from sequifier.rl.checkpoint import save_policy +from sequifier.rl.training import PolicyTrainingResult, train_policy +from sequifier.trajectory.dataset import ( + TrajectoryDataset, + episode_content_key, + split_by_episode, +) +from sequifier.trajectory.store import load_trajectory + + +class AlgorithmRuntime(Protocol): + def prepare(self) -> None: ... + def update(self) -> PolicyTrainingResult: ... + def evaluate(self) -> dict: ... + def checkpoint(self) -> None: ... + + +def trajectory_paths(source: TrajectorySourceConfig) -> list[Path]: + if source.manifest is None: + paths = [Path(path) for path in source.paths] + else: + manifest_path = Path(source.manifest) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict) or manifest.get("version") != 1: + raise ValueError("trajectory manifest must have version 1") + episodes = manifest.get("episodes") + if not isinstance(episodes, list) or not episodes: + raise ValueError("trajectory manifest requires nonempty episodes") + paths = [] + for entry in episodes: + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("path"), str) + or not entry["path"] + ): + raise ValueError("each manifest episode requires a nonempty path") + path = manifest_path.parent / entry["path"] + if "episode_id" in entry: + if load_trajectory(path).episode_id != entry["episode_id"]: + raise ValueError(f"manifest episode ID disagrees with {path}") + paths.append(path) + paths = [path.resolve() for path in paths] + if len(paths) != len(set(paths)): + raise ValueError("trajectory source contains duplicate paths") + return paths + + +@dataclass +class ReturnConditionedRuntime: + config: ResolvedPolicyTrainConfig + training: TrajectoryDataset = field(init=False) + validation: TrajectoryDataset = field(init=False) + excluded_episode_ids: list[str] = field(init=False) + sources: dict[str, list[str]] = field(init=False) + result: PolicyTrainingResult = field(init=False) + + def prepare(self) -> None: + phase = self.config.training_plan.phases[0] + spec = self.config.policies[phase.policy] + all_paths: dict[str, list[Path]] = {} + for name in phase.trajectory_sources + (phase.validation_sources or []): + all_paths[name] = trajectory_paths(self.config.trajectory_sources[name]) + flattened = [path for paths in all_paths.values() for path in paths] + protected_paths = set(flattened) + for name in all_paths: + manifest = self.config.trajectory_sources[name].manifest + if manifest is not None: + protected_paths.add(Path(manifest).resolve()) + if { + Path(self.config.actor_path), + Path(self.config.report_path), + } & protected_paths: + raise ValueError("training outputs must not overwrite trajectory inputs") + if len(flattened) != len(set(flattened)): + raise ValueError( + "trajectory files cannot appear in multiple selected sources" + ) + + episodes = {path: load_trajectory(path) for path in flattened} + # Include episodes excluded from exact return labeling in identity checks. + ids = [episode.episode_id for episode in episodes.values()] + if len(ids) != len(set(ids)): + raise ValueError("duplicate episode IDs across trajectory sources") + + def dataset(names: list[str]) -> TrajectoryDataset: + semantics = { + self.config.trajectory_sources[name].action_semantics for name in names + } + if len(semantics) != 1: + raise ValueError("selected sources must share action_semantics") + return TrajectoryDataset( + [episodes[path] for name in names for path in all_paths[name]], + spec, + action_semantics=next(iter(semantics)), + ) + + corpus = dataset(phase.trajectory_sources) + self.excluded_episode_ids = list(corpus.excluded_episode_ids) + if phase.validation_sources is None: + self.training, self.validation = split_by_episode( + corpus, + self.config.optimization.validation_fraction, + self.config.seed, + group_identical=True, + ) + else: + self.training = corpus + self.validation = dataset(phase.validation_sources) + self.excluded_episode_ids += self.validation.excluded_episode_ids + if {episode_content_key(e) for e in self.training.episodes} & { + episode_content_key(e) for e in self.validation.episodes + }: + raise ValueError( + "identical trajectories cannot cross training/validation partitions" + ) + self.sources = { + name: [str(path) for path in paths] for name, paths in all_paths.items() + } + + def update(self) -> PolicyTrainingResult: + phase = self.config.training_plan.phases[0] + optimization = PolicyTrainingConfig( + **self.config.optimization.model_dump(), + epochs=phase.epochs, + seed=self.config.seed, + ) + self.result = train_policy(self.training, self.validation, optimization) + return self.result + + def evaluate(self) -> dict: + result = self.result + return { + "format": "sequifier.policy-training-report", + "version": 1, + "algorithm": "return_conditioned", + "config": self.config.model_dump(mode="json"), + "sources": self.sources, + "training_episode_ids": list(result.training_episode_ids), + "validation_episode_ids": list(result.validation_episode_ids), + "excluded_episode_ids": self.excluded_episode_ids, + "training_decisions": len(self.training), + "validation_decisions": len(self.validation), + "best_epoch": result.best_epoch, + "optimizer_steps": result.optimizer_steps, + "epochs": list(result.metrics), + "monitor": { + "metric": "action_nll", + "direction": "min", + "value": result.metrics[result.best_epoch]["validation_nll"], + }, + "artifact_kind": "actor_only", + "actor_path": self.config.actor_path, + } + + def checkpoint(self) -> None: + save_policy( + self.result.policy, self.config.actor_path, metadata=self.evaluate() + ) + + +def run_policy_training(config: ResolvedPolicyTrainConfig) -> PolicyTrainingResult: + runtime: AlgorithmRuntime = ReturnConditionedRuntime(config) + runtime.prepare() + result = runtime.update() + report = runtime.evaluate() + runtime.checkpoint() + destination = Path(config.report_path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + return result diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 684e4219..c1ddd37d 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -4,6 +4,7 @@ from dataclasses import dataclass import torch +from loguru import logger from torch.utils.data import DataLoader from sequifier.config.policy_config import PolicyTrainingConfig @@ -50,7 +51,9 @@ def train_policy( torch.manual_seed(config.seed) policy = CategoricalPolicy(training.config) policy.set_normalization(*training.normalization_rows()) - optimization = OptimizationRuntime.create(config, "cpu", policy.parameters()) + optimization = OptimizationRuntime.create( + config, "cpu", policy.parameters(), phase_epochs=config.epochs + ) integrations = IntegrationManager() generator = torch.Generator().manual_seed(config.seed) train_loader = DataLoader( @@ -112,6 +115,10 @@ def train_policy( batch.actions, batch.action_valid_mask, ) + if not torch.isfinite(loss): + raise ValueError( + "validation action likelihood became non-finite" + ) count = int(batch.action_valid_mask.sum()) validation_loss += float(loss) * count validation_count += count @@ -123,6 +130,11 @@ def train_policy( "validation_nll": validation_loss, } ) + logger.info( + f"Policy epoch {epoch + 1}/{config.epochs}: " + f"action_nll={total_loss / decisions:.6f}, " + f"validation_action_nll={validation_loss:.6f}" + ) if validation_loss < best_loss: best_loss = validation_loss best_epoch = epoch diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index b2f22d59..e2e2f484 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -2,17 +2,28 @@ from sequifier.config.policy_config import ( CategoricalPolicyConfig, - ControlExperimentConfig, + PolicyOptimizationConfig, PolicyTrainingConfig, ReturnDefinition, ) +from sequifier.config.policy_train_config import ( + PolicyTrainConfig, + ResolvedPolicyTrainConfig, + ReturnConditionedPhaseConfig, + TrajectorySourceConfig, + resolve_policy_train_config, +) from sequifier.rl.checkpoint import load_policy, save_policy from sequifier.rl.distributions import ( CategoricalActionDistribution, masked_action_likelihood, ) -from sequifier.rl.experiment import collect_control_dataset, run_control_experiment from sequifier.rl.history import PolicyInputs, action_index, build_policy_inputs +from sequifier.rl.learner import ( + AlgorithmRuntime, + ReturnConditionedRuntime, + run_policy_training, +) from sequifier.rl.policy import CategoricalPolicy from sequifier.rl.runtime import PolicyExecutor, collect_policy_rollout from sequifier.rl.training import PolicyTrainingResult, train_policy @@ -24,8 +35,16 @@ from sequifier.trajectory.returns import remaining_return, return_to_go __all__ = [ + "PolicyTrainConfig", + "PolicyOptimizationConfig", + "TrajectorySourceConfig", + "ReturnConditionedPhaseConfig", + "ResolvedPolicyTrainConfig", + "resolve_policy_train_config", + "AlgorithmRuntime", + "ReturnConditionedRuntime", + "run_policy_training", "CategoricalPolicyConfig", - "ControlExperimentConfig", "PolicyTrainingConfig", "ReturnDefinition", "CategoricalActionDistribution", @@ -37,13 +56,11 @@ "action_index", "build_policy_inputs", "collate_policy_examples", - "collect_control_dataset", "collect_policy_rollout", "load_policy", "masked_action_likelihood", "remaining_return", "return_to_go", - "run_control_experiment", "save_policy", "split_by_episode", "train_policy", diff --git a/src/sequifier/rollout/control.py b/src/sequifier/rollout/control.py deleted file mode 100644 index 60af40a7..00000000 --- a/src/sequifier/rollout/control.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Deterministic finite-horizon binary numeric control task.""" - -from sequifier.rollout.contracts import ResetRequest, StepRequest, StepResult -from sequifier.rollout.history import numeric_fields, numeric_scalar - - -class BinaryControlAdapter: - """Action 0 holds position, action 1 increments it; reward equals the action.""" - - protocol_version = 1 - - def __init__(self, horizon: int = 3): - if isinstance(horizon, bool) or not isinstance(horizon, int) or horizon < 1: - raise ValueError("control horizon must be a positive integer") - self.horizon = horizon - self.closed = False - self.episode_id = None - self.steps = 0 - self.position = 0.0 - - def reset(self, request: ResetRequest) -> None: - if self.closed: - raise ValueError("control adapter is closed") - observation = numeric_fields( - {key: values[-1] for key, values in request.seed_history.items()}, - {"position", "remaining_steps"}, - "control seed", - ) - if observation != {"position": 0.0, "remaining_steps": float(self.horizon)}: - raise ValueError( - "control episodes start at position=0 with the full horizon remaining" - ) - self.episode_id = request.episode_id - self.steps = 0 - self.position = 0.0 - - def step(self, request: StepRequest) -> StepResult: - if ( - self.closed - or self.episode_id != request.episode_id - or self.steps >= self.horizon - ): - raise ValueError("control step requires an active reset episode") - if request.step_id != self.steps or request.observation != { - "position": self.position, - "remaining_steps": float(self.horizon - self.steps), - }: - raise ValueError("control request observation/step is out of sync") - action = numeric_scalar(request.predictions.get("action"), "control action") - if action not in (0.0, 1.0) or set(request.predictions) != {"action"}: - raise ValueError("binary control actions must be 0 or 1") - self.position += action - self.steps += 1 - return StepResult( - { - "position": self.position, - "remaining_steps": float(self.horizon - self.steps), - }, - reward=action, - terminated=self.steps == self.horizon, - ) - - def close(self) -> None: - self.closed = True diff --git a/src/sequifier/rollout/dependent.py b/src/sequifier/rollout/dependent.py deleted file mode 100644 index 84e3225d..00000000 --- a/src/sequifier/rollout/dependent.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Small local dependent-feature adapter, also usable as an integration example.""" - -from sequifier.rollout.contracts import ResetRequest, StepRequest, StepResult - - -class LinearDependentAdapter: - """Supply a scaled prediction and use that prediction as the reward.""" - - protocol_version = 1 - - def __init__( - self, prediction="x", external_input="d", multiplier=2.0, terminate_after=3 - ): - if ( - not isinstance(terminate_after, int) - or isinstance(terminate_after, bool) - or terminate_after < 1 - ): - raise ValueError("terminate_after must be a positive integer") - self.prediction = prediction - self.external_input = external_input - self.multiplier = multiplier - self.terminate_after = terminate_after - self.closed = False - self.steps = 0 - self.episode_id = None - - def reset(self, request: ResetRequest) -> None: - if self.closed: - raise ValueError("adapter is closed") - self.steps = 0 - self.episode_id = request.episode_id - - def step(self, request: StepRequest) -> StepResult: - if ( - self.closed - or self.episode_id != request.episode_id - or self.steps >= self.terminate_after - ): - raise ValueError("adapter must be reset before stepping an active episode") - self.steps += 1 - prediction = request.predictions[self.prediction] - return StepResult( - {self.external_input: self.multiplier * prediction}, - reward=prediction, - terminated=self.steps == self.terminate_after, - ) - - def close(self) -> None: - self.closed = True diff --git a/src/sequifier/sequifier.py b/src/sequifier/sequifier.py index 28f70504..7f4f058f 100644 --- a/src/sequifier/sequifier.py +++ b/src/sequifier/sequifier.py @@ -66,11 +66,6 @@ def setup_parser() -> ArgumentParser: ) parser_train = subparsers.add_parser("train", help="Run the training step") parser_infer = subparsers.add_parser("infer", help="Run the inference step") - parser_policy = subparsers.add_parser( - "policy", help="Collect/train/reload numeric return-conditioned and BC policies" - ) - parser_policy.add_argument("--config-path", type=str, required=True) - parser_hyperparameter_search = subparsers.add_parser( "hyperparameter-search", help="Run hyperparamter search" ) @@ -169,12 +164,6 @@ def main() -> None: docs(args.topic) return - if args.command == "policy": - from sequifier.rl.experiment import run_policy_command - - run_policy_command(args.config_path) - return - if args.command != "hyperparameter-search": args_config = build_args_config(args) diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 033f0628..f0720c69 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -19,6 +19,7 @@ load_model_artifact, load_weights_from_run_checkpoint, ) +from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig # noqa: E402 from sequifier.config.train_config import ( # noqa: E402 ResolvedSequifierConfig as TrainModel, ) @@ -162,7 +163,7 @@ def _mp_train_worker_wrapper( @beartype def run_training( - config: TrainModel, + config: TrainModel | ResolvedPolicyTrainConfig, *, integration_specs: tuple[IntegrationSpec, ...] = (), integration_instances: tuple[Any, ...] = (), @@ -170,6 +171,15 @@ def run_training( ) -> None: """Launch canonical service-composed training locally or across workers.""" + if isinstance(config, ResolvedPolicyTrainConfig): + if integration_specs or integration_instances or semantic_optimizer_grouping: + raise ValueError( + "policy training does not yet support custom integrations or parameter grouping" + ) + from sequifier.rl.learner import run_policy_training + + run_policy_training(config) + return if not isinstance(config, TrainModel): raise TypeError("Training requires a canonical resolved config.") spec = config.global_training diff --git a/src/sequifier/trajectory/dataset.py b/src/sequifier/trajectory/dataset.py index 075634e4..b737c418 100644 --- a/src/sequifier/trajectory/dataset.py +++ b/src/sequifier/trajectory/dataset.py @@ -1,5 +1,7 @@ """Episode-split training windows with explicit current-action policy alignment.""" +import hashlib +import json import random from collections.abc import Sequence from dataclasses import dataclass @@ -198,21 +200,43 @@ def normalization_rows(self) -> tuple[torch.Tensor, torch.Tensor]: ) +def episode_content_key(episode: Trajectory) -> str: + """Group repeated demonstrations independently of generated episode IDs.""" + values = episode.model_dump(mode="json") + values.pop("episode_id") + values.pop("metadata") + for transition in values["transitions"]: + transition.pop("episode_id") + return hashlib.sha256(json.dumps(values, sort_keys=True).encode()).hexdigest() + + def split_by_episode( - dataset: TrajectoryDataset, validation_fraction: float = 0.2, seed: int = 1010 + dataset: TrajectoryDataset, + validation_fraction: float = 0.2, + seed: int = 1010, + *, + group_identical: bool = False, ) -> tuple[TrajectoryDataset, TrajectoryDataset]: if not 0 < validation_fraction < 1 or len(dataset.episodes) < 2: raise ValueError( "episode splitting requires at least two eligible episodes and validation_fraction in (0,1)" ) - episodes = list(dataset.episodes) - random.Random(seed).shuffle(episodes) - count = min(len(episodes) - 1, max(1, int(len(episodes) * validation_fraction))) + groups = {} + for episode in dataset.episodes: + key = episode_content_key(episode) if group_identical else episode.episode_id + groups.setdefault(key, []).append(episode) + partitions = list(groups.values()) + if len(partitions) < 2: + raise ValueError("splitting requires at least two distinct trajectory groups") + random.Random(seed).shuffle(partitions) + count = min(len(partitions) - 1, max(1, int(len(partitions) * validation_fraction))) + validation = [episode for group in partitions[:count] for episode in group] + training = [episode for group in partitions[count:] for episode in group] return ( TrajectoryDataset( - episodes[count:], dataset.config, action_semantics=dataset.action_semantics + training, dataset.config, action_semantics=dataset.action_semantics ), TrajectoryDataset( - episodes[:count], dataset.config, action_semantics=dataset.action_semantics + validation, dataset.config, action_semantics=dataset.action_semantics ), ) From 07e49947c8c95eccd185dba920c17b29e92603dd Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 12:23:07 +0100 Subject: [PATCH 05/71] integrate --- README.md | 2 +- documentation/configs/infer.md | 11 + documentation/configs/train.md | 11 +- .../example_inputs/configs/infer-policy.yaml | 15 + .../configs/train-return-conditioned.yaml | 5 +- ...nforcement-learning-implementation-plan.md | 8 + documentation/policy-learning.md | 442 +++++++++-------- release-notes/unreleased-policy-training.md | 40 +- src/sequifier/api.py | 2 + src/sequifier/artifacts/loading.py | 28 +- src/sequifier/artifacts/model_artifact.py | 51 +- src/sequifier/artifacts/model_config.py | 1 + src/sequifier/artifacts/model_export.py | 4 + src/sequifier/artifacts/run_checkpoint.py | 11 +- src/sequifier/config/infer_config.py | 17 +- src/sequifier/config/policy_config.py | 43 +- src/sequifier/config/policy_infer_config.py | 85 ++++ src/sequifier/config/policy_train_config.py | 28 +- src/sequifier/config/train_config.py | 5 +- src/sequifier/consolidated-docs.md | 466 ++++++++++-------- src/sequifier/infer.py | 8 + src/sequifier/make.py | 17 + src/sequifier/rl/checkpoint.py | 93 ++-- src/sequifier/rl/distributions.py | 10 +- src/sequifier/rl/history.py | 44 ++ src/sequifier/rl/inference.py | 246 +++++++++ src/sequifier/rl/learner.py | 37 +- src/sequifier/rl/policy.py | 277 ++++++++--- src/sequifier/rl/runtime.py | 12 +- src/sequifier/rl/training.py | 231 ++++++++- src/sequifier/rl_api.py | 2 + src/sequifier/train.py | 25 +- src/sequifier/trajectory/dataset.py | 261 +++++++--- 33 files changed, 1795 insertions(+), 743 deletions(-) create mode 100644 documentation/example_inputs/configs/infer-policy.yaml create mode 100644 src/sequifier/config/policy_infer_config.py create mode 100644 src/sequifier/rl/inference.py diff --git a/README.md b/README.md index 17f8066a..443d3cf6 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ There are seven standalone commands within sequifier: `make`, `preprocess`, `tra | `make` | Create a new sequifier project with config templates. | | `preprocess` | Convert input data into fixed-length subsequences. | | `train` | Train a supervised model or a trajectory-backed categorical policy. | -| `infer` | Generate predictions, probabilities, or embeddings. | +| `infer` | Generate predictions, probabilities, embeddings, or policy actions and rollouts. | | `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | | `visualize-training` | Generate interactive HTML plots from structured training metrics. | | `docs` | Print the consolidated documentation, or the guide for `train`, `preprocess`, or `infer`. | diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index 9f333c1d..a40f6ba3 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -263,3 +263,14 @@ incorrect ranks/capacities, forbidden gaps, illegal categories, and non-finite values. Legal masked values are sanitized before narrowing and inside the graph. The declared provider must be available; CUDA inference requires a CUDA-enabled ORT installation. Outputs stay indexed by outer item coordinates. + + +## Policy model interfaces + +`sequifier infer` recognizes a policy interface in a portable PyTorch model or +run checkpoint. Use raw CSV/Parquet observation rows and optional recorded action +histories; alignment, vocabulary, and normalization come from the artifact. +`model_type: generative` and `model_type: policy` are accepted. Environment +execution uses the existing `rollout` section with `mode: policy`. See the +[policy guide](../policy-learning.md#predict-actions-with-sequifier-infer) and +[inference example](../example_inputs/configs/infer-policy.yaml). diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 0752cb44..0c4e5f53 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -1,8 +1,7 @@ # Train Command Guide -For trajectory-backed categorical control learning and the local RC/BC -comparison, see [policy learning](../policy-learning.md) and the separate -`sequifier policy` command. +For trajectory-backed categorical policy training and inference through the same +commands, see [policy learning](../policy-learning.md). `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated @@ -772,10 +771,14 @@ gradients are absent. ## Trajectory-backed policy training `sequifier train` also accepts one `return_conditioned` phase with named -trajectory sources and a categorical policy registry. This CPU workflow supports +trajectory sources and a categorical policy registry. This workflow supports return conditioning or BC, with no supervised dataset placeholders. See the [policy training guide](../policy-learning.md) and [complete example](../example_inputs/configs/train-return-conditioned.yaml). The phase owns `epochs`; the run owns `seed`; `optimization` configures updates. Existing phases default to `algorithm: supervised`. Mixed algorithm plans and IQL/PPO remain planned. + +Policy models use the shared configurable network and portable artifacts. They +support CPU/CUDA/MPS, backbone transfer with `initialize_from`, bounded file-backed +episode loading, and epoch-boundary resume through `optimization.resume`. diff --git a/documentation/example_inputs/configs/infer-policy.yaml b/documentation/example_inputs/configs/infer-policy.yaml new file mode 100644 index 00000000..a785e027 --- /dev/null +++ b/documentation/example_inputs/configs/infer-policy.yaml @@ -0,0 +1,15 @@ +project_root: . +model_path: models/policy/policy.pt +# Raw observations/static inputs in original units; no forecast preprocessing required. +data_path: data/policy-observations.csv +model_type: generative +read_format: csv +write_format: csv +device: cpu +deterministic: true +# Omit this for behavior cloning. Alternatively provide a desired_return column. +desired_return: 10.0 +# For sequential input, include sequenceId and recorded action values, or: +# previous_action_column: previous_action +# output_probabilities: true +# output_path: outputs/policy-actions.csv diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index b6c5636e..14e55b34 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -37,11 +37,14 @@ training_plan: optimization: batch_size: 64 + num_workers: 0 + episode_cache_size: 8 + resume: {policy: never} learning_rate: 0.005 gradient_clip: 1.0 validation_fraction: 0.2 optimizer: {name: Adam} scheduler: {name: ConstantLR, factor: 1.0, total_iters: 1} -actor_path: models/policy/actor.pt +model_path: models/policy/policy.pt report_path: outputs/policy/training-report.json diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md index 8d0ca600..5be2c7fd 100644 --- a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md +++ b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md @@ -27,6 +27,14 @@ Mixed phases, phase transfer, sharded/multimodal storage, general rollout infrastructure, IQL/PPO, distributed RL, exact resume and exports remain planned. No tests were written or run for this increment, as requested. +Integration follow-up: policy alignment now wraps the canonical configurable +network, with standard portable model and resumable run artifacts. `sequifier +infer` supports raw policy decisions and shared-coordinator environment rollouts. +Matching backbone transfer, CPU/CUDA/MPS execution, bounded file-backed episode +caching, context-sized windows, and BC eligibility without return labels are +implemented. The broader stages below remain planned. No tests or training runs +were executed for this follow-up. + Deliver this as one coordinated update, built and verified in the three dependency-ordered stages below. Completion means all three stages work together; stage boundaries are engineering checkpoints, not separate proposed releases. This document specifies new behavior and proposed API names, not functionality already present. ## 1. Outcome and scope diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 3c39e197..b9145e7d 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -1,228 +1,264 @@ -# Policy training through `sequifier train` +# Policy learning through `sequifier train` and `sequifier infer` -Train an actor on saved trajectories with: +Policy learning uses Sequifier's configurable ingestion, transformer network, +decoders, portable model artifacts, optimizer transactions, and run checkpoints. +A policy adds a decision-alignment contract to a model interface: at time `t`, +the inputs contain the current observation, the **previous** action, and, +optionally, a desired return. The current action is the supervised target. + +## Train from trajectories ```console -sequifier train --config-path documentation/example_inputs/configs/train-return-conditioned.yaml +sequifier train --config-path configs/train-policy.yaml +sequifier infer --config-path configs/infer-policy.yaml ``` -The configuration declares named `trajectory_sources`, named `policies`, and one -`training_plan.phases` entry with `algorithm: return_conditioned`. Ordinary -training phases default to `algorithm: supervised`. Policy phases have their own -source and model contract and require no supervised dataset, criterion, or -preprocessing metadata. This increment supports CPU execution and one categorical -actor in one phase. Mixed algorithm plans, phase transfer, IQL, PPO, distributed -policy learning, custom training integrations, and policy export remain planned. -Unsupported run fields are rejected before learning. - -A trajectory source specifies either `paths: [episode-a.json, episode-b.json]` or -`manifest: dataset/manifest.json`. A version-1 manifest has an `episodes` list of -objects with a `path` for each file. Only listed files are loaded. Run paths resolve -relative to `project_root`; manifest entries resolve relative to the manifest's -directory. These JSON episodes are the interim numeric trajectory format, preceding -the plan's sharded multimodal storage. - -The phase references `trajectory_sources: [demonstrations]` and `policy: actor`. -Set `epochs` on the phase and `seed` on the run. Optimizer, scheduler, batch size, -gradient clipping and automatic `validation_fraction` belong in `optimization`. -The actor's `returns` declares gamma, scale, and optional finite horizon. A concise -`training_plan: {algorithm: return_conditioned, epochs: 80, trajectory_sources: -[demonstrations], policy: actor}` is also accepted. +A complete training configuration is in +[train-return-conditioned.yaml](example_inputs/configs/train-return-conditioned.yaml). +`sequifier make` also creates `configs/train-policy.yaml`. Define named +`trajectory_sources`, named `policies`, and one `training_plan` phase with +`algorithm: return_conditioned`. Set `return_conditioned: false` on the policy +for behavior cloning (BC). Offline training does not start an environment. + +Each source specifies either `paths: [episode-a.json, episode-b.json]` or +`manifest: data/trajectories/manifest.json`. Version-1 manifests contain an +`episodes` list of objects with `path` and optional `episode_id`. Only listed +files are selected. Run paths resolve relative to `project_root`; manifest +entries resolve relative to the manifest directory. + +Set `epochs` on the phase, `seed` and `device` on the run, and update settings +in `optimization`. Devices follow PyTorch's CPU, CUDA, and MPS conventions. +`optimization.num_workers` controls data-loader workers and +`optimization.episode_cache_size` bounds the number of cached file-backed +episodes per worker (default 8). The learner retains episode metadata and loads +episodes on demand. It shuffles episodes and the decisions within each episode +to keep disk access local. Normalization combines episode statistics +incrementally. JSON storage still requires an individual episode to fit in memory. Without `validation_sources`, the runner splits whole episodes deterministically, -keeping identical recorded trajectories together. Repeated copies therefore do -not cross the partition boundary. At least two distinct trajectory groups are -required. For authored partitions, add `validation_sources: [held_out]` to the -phase and declare that separate source in the registry. Duplicate file paths, -episode IDs, or identical trajectory content across partitions are rejected. -Normalization is fit only on training episodes. Truncations and collection cutoffs -with unknown returns are excluded under the existing return-label contract; -excluded IDs are recorded in the report. - -The runtime uses the existing optimization transactions and selects the best -actor by held-out action NLL. `actor_path` defaults to -`models//actor.pt`; `report_path` defaults to -`outputs/training//report.json`. The report records resolved config, -selected files, partition IDs, decision counts, excluded episodes, per-epoch NLL, -optimizer steps and checkpoint selection. Actor files contain model weights, -normalization buffers, vocabulary, alignment and return settings. They are -actor-only inference artifacts; optimizer and RNG state for exact resume are not -included. Use `load_policy` and `collect_policy_rollout` to execute them. - -For behavior cloning, set the actor's `return_conditioned: false` in the same -workflow. Offline training requires only saved trajectories; it does not start an -environment. Python callers use `load_train_config` and `run_training` with the -same configuration as the CLI. - -`sequifier docs policy` prints this guide. - -## Policy alignment and causal windows - -At decision position `t`, the policy receives `o_t`, `a_(t-1)`, and, for return -conditioning, `G_t`. The current action `a_t` is a separate target. The first -previous action is a dedicated start-token ID equal to the number of executable -actions. It cannot be selected by the action head. - -Offline and online execution use `build_policy_inputs` to construct the same -left-padded causal windows. Observations, previous actions, and returns are -trimmed together. Padding has a false attention mask. Only the last position -of each training example has a true action-likelihood mask, so every recorded -decision contributes once even when context windows overlap. - -The actor uses numeric projection, previous-action embedding, the same -`TransformerBackbone` and encoder layers as `sequifier train`, and one categorical -head. Attention is causal and excludes padded keys; padded logits are zero. -The model recomputes each finite context window rather than retaining recurrent -state across windows. -Forward calls take `PolicyInputs`, which contain no action labels or observed -current rewards. A change to a current-action label cannot affect that position's -inputs; future observations are excluded from its window. Return-to-go is the -intentional reward-derived training condition, with requested budgets replacing -it online. Changing rewards can therefore change the return condition. - -Each entry in the `policies` registry and `CategoricalPolicyConfig` accepts a -`backbone` section using the same architecture fields as training's -`model.backbone.architecture`: +keeping identical demonstrations together. At least two distinct trajectory +groups are required. To supply held-out data, declare a separate source and add +`validation_sources: [held_out]` to the phase. Duplicate paths and episode IDs, +and identical content across training and validation, are rejected. +Normalization fits only the training partition. + +The best model is selected by held-out action negative log likelihood (NLL). +`model_path` defaults to `models//.pt` and contains an +ordinary `sequifier_model` artifact. `actor_path` remains an alias for `model_path` for existing configurations. `report_path` +defaults to `outputs/training//report.json`; it records sources, +partition IDs, exclusions, decision counts, per-epoch NLL, and optimizer steps. + +## Configure the shared model + +The policy's `backbone` accepts the architecture fields from ordinary training's +`model.backbone.architecture`. Optional `ingestion` and `decoder` use the same +component schemas and builders as forecasting. For example: + +```yaml +policies: + actor: + observation_columns: [position, velocity] + action_column: action + action_values: [-1.0, 1.0] + context_length: 16 + return_conditioned: true + backbone: + dim_model: 64 + num_layers: 3 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} + positional_encoding_scope: global + decoder: {type: mlp, hidden_dims: [64], prediction_length: 1, support: 1} +``` + +Without an authored ingestion, the standard embedding ingestion projects real +features and embeds categorical features before an adapter maps to the backbone +input width. Real features receive one embedding channel and categorical features +share the remaining width. The default decoder is linear. A policy decoder must +have `support: 1` and `prediction_length: 1`. + +The internal input names `__policy_previous_action` and `__policy_return` are +reserved. The latter exists only for return-conditioned policies. Static columns +are broadcast over the decision window. Real observation and static statistics +are persisted in the model interface's preprocessing metadata. Encoded categorical +observations can be declared with `observation_n_classes: {state_id: 8}`; their +recorded values must be integral IDs in `[0, 8)` and are not normalized. +When authoring a mixed real/categorical embedding ingestion, provide +`feature_embedding_dims` under the normal ingestion rules, including the internal +previous-action column and, when enabled, return column. + +Actions use the ordered IDs of `action_values`; an extra input-only ID equal to +the action count represents episode start. It is excluded from decoder outputs. +An observation may have the same external name as the action; in that case the +model uses `__policy_action` as its separate target name. Default decoders infer +the target automatically. + +The default backbone has width 32, two layers, one attention head, and +feed-forward width 128. Context length must fit `max_context_length`. +Positional scope defaults to `global`; the standard ingestion also supports +feature-scoped learned positions when configured. +The standard attention, normalization, position encodings, dropout, and layer +sharing options remain available. + +To transfer a forecast or policy backbone, set a top-level +`initialize_from: models/forecast.pt`. Portable models and run checkpoints are +accepted. The source must have the same backbone architecture. Ingestion and +decoder parameters are initialized for the policy's own interface. Resumption +restores the entire existing policy instead of applying initialization again. + +## Resume training + +Every completed epoch saves a standard `sequifier_run_checkpoint` at +`checkpoints/runs//-latest.pt`. It contains current model +weights, preprocessing and policy semantics, optimizer/scheduler/scaler state, +random state, data-order generator state, best weights, and training progress. +The portable model at `model_path` contains the best validation weights; the run +checkpoint contains the latest training weights. ```yaml -context_length: 4 -backbone: - dim_model: 32 - num_layers: 2 - max_context_length: 4 - attention: {type: mha, n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: learned} - positional_encoding_scope: global - dropout: 0.0 - shared_layer_groups: [] +optimization: + resume: + policy: if_exists ``` -This includes MHA/MQA/GQA, feed-forward activation, normalization, positional -encoding (learned, RoPE, sinusoidal, range or range-concat), dropout and layer -sharing. `context_length` must fit `max_context_length`. Learned positions require -`global` scope because policy inputs use a combined numeric projection rather -than per-feature position embeddings. Scope defaults to `global` for policies. -Without `backbone`, defaults are two layers, width 32, one attention head and -feed-forward width 128. Configure width with `backbone.dim_model`; the experimental -`hidden_size` option has been removed. - -Transformer actor artifacts use version 2. Version 1 GRU artifacts cannot be -loaded into this architecture and must be retrained. Sharing backbone primitives -does not yet provide forecast-checkpoint transfer. The numeric policy input -projection, action head and CPU-only execution remain policy-specific. - -Observation/static normalization is fit on the training episode partition only. -Its fixed means/scales are model buffers, saved with the actor. BC disables the return channel by feeding zero and retains the same parameter -count as the corresponding return-conditioned actor. - -## Return definitions and episode boundaries - -Raw labels use `G_t = r_t + gamma * G_(t+1)` and terminal `G_T = 0`. -For rewards `[1, 2, 3]`, gamma 1 produces `[6, 5, 3]`; gamma 0.5 produces -`[2.75, 3.5, 3]`. `returns.gamma` is explicit in `(0, 1]`, defaulting to 1. -`returns.scale` divides returns only at the numeric encoder boundary and defaults -to 1. Artifacts and execution traces preserve raw reward units. - -By default, only trajectories with true termination have exact return labels. -Collection cutoffs and truncation without termination are excluded and their -episode IDs are exposed by the dataset. An explicit `returns.finite_horizon: H` -allows labeling an observed prefix of exactly H decisions, including a cutoff -at H. A fragment shorter than H is still excluded unless it truly terminates. -No unknown tail is silently assigned zero. For online use of this objective, -`rollout.max_episode_steps` must equal H. - -Execution starts from the caller's requested return and updates the raw budget -after each executed reward: +`never` is the default. `required` also requires an explicit `checkpoint_path`, +using the shared resume configuration. Resume occurs at epoch boundaries; +an interrupted epoch is repeated. Trajectory content, partition order, policy, +and learning settings must match. Device, output paths, worker/cache settings, +and total epoch budget may change. Restored scheduler state retains its existing +schedule when extending the epoch budget. Numerical reproducibility across +devices is governed by PyTorch. + +## Predict actions with `sequifier infer` + +Inference reads policy alignment and preprocessing directly from a portable +model or run checkpoint; a training YAML or preprocessing metadata file is not +needed. It accepts `model_type: generative` (or the explicit alias `policy`): + +```yaml +project_root: . +model_path: models/policy/policy.pt +data_path: data/observations.csv +read_format: csv +write_format: csv +device: cpu +deterministic: true +desired_return: 10.0 +``` + +See [infer-policy.yaml](example_inputs/configs/infer-policy.yaml). CSV and Parquet +inputs contain observations and any static columns in original units. Without +a `sequenceId` column, each row is an independent first decision. Within each +sequence, row order defines decision order and static inputs must be constant. +Provide the recorded `action_column` to derive previous actions by shifting one +row, or set `previous_action_column` to a column containing previous **domain +action values**, with null for episode start. Use a new sequence ID for each +episode; null previous actions inside a sequence are rejected. When observation +and action names coincide, use an explicit `previous_action_column` for sequential +CSV/Parquet input. Current recorded actions never +enter the inputs for the same decision. Historical context is bounded by the +model's context length. + +Return-conditioned models accept a fixed `desired_return` or a per-row column +named by `desired_return_column` (default `desired_return`). Per-row returns are +supplied by the caller; offline inference does not update budgets from rewards. +BC requires no return input and rejects a scalar `desired_return`. + +`inference_batch_size` bounds batched model calls (default 64). Predictions go to `outputs/predictions//part-000.csv` by default. +`output_path` overrides that location. Sequence IDs and `itemPosition`, when +present, are retained in input order. `output_probabilities: true` adds one +`__probability_` column per action. Set +`deterministic: false` for seeded categorical sampling. Select `model_interface` +when an artifact contains multiple policy interfaces. + +## Execute an environment rollout + +Add the ordinary `rollout` configuration to policy inference: + +```yaml +autoregressive: true +generation_steps: 1000 +rollout: + mode: policy + adapter: {factory: my_environment:create_adapter, config: {}} + bindings: + - {input: position, source: environment, key: position} + - {input: velocity, source: environment, key: velocity} + trajectory_path: outputs/trajectories/episode.json + seed_index: 0 +``` + +`seed_index` selects one raw input row, forming a single-observation reset seed. +The shared rollout coordinator owns reset, step ordering, boundaries, and adapter +closure. Each predicted action enters policy history only after it is executed. +The [adapter protocol](rollout.md#adapter-protocol) is the same as for forecasts. +The resulting trajectory can be used as a later training source. + +The raw return budget starts at `desired_return` (or the seed row's return +column) and updates after each executed reward: ```text G_next = (G_current - reward) / gamma ``` -The request is not clamped to an inferred feasible range. Each episode reset -restores the requested budget, clears previous-action/history state, and uses -the start token again. A collection cutoff preserves the unfinished episode -flag; it does not fabricate a reset. Each call collects one episode/fragment, -so collecting another episode uses a fresh adapter and an explicit reset. - -Policy trajectories use `format: sequifier.numeric-policy-trajectory`, version 1, -with the same seed/transition schema and storage as numeric forecast trajectories. -`mode: policy` makes this alignment explicit. Forecast artifacts retain their -existing format and require an explicit declaration before their executed -predictions can be used as demonstration actions. -Policy episodes start with one valid observation because historical actions -before reset are unknown. The actor builds subsequent history from executed -transitions. - -Execution metadata stores the full policy/return configuration, initial request, -remaining budgets (including the final successor budget), previous-action IDs, -and selected-action log probabilities. The trajectory validator checks action -vocabulary, previous-action alignment, budget counts, and reward/gamma arithmetic -in addition to ordinary continuity, reward, and boundary validation. Both flags -may be true; termination takes precedence for bootstrap. Adapter closure remains -owned by the shared coordinator on success and failure. - -## Reusable Python interfaces - -The public policy entry points live in `sequifier.rl_api`. Python callers can -use the same configuration loader and training entry point as the CLI: +Each reset restores the requested budget and clears policy history. The trajectory +records action log probabilities, previous-action IDs, and remaining raw budgets. +Termination and truncation remain separate; a collection cutoff preserves the +unfinished-episode flag. The coordinator always closes the adapter. + +## Labels, windows, and boundaries + +Raw return labels obey `G_t = r_t + gamma * G_(t+1)`, with terminal `G_T = 0`. +`returns.gamma` defaults to 1 and is in `(0, 1]`; `returns.scale` defaults to 1 +and divides the return only at ingestion. For rewards `[1, 2, 3]`, gamma 1 gives +`[6, 5, 3]`, and gamma 0.5 gives `[2.75, 3.5, 3]`. + +Return-conditioned training excludes truncations and collection cutoffs with +unknown tails. `returns.finite_horizon: H` can label a fully observed prefix of +H decisions. Shorter fragments need true termination. Online finite-horizon +execution requires `rollout.max_episode_steps: H`. No unknown tail is assigned +zero. BC uses **all observed decisions**, including truncated and cutoff episodes, +because its loss does not consume return labels. + +Offline and online paths share `build_policy_inputs`, producing left-padded, +causal histories. The dataset slices to the context window before building +inputs, retaining the actual preceding action at a cropped boundary. Each +training example labels only its last position, so every decision contributes +once. Current actions, current rewards, and future observations are excluded +from inputs; return-to-go is the intentional future-derived condition. + +Policy trajectories use `sequifier.numeric-policy-trajectory`, version 1. +Forecast trajectories can be demonstrations with explicit +`action_semantics: executed_predictions` and a finite `action_values` vocabulary. +They retain the latest reset observation and start previous-action history with +the start token, because earlier forecast seed actions are unknown. + +## Python interfaces and scope ```python from sequifier.config.train_config import load_train_config from sequifier.train import run_training -from sequifier.rl_api import load_policy +from sequifier.rl_api import load_policy, build_policy_inputs config = load_train_config("configs/train-policy.yaml", {}, False) run_training(config) -actor = load_policy(config.actor_path) -``` - -First-increment forecast trajectories can also supply demonstrations when their -executed numeric predictions have a declared finite categorical vocabulary: -pass `action_semantics="executed_predictions"` to `TrajectoryDataset` and set -the appropriate `action_column` and `action_values` in the policy specification. -This is an explicit data interpretation, not a conversion of forecast model -alignment or weights. It uses the latest seed observation as decision zero, -discards earlier seed observations whose actions were never recorded, and -starts previous-action history with the start token. Episode splitting preserves -this declaration. Observation and action names may coincide because their -values are read from separate observation/target records; current labels are -never copied into current inputs. - -Action values are original numeric domain values; the distribution uses their -ordered categorical IDs internally. `actor.distribution(inputs)` returns a -`CategoricalActionDistribution` supporting `mode()`, `sample(generator)`, -`log_prob(recorded_action_ids)`, and `entropy()`. `masked_action_likelihood` -uses the dataset's independent action-valid mask. Training calls the existing -`OptimizationRuntime` for backward, clipping, optimizer, scaler, and scheduler -steps. The validation partition selects the saved best actor by action NLL. - -For execution, supply an application-owned environment adapter, a validated -`SeedHistory`, and a `RolloutConfig` with `mode: policy`. Bind each changing input -to the appropriate environment field or executed action. The adapter follows the -[rollout protocol](rollout.md#adapter-protocol). - -```python -from sequifier.rl_api import collect_policy_rollout -from sequifier.rollout_api import save_trajectory - -# actor, adapter, seed, rollout_config and requested_return come from the application. -trajectory = collect_policy_rollout( - actor, adapter, seed, rollout_config, - max_steps=1000, desired_return=requested_return, deterministic=True, -) -save_trajectory(trajectory, rollout_config.trajectory_path) +policy = load_policy(config.model_path, device="cpu") +# inputs = build_policy_inputs(policy.config, observations, previous_action_ids, +# desired_returns, static_inputs).to(policy.device) +# distribution = policy.distribution(inputs) ``` -Set `return_conditioned=False` in the policy specification to train BC, and omit -`desired_return` when executing it. The actor artifacts contain validated CPU -weights, normalization buffers, action vocabulary, policy alignment, and return -semantics. Loading uses strict state-dict matching. These local inference -artifacts do not contain optimizer/RNG state for exact training resumption and -do not use forecast export or inference formats. - -This increment has no critics, Q networks, PPO buffers, distributed execution, -or policy export additions. Return conditioning and unconditioned behavior -cloning are independent actor routes. Future offline IQL can reuse the -trajectory windows and distribution interface without requiring RC pretraining. +`CategoricalPolicy` is an alignment adapter over the ordinary network, available +as `policy.network`. `policy.execution_config` is the canonical model config, +compatible with the shared artifact and export services. Distribution methods +include `mode`, `sample`, `log_prob`, and `entropy`. `collect_policy_rollout` +remains available for application-owned adapters. Generic analysis/loading tools +also accept policy model and run artifacts. + +The previous standalone numeric-policy artifacts (including versions 1 and 2) +use a different architecture and must be retrained. This workflow currently +supports one offline categorical actor phase. Mixed supervised/policy plans, +critics, IQL/PPO, distributed policy learning, custom run integrations, multimodal +trajectory storage, and ONNX policy execution through the CLI remain outside +this workflow. PyTorch policy execution supports CPU, CUDA, and MPS. diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index c38646ed..1e356682 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -1,21 +1,27 @@ -# Trajectory-backed policy training (unreleased) +# Integrated policy training and inference (unreleased) -`sequifier train` accepts a single `return_conditioned` phase with named trajectory -sources and categorical actors. Set `return_conditioned: false` on the actor to -train behavior cloning through the same workflow. Sources use explicit JSON file -lists or version-1 manifests, with automatic grouped episode splitting or authored -training/validation sources. Normalization fits the training partition only. +`sequifier train` accepts one trajectory-backed `return_conditioned` phase; +`return_conditioned: false` selects behavior cloning. Policies now use the +standard configurable ingestion, backbone, decoder, and network implementation. +Encoded categorical observations and matching forecast-backbone transfer are +supported. PyTorch execution supports CPU, CUDA, and MPS. -Runs save a best-action-NLL actor and a training report with source paths, -partition IDs, exclusions, metrics and optimizer-step counts. These CPU actor -artifacts support `load_policy` and `collect_policy_rollout`; exact training resume -and policy exports remain planned. +The selected best model is an ordinary `sequifier_model` artifact, usable with +`sequifier infer`, Python policy APIs, and generic model-loading tools. Inference +supports raw CSV/Parquet decisions and environment execution through the shared +rollout coordinator. Each training epoch writes a standard resumable run +checkpoint, including optimizer/scheduler/scaler, random/data-order state, +training progress, and best weights. The older standalone actor format requires +retraining. The default model path is now `models//.pt`. -The experimental `sequifier policy` command and synthetic control/dependent-feature -demos have been removed, including their configuration types and public exports. -Applications supply their own trajectories and environment adapters. -`sequifier make` now includes `configs/train-policy.yaml`. +File-backed training uses a bounded episode cache, incremental normalization, +and context-sized window construction. Behavior cloning retains observed +state/action pairs from truncated and cutoff episodes without return labels. +Grouped episode splitting and training-only normalization remain in place. -Mixed algorithm phases, transfer, IQL/PPO, distributed RL, multimodal trajectories -and scalable sharded storage remain future increments. No tests or training runs -were executed for this increment, as requested. +`sequifier make` includes policy train and infer templates. Mixed algorithm +phases, critics, IQL/PPO, distributed policy learning, multimodal trajectories, +and CLI ONNX policy execution remain future work. + +No tests or training runs were executed, as requested. Validation used static +lint, formatting, and type checks. diff --git a/src/sequifier/api.py b/src/sequifier/api.py index b772d277..95792331 100644 --- a/src/sequifier/api.py +++ b/src/sequifier/api.py @@ -4,6 +4,7 @@ ModelArtifact, ModelArtifactMetadata, ModelExecutionConfig, + load_execution_model, load_model_artifact, load_weights_from_run_checkpoint, ) @@ -49,6 +50,7 @@ "build_transformer_network", "canonical_parameter_name", "canonicalize_state_dict", + "load_execution_model", "load_model_artifact", "load_weights_from_run_checkpoint", ] diff --git a/src/sequifier/artifacts/loading.py b/src/sequifier/artifacts/loading.py index 032c9d37..a392811a 100644 --- a/src/sequifier/artifacts/loading.py +++ b/src/sequifier/artifacts/loading.py @@ -9,10 +9,7 @@ import torch from torch import nn -from sequifier.artifacts.model_artifact import ( - load_model_artifact, - load_weights_from_run_checkpoint, -) +from sequifier.artifacts.model_artifact import ModelArtifact, restore_model_artifact from sequifier.model.parameter_catalog import ParameterCatalog @@ -47,20 +44,15 @@ def load_model_for_analysis( artifact_path = Path(path).expanduser().resolve() payload = torch.load(artifact_path, map_location="cpu", weights_only=False) artifact_type = payload.get("artifact_type") if isinstance(payload, dict) else None - if artifact_type == "sequifier_model": - network, resolved_config, artifact = load_model_artifact( - artifact_path, - device=options.device, - interface_name=interface_name, - ) - elif artifact_type == "sequifier_run_checkpoint": - network, resolved_config, artifact = load_weights_from_run_checkpoint( - artifact_path, - device=options.device, - interface_name=interface_name, - ) - else: - raise ValueError(f"Unsupported Sequifier artifact: {artifact_path}.") + if artifact_type == "sequifier_run_checkpoint": + payload = payload["model"] + if not isinstance(payload, dict): + raise ValueError(f"Unsupported Sequifier artifact: {artifact_path}") + network, resolved_config, artifact = restore_model_artifact( + ModelArtifact.from_state_dict(payload), + device=options.device, + interface_name=interface_name, + ) network.train(options.training_mode) from sequifier.model.layers import SelfAttention diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index 8b4d111c..e68a0f63 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -144,6 +144,18 @@ def load_model_artifact( weights_only=False, ) artifact = ModelArtifact.from_state_dict(payload) + return restore_model_artifact( + artifact, device=device, interface_name=interface_name + ) + + +def restore_model_artifact( + artifact: ModelArtifact, + *, + device: str = "cpu", + interface_name: str | None = None, +) -> tuple[Any, Any, ModelArtifact]: + """Reconstruct execution identically for portable models and run checkpoints.""" config, _ = resolved_config_from_model_config( artifact.model_config.values, device=device, @@ -159,6 +171,32 @@ def load_model_artifact( return built.network, config, artifact +def read_execution_artifact(path: str | Path) -> ModelArtifact: + """Read the portable execution payload from either supported artifact kind.""" + payload = torch.load( + Path(path).expanduser().resolve(), map_location="cpu", weights_only=False + ) + if ( + isinstance(payload, dict) + and payload.get("artifact_type") == "sequifier_run_checkpoint" + ): + payload = payload["model"] + if not isinstance(payload, dict): + raise ValueError("Sequifier model artifact must be a mapping") + return ModelArtifact.from_state_dict(payload) + + +def load_execution_model( + path: str | Path, + *, + device: str = "cpu", + interface_name: str | None = None, +) -> tuple[Any, Any, ModelArtifact]: + return restore_model_artifact( + read_execution_artifact(path), device=device, interface_name=interface_name + ) + + def model_artifact_from_run_checkpoint(path: str | Path) -> ModelArtifact: payload = torch.load(Path(path), map_location="cpu", weights_only=False) if payload.get("artifact_type") != "sequifier_run_checkpoint": @@ -172,17 +210,8 @@ def load_weights_from_run_checkpoint( device: str = "cpu", interface_name: str | None = None, ) -> tuple[Any, Any, ModelArtifact]: - artifact = model_artifact_from_run_checkpoint(path) - config, _ = resolved_config_from_model_config( - artifact.model_config.values, + return restore_model_artifact( + model_artifact_from_run_checkpoint(path), device=device, interface_name=interface_name, ) - built = build_transformer_network( - config, device=torch.device(device), initialize=False - ) - built.network.load_state_dict(artifact.model_state_dict) - from sequifier.model.factory import validate_hash_contracts - - validate_hash_contracts(built.network, config) - return built.network, config, artifact diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index d729aee5..d6dabb98 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -117,6 +117,7 @@ def resolved_config_from_model_config( ) global_to_decoder = interface.get("target_global_to_decoder", {}) resolved = ResolvedModelInterface( + policy=interface.get("policy"), depth_layouts=interface.get("depth_layouts", {}), tensor_payload_version=interface.get("tensor_payload_version", 1), split_paths=interface.get("split_paths", []), diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index f99a8e4f..3d0f564c 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -91,6 +91,10 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: "version": interface.storage_layout.version, }, } + if interface.policy is not None: + interfaces[interface.name]["policy"] = interface.policy.model_dump( + mode="json" + ) spec = training_config.global_training return { "training_objective": spec.training_objective, diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 745daf66..53d1c959 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -80,8 +80,15 @@ def state_dict(self) -> dict[str, Any]: def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": if payload.get("artifact_type") != "sequifier_run_checkpoint": raise ValueError("Unsupported checkpoint: expected new Sequifier format.") + from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig from sequifier.config.train_config import ResolvedSequifierConfig + config_type = ( + ResolvedPolicyTrainConfig + if "trajectory_sources" in payload["training_config"] + else ResolvedSequifierConfig + ) + if ( payload.get("initialization_policy_version") != 1 or payload.get("initialization_seed_derivation_version") != 1 @@ -99,9 +106,7 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": integration_state=IntegrationState( values=dict(payload["integration_state"]["values"]) ), - training_config=ResolvedSequifierConfig.model_validate( - payload["training_config"] - ), + training_config=config_type.model_validate(payload["training_config"]), ) checkpoint.validate() return checkpoint diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index bc750bc1..4dc46a0d 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -23,6 +23,10 @@ extract_inline_metadata, load_dataset_metadata, ) +from sequifier.config.policy_infer_config import ( + PolicyInferenceConfig, + policy_inference_config, +) from sequifier.config.rollout_config import RolloutConfig from sequifier.helpers import ( ModelWindowView, @@ -389,7 +393,7 @@ def _assert_folder_metadata_matches( @beartype def load_inferer_config( config_path: str, args_config: dict, skip_metadata: bool -) -> "ResolvedInferenceConfig": +) -> "ResolvedInferenceConfig | PolicyInferenceConfig": """Compose inference YAML, load model defaults, and resolve metadata.""" config_values = load_composed_yaml_config(config_path) @@ -398,6 +402,9 @@ def load_inferer_config( key: value for key, value in args_config.items() if key != "skip_metadata" } authored_values = merge_config_fragments((config_values, cli_values)) + policy_config = policy_inference_config(authored_values) + if policy_config is not None: + return policy_config authored_values, extracted_metadata_values = extract_inline_metadata( authored_values ) @@ -436,7 +443,7 @@ def load_inferer_config( if isinstance(training, ResolvedPolicyTrainConfig): raise ValueError( - "policy actors execute through collect_policy_rollout; forecast inference requires a supervised training config" + "policy inference requires model_path pointing to its portable artifact" ) datasets = training.dataset_training if dataset_name is not None: @@ -503,6 +510,8 @@ def load_inferer_config( map_location="cpu", weights_only=False, ) + if payload.get("artifact_type") == "sequifier_run_checkpoint": + payload = payload["model"] model_config = payload.get("model_config") elif model_path.lower().endswith(".onnx"): import onnx @@ -1020,9 +1029,7 @@ def validate_numeric_rollout(self): if self.rollout is None: return self if self.rollout.mode != "forecast": - raise ValueError( - "policy alignment uses the policy API, not forecast inference" - ) + raise ValueError("rollout.mode: policy requires a policy model interface") if not self.autoregressive or self.generation_steps is None: raise ValueError( "rollout requires autoregressive: true and generation_steps" diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 5651970f..fc8b9c4b 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -1,10 +1,16 @@ -"""CPU categorical policy model and optimization configuration.""" +"""Policy alignment and standard model-component configuration.""" -from typing import Literal +from typing import Annotated, Literal from pydantic import BaseModel, ConfigDict, Field, FiniteFloat, model_validator -from sequifier.config.components import BackboneArchitectureConfig, ComponentSpec +from sequifier.config.components import ( + BackboneArchitectureConfig, + ComponentSpec, + DecoderComponentConfig, + IngestionComponentConfig, + ResumeConfig, +) class PolicyConfigBase(BaseModel): @@ -20,6 +26,8 @@ class ReturnDefinition(PolicyConfigBase): class PolicyModelConfig(PolicyConfigBase): context_length: int = Field(default=4, gt=0) backbone: BackboneArchitectureConfig + ingestion: IngestionComponentConfig | None = None + decoder: DecoderComponentConfig | None = None @model_validator(mode="before") @classmethod @@ -48,13 +56,6 @@ def validate_backbone(self): raise ValueError( "policy context_length exceeds backbone max_context_length" ) - if ( - self.backbone.position_encoding.type == "learned" - and self.backbone.positional_encoding_scope != "global" - ): - raise ValueError( - "policy learned positions require positional_encoding_scope: global" - ) return self @@ -64,6 +65,10 @@ class CategoricalPolicyConfig(PolicyModelConfig): ) observation_columns: list[str] = Field(min_length=1) static_columns: list[str] = Field(default_factory=list) + # Numeric trajectory fields may also carry already encoded categorical IDs. + observation_n_classes: dict[ + str, Annotated[int, Field(strict=True, ge=2, le=2**24)] + ] = Field(default_factory=dict) action_column: str = Field(default="action", min_length=1) action_values: list[FiniteFloat] = Field( default_factory=lambda: [0.0, 1.0], min_length=2 @@ -78,6 +83,21 @@ def validate_names(self): raise ValueError( "policy observation/static columns must be nonempty and disjoint" ) + reserved = {"__policy_previous_action", "__policy_return", "__policy_action"} + if reserved.intersection(names + [self.action_column]): + raise ValueError("policy feature names use reserved internal columns") + if set(self.observation_n_classes) - set(self.observation_columns) or any( + count < 2 for count in self.observation_n_classes.values() + ): + raise ValueError( + "observation_n_classes requires observation columns with at least two classes" + ) + if self.decoder is not None and ( + self.decoder.support != 1 or self.decoder.prediction_length != 1 + ): + raise ValueError( + "policy decoders require support=1 and prediction_length=1" + ) if len(self.action_values) != len(set(self.action_values)): raise ValueError("policy action_values must be unique") return self @@ -85,6 +105,9 @@ def validate_names(self): class PolicyOptimizationConfig(PolicyConfigBase): batch_size: int = Field(default=64, gt=0) + num_workers: int = Field(default=0, ge=0) + episode_cache_size: int = Field(default=8, gt=0) + resume: ResumeConfig | None = None validation_fraction: float = Field(default=0.2, gt=0, lt=1) learning_rate: float = Field(default=0.005, gt=0, allow_inf_nan=False) gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py new file mode 100644 index 00000000..ac5c6e2b --- /dev/null +++ b/src/sequifier/config/policy_infer_config.py @@ -0,0 +1,85 @@ +"""Inference options for policy interfaces in ordinary portable model artifacts.""" + +from pathlib import Path +from typing import Literal + +import torch +from pydantic import Field, model_validator + +from sequifier.config.policy_config import PolicyConfigBase +from sequifier.config.rollout_config import RolloutConfig + + +class PolicyInferenceConfig(PolicyConfigBase): + project_root: str = "." + model_path: str + model_type: Literal["policy", "generative"] = "generative" + model_interface: str | None = None + data_path: str + read_format: Literal["csv", "parquet"] = "csv" + write_format: Literal["csv", "parquet"] = "csv" + output_path: str | None = None + output_probabilities: bool = False + inference_batch_size: int = Field(default=64, gt=0) + device: str = "cpu" + seed: int = 1010 + deterministic: bool = True + desired_return: float | None = Field(default=None, allow_inf_nan=False) + desired_return_column: str = "desired_return" + previous_action_column: str | None = None + sequence_id_column: str = "sequenceId" + rollout: RolloutConfig | None = None + autoregressive: bool = False + generation_steps: int | None = Field(default=None, gt=0) + + @model_validator(mode="after") + def validate_execution(self): + if self.rollout is not None: + if self.rollout.mode != "policy": + raise ValueError("policy artifacts require rollout.mode: policy") + if not self.autoregressive or self.generation_steps is None: + raise ValueError( + "policy rollout requires autoregressive: true and generation_steps" + ) + if self.output_probabilities: + raise ValueError( + "rollout writes a trajectory including action log probabilities" + ) + elif self.autoregressive or self.generation_steps is not None: + raise ValueError("policy generation requires an environment rollout") + return self + + +def policy_inference_config(values: dict) -> PolicyInferenceConfig | None: + """Dispatch from the persisted interface contract, without training YAML.""" + model_path = values.get("model_path") + paths = [model_path] if isinstance(model_path, str) else model_path or [] + selected = values.get("model_interface") + policy_routes = [] + for path in paths: + if not isinstance(path, str) or not path.lower().endswith(".pt"): + continue + payload = torch.load( + Path(values.get("project_root", ".")) / path, + map_location="cpu", + weights_only=False, + ) + if payload.get("artifact_type") == "sequifier_run_checkpoint": + payload = payload["model"] + for name, interface in ( + payload.get("model_config", {}).get("interfaces", {}).items() + ): + if interface.get("policy") is not None and ( + selected is None or selected == name + ): + policy_routes.append(name) + if not policy_routes and values.get("model_type") != "policy": + return None + if len(paths) != 1 or len(policy_routes) != 1: + raise ValueError( + "policy inference requires one artifact and one selected policy interface" + ) + authored = dict(values) + authored["model_path"] = paths[0] + authored["model_interface"] = policy_routes[0] + return PolicyInferenceConfig.model_validate(authored) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index ceaf8279..252a0a9e 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Literal -from pydantic import Field, field_validator, model_validator +from pydantic import AliasChoices, Field, field_validator, model_validator from sequifier.config.policy_config import ( CategoricalPolicyConfig, @@ -42,7 +42,8 @@ class PolicyTrainingPlanConfig(PolicyConfigBase): class _PolicyTrainConfigBase(PolicyConfigBase): project_root: str = "." model_name: str = Field(min_length=1) - device: Literal["cpu"] = "cpu" + device: str = "cpu" + initialize_from: str | None = None seed: int = Field(default=1010, ge=0) trajectory_sources: dict[str, TrajectorySourceConfig] = Field(min_length=1) policies: dict[str, CategoricalPolicyConfig] = Field(min_length=1) @@ -52,6 +53,11 @@ class _PolicyTrainConfigBase(PolicyConfigBase): default_factory=PolicyOptimizationConfig ) + @property + def global_training(self) -> PolicyOptimizationConfig: + """Expose the shared checkpoint-selection contract.""" + return self.optimization + @field_validator("model_name") @classmethod def validate_model_name(cls, value): @@ -99,16 +105,26 @@ def validate_output_path(cls, value): class PolicyTrainConfig(_PolicyTrainConfigBase): """Authored paths may be omitted to select the standard output locations.""" - actor_path: str | None = None + actor_path: str | None = Field( + default=None, validation_alias=AliasChoices("model_path", "actor_path") + ) report_path: str | None = None + @property + def model_path(self) -> str | None: + return self.actor_path + class ResolvedPolicyTrainConfig(_PolicyTrainConfigBase): """Run paths are absolute and the authored phase has been validated.""" - actor_path: str + actor_path: str = Field(validation_alias=AliasChoices("model_path", "actor_path")) report_path: str + @property + def model_path(self) -> str: + return self.actor_path + def resolve_policy_train_config(config: PolicyTrainConfig) -> ResolvedPolicyTrainConfig: root = Path(config.project_root).resolve() @@ -122,8 +138,10 @@ def absolute(path): source["paths"] = [absolute(path) for path in source["paths"]] if source["manifest"] is not None: source["manifest"] = absolute(source["manifest"]) + if config.initialize_from is not None: + values["initialize_from"] = absolute(config.initialize_from) values["actor_path"] = absolute( - config.actor_path or f"models/{config.model_name}/actor.pt" + config.actor_path or f"models/{config.model_name}/{config.model_name}.pt" ) values["report_path"] = absolute( config.report_path or f"outputs/training/{config.model_name}/report.json" diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 43a85caa..96cef23e 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -59,6 +59,7 @@ OptimizerPlan, resolve_plan_scheduler_arguments, ) +from sequifier.config.policy_config import CategoricalPolicyConfig from sequifier.config.policy_train_config import ( PolicyTrainConfig, ResolvedPolicyTrainConfig, @@ -1165,6 +1166,7 @@ class ResolvedDatasetPart(BaseModel): class ResolvedModelInterface(BaseModel): + policy: CategoricalPolicyConfig | None = None model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") depth_layouts: DepthLayoutRegistryModel = Field( @@ -1677,8 +1679,7 @@ def _resolve_interface( if global_id in collision_ids } raise ValueError( - f"categorical_hashing[{column!r}]: {error} " - f"(category labels: {labels})" + f"categorical_hashing[{column!r}]: {error} (category labels: {labels})" ) from error hash_contracts[column] = codec.contract() generated_metric_names = { diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index c3584064..b2b0e6d4 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -105,7 +105,7 @@ There are seven standalone commands within sequifier: `make`, `preprocess`, `tra | `make` | Create a new sequifier project with config templates. | | `preprocess` | Convert input data into fixed-length subsequences. | | `train` | Train a supervised model or a trajectory-backed categorical policy. | -| `infer` | Generate predictions, probabilities, or embeddings. | +| `infer` | Generate predictions, probabilities, embeddings, or policy actions and rollouts. | | `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | | `visualize-training` | Generate interactive HTML plots from structured training metrics. | | `docs` | Print the consolidated documentation, or the guide for `train`, `preprocess`, or `infer`. | @@ -937,9 +937,8 @@ features from it. See [the training guide](train.md#depth-encoders-and-nested-co # Train Command Guide -For trajectory-backed categorical control learning and the local RC/BC -comparison, see [policy learning](../policy-learning.md) and the separate -`sequifier policy` command. +For trajectory-backed categorical policy training and inference through the same +commands, see [policy learning](../policy-learning.md). `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated @@ -1709,7 +1708,7 @@ gradients are absent. ## Trajectory-backed policy training `sequifier train` also accepts one `return_conditioned` phase with named -trajectory sources and a categorical policy registry. This CPU workflow supports +trajectory sources and a categorical policy registry. This workflow supports return conditioning or BC, with no supervised dataset placeholders. See the [policy training guide](../policy-learning.md) and [complete example](../example_inputs/configs/train-return-conditioned.yaml). @@ -1717,6 +1716,10 @@ The phase owns `epochs`; the run owns `seed`; `optimization` configures updates. Existing phases default to `algorithm: supervised`. Mixed algorithm plans and IQL/PPO remain planned. +Policy models use the shared configurable network and portable artifacts. They +support CPU/CUDA/MPS, backbone transfer with `initialize_from`, bounded file-backed +episode loading, and epoch-boundary resume through `optimization.resume`. + # Infer Command Guide @@ -1985,6 +1988,17 @@ The declared provider must be available; CUDA inference requires a CUDA-enabled ORT installation. Outputs stay indexed by outer item coordinates. +## Policy model interfaces + +`sequifier infer` recognizes a policy interface in a portable PyTorch model or +run checkpoint. Use raw CSV/Parquet observation rows and optional recorded action +histories; alignment, vocabulary, and normalization come from the artifact. +`model_type: generative` and `model_type: policy` are accepted. Environment +execution uses the existing `rollout` section with `mode: policy`. See the +[policy guide](../policy-learning.md#predict-actions-with-sequifier-infer) and +[inference example](../example_inputs/configs/infer-policy.yaml). + + # Visualize Training Command Guide The `sequifier visualize-training` command reads the structured metric files generated during training and hyperparameter search to create interactive Plotly HTML visualizations of the training and validation losses. It supports viewing a single model's progress or comparing multiple models side-by-side. @@ -2303,231 +2317,267 @@ srun torchrun \ * **Data Access:** All nodes must have access to the same shared filesystem (e.g., NFS, GPFS) where the `project_root` and the sharded preprocessing output are stored. -# Policy training through `sequifier train` +# Policy learning through `sequifier train` and `sequifier infer` + +Policy learning uses Sequifier's configurable ingestion, transformer network, +decoders, portable model artifacts, optimizer transactions, and run checkpoints. +A policy adds a decision-alignment contract to a model interface: at time `t`, +the inputs contain the current observation, the **previous** action, and, +optionally, a desired return. The current action is the supervised target. -Train an actor on saved trajectories with: +## Train from trajectories ```console -sequifier train --config-path documentation/example_inputs/configs/train-return-conditioned.yaml +sequifier train --config-path configs/train-policy.yaml +sequifier infer --config-path configs/infer-policy.yaml ``` -The configuration declares named `trajectory_sources`, named `policies`, and one -`training_plan.phases` entry with `algorithm: return_conditioned`. Ordinary -training phases default to `algorithm: supervised`. Policy phases have their own -source and model contract and require no supervised dataset, criterion, or -preprocessing metadata. This increment supports CPU execution and one categorical -actor in one phase. Mixed algorithm plans, phase transfer, IQL, PPO, distributed -policy learning, custom training integrations, and policy export remain planned. -Unsupported run fields are rejected before learning. - -A trajectory source specifies either `paths: [episode-a.json, episode-b.json]` or -`manifest: dataset/manifest.json`. A version-1 manifest has an `episodes` list of -objects with a `path` for each file. Only listed files are loaded. Run paths resolve -relative to `project_root`; manifest entries resolve relative to the manifest's -directory. These JSON episodes are the interim numeric trajectory format, preceding -the plan's sharded multimodal storage. - -The phase references `trajectory_sources: [demonstrations]` and `policy: actor`. -Set `epochs` on the phase and `seed` on the run. Optimizer, scheduler, batch size, -gradient clipping and automatic `validation_fraction` belong in `optimization`. -The actor's `returns` declares gamma, scale, and optional finite horizon. A concise -`training_plan: {algorithm: return_conditioned, epochs: 80, trajectory_sources: -[demonstrations], policy: actor}` is also accepted. +A complete training configuration is in +[train-return-conditioned.yaml](example_inputs/configs/train-return-conditioned.yaml). +`sequifier make` also creates `configs/train-policy.yaml`. Define named +`trajectory_sources`, named `policies`, and one `training_plan` phase with +`algorithm: return_conditioned`. Set `return_conditioned: false` on the policy +for behavior cloning (BC). Offline training does not start an environment. + +Each source specifies either `paths: [episode-a.json, episode-b.json]` or +`manifest: data/trajectories/manifest.json`. Version-1 manifests contain an +`episodes` list of objects with `path` and optional `episode_id`. Only listed +files are selected. Run paths resolve relative to `project_root`; manifest +entries resolve relative to the manifest directory. + +Set `epochs` on the phase, `seed` and `device` on the run, and update settings +in `optimization`. Devices follow PyTorch's CPU, CUDA, and MPS conventions. +`optimization.num_workers` controls data-loader workers and +`optimization.episode_cache_size` bounds the number of cached file-backed +episodes per worker (default 8). The learner retains episode metadata and loads +episodes on demand. It shuffles episodes and the decisions within each episode +to keep disk access local. Normalization combines episode statistics +incrementally. JSON storage still requires an individual episode to fit in memory. Without `validation_sources`, the runner splits whole episodes deterministically, -keeping identical recorded trajectories together. Repeated copies therefore do -not cross the partition boundary. At least two distinct trajectory groups are -required. For authored partitions, add `validation_sources: [held_out]` to the -phase and declare that separate source in the registry. Duplicate file paths, -episode IDs, or identical trajectory content across partitions are rejected. -Normalization is fit only on training episodes. Truncations and collection cutoffs -with unknown returns are excluded under the existing return-label contract; -excluded IDs are recorded in the report. - -The runtime uses the existing optimization transactions and selects the best -actor by held-out action NLL. `actor_path` defaults to -`models//actor.pt`; `report_path` defaults to -`outputs/training//report.json`. The report records resolved config, -selected files, partition IDs, decision counts, excluded episodes, per-epoch NLL, -optimizer steps and checkpoint selection. Actor files contain model weights, -normalization buffers, vocabulary, alignment and return settings. They are -actor-only inference artifacts; optimizer and RNG state for exact resume are not -included. Use `load_policy` and `collect_policy_rollout` to execute them. - -For behavior cloning, set the actor's `return_conditioned: false` in the same -workflow. Offline training requires only saved trajectories; it does not start an -environment. Python callers use `load_train_config` and `run_training` with the -same configuration as the CLI. - -`sequifier docs policy` prints this guide. - -## Policy alignment and causal windows - -At decision position `t`, the policy receives `o_t`, `a_(t-1)`, and, for return -conditioning, `G_t`. The current action `a_t` is a separate target. The first -previous action is a dedicated start-token ID equal to the number of executable -actions. It cannot be selected by the action head. - -Offline and online execution use `build_policy_inputs` to construct the same -left-padded causal windows. Observations, previous actions, and returns are -trimmed together. Padding has a false attention mask. Only the last position -of each training example has a true action-likelihood mask, so every recorded -decision contributes once even when context windows overlap. - -The actor uses numeric projection, previous-action embedding, the same -`TransformerBackbone` and encoder layers as `sequifier train`, and one categorical -head. Attention is causal and excludes padded keys; padded logits are zero. -The model recomputes each finite context window rather than retaining recurrent -state across windows. -Forward calls take `PolicyInputs`, which contain no action labels or observed -current rewards. A change to a current-action label cannot affect that position's -inputs; future observations are excluded from its window. Return-to-go is the -intentional reward-derived training condition, with requested budgets replacing -it online. Changing rewards can therefore change the return condition. - -Each entry in the `policies` registry and `CategoricalPolicyConfig` accepts a -`backbone` section using the same architecture fields as training's -`model.backbone.architecture`: +keeping identical demonstrations together. At least two distinct trajectory +groups are required. To supply held-out data, declare a separate source and add +`validation_sources: [held_out]` to the phase. Duplicate paths and episode IDs, +and identical content across training and validation, are rejected. +Normalization fits only the training partition. + +The best model is selected by held-out action negative log likelihood (NLL). +`model_path` defaults to `models//.pt` and contains an +ordinary `sequifier_model` artifact. `actor_path` remains an alias for `model_path` for existing configurations. `report_path` +defaults to `outputs/training//report.json`; it records sources, +partition IDs, exclusions, decision counts, per-epoch NLL, and optimizer steps. + +## Configure the shared model + +The policy's `backbone` accepts the architecture fields from ordinary training's +`model.backbone.architecture`. Optional `ingestion` and `decoder` use the same +component schemas and builders as forecasting. For example: ```yaml -context_length: 4 -backbone: - dim_model: 32 - num_layers: 2 - max_context_length: 4 - attention: {type: mha, n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: learned} - positional_encoding_scope: global - dropout: 0.0 - shared_layer_groups: [] +policies: + actor: + observation_columns: [position, velocity] + action_column: action + action_values: [-1.0, 1.0] + context_length: 16 + return_conditioned: true + backbone: + dim_model: 64 + num_layers: 3 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} + positional_encoding_scope: global + decoder: {type: mlp, hidden_dims: [64], prediction_length: 1, support: 1} ``` -This includes MHA/MQA/GQA, feed-forward activation, normalization, positional -encoding (learned, RoPE, sinusoidal, range or range-concat), dropout and layer -sharing. `context_length` must fit `max_context_length`. Learned positions require -`global` scope because policy inputs use a combined numeric projection rather -than per-feature position embeddings. Scope defaults to `global` for policies. -Without `backbone`, defaults are two layers, width 32, one attention head and -feed-forward width 128. Configure width with `backbone.dim_model`; the experimental -`hidden_size` option has been removed. - -Transformer actor artifacts use version 2. Version 1 GRU artifacts cannot be -loaded into this architecture and must be retrained. Sharing backbone primitives -does not yet provide forecast-checkpoint transfer. The numeric policy input -projection, action head and CPU-only execution remain policy-specific. - -Observation/static normalization is fit on the training episode partition only. -Its fixed means/scales are model buffers, saved with the actor. BC disables the return channel by feeding zero and retains the same parameter -count as the corresponding return-conditioned actor. - -## Return definitions and episode boundaries - -Raw labels use `G_t = r_t + gamma * G_(t+1)` and terminal `G_T = 0`. -For rewards `[1, 2, 3]`, gamma 1 produces `[6, 5, 3]`; gamma 0.5 produces -`[2.75, 3.5, 3]`. `returns.gamma` is explicit in `(0, 1]`, defaulting to 1. -`returns.scale` divides returns only at the numeric encoder boundary and defaults -to 1. Artifacts and execution traces preserve raw reward units. - -By default, only trajectories with true termination have exact return labels. -Collection cutoffs and truncation without termination are excluded and their -episode IDs are exposed by the dataset. An explicit `returns.finite_horizon: H` -allows labeling an observed prefix of exactly H decisions, including a cutoff -at H. A fragment shorter than H is still excluded unless it truly terminates. -No unknown tail is silently assigned zero. For online use of this objective, -`rollout.max_episode_steps` must equal H. - -Execution starts from the caller's requested return and updates the raw budget -after each executed reward: +Without an authored ingestion, the standard embedding ingestion projects real +features and embeds categorical features before an adapter maps to the backbone +input width. Real features receive one embedding channel and categorical features +share the remaining width. The default decoder is linear. A policy decoder must +have `support: 1` and `prediction_length: 1`. + +The internal input names `__policy_previous_action` and `__policy_return` are +reserved. The latter exists only for return-conditioned policies. Static columns +are broadcast over the decision window. Real observation and static statistics +are persisted in the model interface's preprocessing metadata. Encoded categorical +observations can be declared with `observation_n_classes: {state_id: 8}`; their +recorded values must be integral IDs in `[0, 8)` and are not normalized. +When authoring a mixed real/categorical embedding ingestion, provide +`feature_embedding_dims` under the normal ingestion rules, including the internal +previous-action column and, when enabled, return column. + +Actions use the ordered IDs of `action_values`; an extra input-only ID equal to +the action count represents episode start. It is excluded from decoder outputs. +An observation may have the same external name as the action; in that case the +model uses `__policy_action` as its separate target name. Default decoders infer +the target automatically. + +The default backbone has width 32, two layers, one attention head, and +feed-forward width 128. Context length must fit `max_context_length`. +Positional scope defaults to `global`; the standard ingestion also supports +feature-scoped learned positions when configured. +The standard attention, normalization, position encodings, dropout, and layer +sharing options remain available. + +To transfer a forecast or policy backbone, set a top-level +`initialize_from: models/forecast.pt`. Portable models and run checkpoints are +accepted. The source must have the same backbone architecture. Ingestion and +decoder parameters are initialized for the policy's own interface. Resumption +restores the entire existing policy instead of applying initialization again. + +## Resume training + +Every completed epoch saves a standard `sequifier_run_checkpoint` at +`checkpoints/runs//-latest.pt`. It contains current model +weights, preprocessing and policy semantics, optimizer/scheduler/scaler state, +random state, data-order generator state, best weights, and training progress. +The portable model at `model_path` contains the best validation weights; the run +checkpoint contains the latest training weights. + +```yaml +optimization: + resume: + policy: if_exists +``` + +`never` is the default. `required` also requires an explicit `checkpoint_path`, +using the shared resume configuration. Resume occurs at epoch boundaries; +an interrupted epoch is repeated. Trajectory content, partition order, policy, +and learning settings must match. Device, output paths, worker/cache settings, +and total epoch budget may change. Restored scheduler state retains its existing +schedule when extending the epoch budget. Numerical reproducibility across +devices is governed by PyTorch. + +## Predict actions with `sequifier infer` + +Inference reads policy alignment and preprocessing directly from a portable +model or run checkpoint; a training YAML or preprocessing metadata file is not +needed. It accepts `model_type: generative` (or the explicit alias `policy`): + +```yaml +project_root: . +model_path: models/policy/policy.pt +data_path: data/observations.csv +read_format: csv +write_format: csv +device: cpu +deterministic: true +desired_return: 10.0 +``` + +See [infer-policy.yaml](example_inputs/configs/infer-policy.yaml). CSV and Parquet +inputs contain observations and any static columns in original units. Without +a `sequenceId` column, each row is an independent first decision. Within each +sequence, row order defines decision order and static inputs must be constant. +Provide the recorded `action_column` to derive previous actions by shifting one +row, or set `previous_action_column` to a column containing previous **domain +action values**, with null for episode start. Use a new sequence ID for each +episode; null previous actions inside a sequence are rejected. When observation +and action names coincide, use an explicit `previous_action_column` for sequential +CSV/Parquet input. Current recorded actions never +enter the inputs for the same decision. Historical context is bounded by the +model's context length. + +Return-conditioned models accept a fixed `desired_return` or a per-row column +named by `desired_return_column` (default `desired_return`). Per-row returns are +supplied by the caller; offline inference does not update budgets from rewards. +BC requires no return input and rejects a scalar `desired_return`. + +`inference_batch_size` bounds batched model calls (default 64). Predictions go to `outputs/predictions//part-000.csv` by default. +`output_path` overrides that location. Sequence IDs and `itemPosition`, when +present, are retained in input order. `output_probabilities: true` adds one +`__probability_` column per action. Set +`deterministic: false` for seeded categorical sampling. Select `model_interface` +when an artifact contains multiple policy interfaces. + +## Execute an environment rollout + +Add the ordinary `rollout` configuration to policy inference: + +```yaml +autoregressive: true +generation_steps: 1000 +rollout: + mode: policy + adapter: {factory: my_environment:create_adapter, config: {}} + bindings: + - {input: position, source: environment, key: position} + - {input: velocity, source: environment, key: velocity} + trajectory_path: outputs/trajectories/episode.json + seed_index: 0 +``` + +`seed_index` selects one raw input row, forming a single-observation reset seed. +The shared rollout coordinator owns reset, step ordering, boundaries, and adapter +closure. Each predicted action enters policy history only after it is executed. +The [adapter protocol](rollout.md#adapter-protocol) is the same as for forecasts. +The resulting trajectory can be used as a later training source. + +The raw return budget starts at `desired_return` (or the seed row's return +column) and updates after each executed reward: ```text G_next = (G_current - reward) / gamma ``` -The request is not clamped to an inferred feasible range. Each episode reset -restores the requested budget, clears previous-action/history state, and uses -the start token again. A collection cutoff preserves the unfinished episode -flag; it does not fabricate a reset. Each call collects one episode/fragment, -so collecting another episode uses a fresh adapter and an explicit reset. - -Policy trajectories use `format: sequifier.numeric-policy-trajectory`, version 1, -with the same seed/transition schema and storage as numeric forecast trajectories. -`mode: policy` makes this alignment explicit. Forecast artifacts retain their -existing format and require an explicit declaration before their executed -predictions can be used as demonstration actions. -Policy episodes start with one valid observation because historical actions -before reset are unknown. The actor builds subsequent history from executed -transitions. - -Execution metadata stores the full policy/return configuration, initial request, -remaining budgets (including the final successor budget), previous-action IDs, -and selected-action log probabilities. The trajectory validator checks action -vocabulary, previous-action alignment, budget counts, and reward/gamma arithmetic -in addition to ordinary continuity, reward, and boundary validation. Both flags -may be true; termination takes precedence for bootstrap. Adapter closure remains -owned by the shared coordinator on success and failure. - -## Reusable Python interfaces - -The public policy entry points live in `sequifier.rl_api`. Python callers can -use the same configuration loader and training entry point as the CLI: +Each reset restores the requested budget and clears policy history. The trajectory +records action log probabilities, previous-action IDs, and remaining raw budgets. +Termination and truncation remain separate; a collection cutoff preserves the +unfinished-episode flag. The coordinator always closes the adapter. + +## Labels, windows, and boundaries + +Raw return labels obey `G_t = r_t + gamma * G_(t+1)`, with terminal `G_T = 0`. +`returns.gamma` defaults to 1 and is in `(0, 1]`; `returns.scale` defaults to 1 +and divides the return only at ingestion. For rewards `[1, 2, 3]`, gamma 1 gives +`[6, 5, 3]`, and gamma 0.5 gives `[2.75, 3.5, 3]`. + +Return-conditioned training excludes truncations and collection cutoffs with +unknown tails. `returns.finite_horizon: H` can label a fully observed prefix of +H decisions. Shorter fragments need true termination. Online finite-horizon +execution requires `rollout.max_episode_steps: H`. No unknown tail is assigned +zero. BC uses **all observed decisions**, including truncated and cutoff episodes, +because its loss does not consume return labels. + +Offline and online paths share `build_policy_inputs`, producing left-padded, +causal histories. The dataset slices to the context window before building +inputs, retaining the actual preceding action at a cropped boundary. Each +training example labels only its last position, so every decision contributes +once. Current actions, current rewards, and future observations are excluded +from inputs; return-to-go is the intentional future-derived condition. + +Policy trajectories use `sequifier.numeric-policy-trajectory`, version 1. +Forecast trajectories can be demonstrations with explicit +`action_semantics: executed_predictions` and a finite `action_values` vocabulary. +They retain the latest reset observation and start previous-action history with +the start token, because earlier forecast seed actions are unknown. + +## Python interfaces and scope ```python from sequifier.config.train_config import load_train_config from sequifier.train import run_training -from sequifier.rl_api import load_policy +from sequifier.rl_api import load_policy, build_policy_inputs config = load_train_config("configs/train-policy.yaml", {}, False) run_training(config) -actor = load_policy(config.actor_path) -``` - -First-increment forecast trajectories can also supply demonstrations when their -executed numeric predictions have a declared finite categorical vocabulary: -pass `action_semantics="executed_predictions"` to `TrajectoryDataset` and set -the appropriate `action_column` and `action_values` in the policy specification. -This is an explicit data interpretation, not a conversion of forecast model -alignment or weights. It uses the latest seed observation as decision zero, -discards earlier seed observations whose actions were never recorded, and -starts previous-action history with the start token. Episode splitting preserves -this declaration. Observation and action names may coincide because their -values are read from separate observation/target records; current labels are -never copied into current inputs. - -Action values are original numeric domain values; the distribution uses their -ordered categorical IDs internally. `actor.distribution(inputs)` returns a -`CategoricalActionDistribution` supporting `mode()`, `sample(generator)`, -`log_prob(recorded_action_ids)`, and `entropy()`. `masked_action_likelihood` -uses the dataset's independent action-valid mask. Training calls the existing -`OptimizationRuntime` for backward, clipping, optimizer, scaler, and scheduler -steps. The validation partition selects the saved best actor by action NLL. - -For execution, supply an application-owned environment adapter, a validated -`SeedHistory`, and a `RolloutConfig` with `mode: policy`. Bind each changing input -to the appropriate environment field or executed action. The adapter follows the -[rollout protocol](rollout.md#adapter-protocol). - -```python -from sequifier.rl_api import collect_policy_rollout -from sequifier.rollout_api import save_trajectory - -# actor, adapter, seed, rollout_config and requested_return come from the application. -trajectory = collect_policy_rollout( - actor, adapter, seed, rollout_config, - max_steps=1000, desired_return=requested_return, deterministic=True, -) -save_trajectory(trajectory, rollout_config.trajectory_path) +policy = load_policy(config.model_path, device="cpu") +# inputs = build_policy_inputs(policy.config, observations, previous_action_ids, +# desired_returns, static_inputs).to(policy.device) +# distribution = policy.distribution(inputs) ``` -Set `return_conditioned=False` in the policy specification to train BC, and omit -`desired_return` when executing it. The actor artifacts contain validated CPU -weights, normalization buffers, action vocabulary, policy alignment, and return -semantics. Loading uses strict state-dict matching. These local inference -artifacts do not contain optimizer/RNG state for exact training resumption and -do not use forecast export or inference formats. - -This increment has no critics, Q networks, PPO buffers, distributed execution, -or policy export additions. Return conditioning and unconditioned behavior -cloning are independent actor routes. Future offline IQL can reuse the -trajectory windows and distribution interface without requiring RC pretraining. +`CategoricalPolicy` is an alignment adapter over the ordinary network, available +as `policy.network`. `policy.execution_config` is the canonical model config, +compatible with the shared artifact and export services. Distribution methods +include `mode`, `sample`, `log_prob`, and `entropy`. `collect_policy_rollout` +remains available for application-owned adapters. Generic analysis/loading tools +also accept policy model and run artifacts. + +The previous standalone numeric-policy artifacts (including versions 1 and 2) +use a different architecture and must be retrained. This workflow currently +supports one offline categorical actor phase. Mixed supervised/policy plans, +critics, IQL/PPO, distributed policy learning, custom run integrations, multimodal +trajectory storage, and ONNX policy execution through the CLI remain outside +this workflow. PyTorch policy execution supports CPU, CUDA, and MPS. diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index 92f1d584..fb796708 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -167,6 +167,14 @@ def infer(args: Any, args_config: dict[str, Any]) -> None: skip_metadata = args_config.get("skip_metadata", False) config = load_inferer_config(config_path, args_config, skip_metadata) + from sequifier.config.policy_infer_config import PolicyInferenceConfig + + if isinstance(config, PolicyInferenceConfig): + from sequifier.rl.inference import infer_policy + + infer_policy(config) + return + if config.decode_categories or "real" in config.target_column_types.values(): metadata = config.dataset_metadata if metadata is None: diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 4c298e43..72bc935e 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -115,6 +115,20 @@ batch_size: 64 learning_rate: 0.005 validation_fraction: 0.2 + episode_cache_size: 8 + resume: {policy: never} +""" + +policy_infer_config_string = """project_root: . +model_path: models/policy/policy.pt +data_path: PLEASE_FILL +model_type: generative +read_format: csv +write_format: csv +device: cpu +deterministic: true +# Omit desired_return for behavior cloning. +desired_return: 10.0 """ gitignore_string = """models/ @@ -151,3 +165,6 @@ def make(args): with open(f"{project_name}/configs/train-policy.yaml", "w") as f: f.write(policy_train_config_string) + + with open(f"{project_name}/configs/infer-policy.yaml", "w") as f: + f.write(policy_infer_config_string) diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 24f7c1fe..b8304048 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -1,69 +1,60 @@ -"""Versioned local actor weights, input transforms, and policy/return semantics.""" +"""Policies use the same portable model and run-checkpoint artifacts as forecasts.""" -import json -import os -import tempfile from pathlib import Path import torch -from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.artifacts.model_artifact import ( + build_model_artifact, + load_execution_model, + save_model_artifact, +) from sequifier.rl.policy import CategoricalPolicy def validate_policy_weights(policy: CategoricalPolicy) -> None: - for name, value in policy.state_dict().items(): - if value.device.type != "cpu" or not torch.isfinite(value).all(): - raise ValueError(f"policy tensor {name!r} must be finite and on CPU") - if (policy.observation_scale <= 0).any() or (policy.static_scale <= 0).any(): - raise ValueError("policy normalization scales must be positive") + for name, value in policy.network.state_dict().items(): + if not torch.isfinite(value).all(): + raise ValueError(f"policy tensor {name!r} must be finite") + statistics = policy.interface.selected_columns_statistics + policy.set_normalization_statistics(statistics) def save_policy( policy: CategoricalPolicy, path: str | Path, *, metadata: dict | None = None ) -> None: validate_policy_weights(policy) - json.dumps(metadata or {}, allow_nan=False) - payload = { - "format": "sequifier.numeric-categorical-policy", - "version": 2, - "config": policy.config.model_dump(mode="json"), - "state_dict": policy.state_dict(), - "metadata": metadata or {}, - } - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - temporary = None - try: - with tempfile.NamedTemporaryFile( - dir=destination.parent, delete=False - ) as stream: - temporary = stream.name - torch.save(payload, stream) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, destination) - finally: - if temporary is not None and os.path.exists(temporary): - os.unlink(temporary) - - -def load_policy(path: str | Path) -> CategoricalPolicy: - payload = torch.load(path, map_location="cpu", weights_only=True) - if ( - not isinstance(payload, dict) - or payload.get("format") != "sequifier.numeric-categorical-policy" - or payload.get("version") != 2 - ): - raise ValueError( - "unsupported numeric categorical policy artifact; transformer actors require " - "version 2 (version 1 GRU weights cannot be loaded)" - ) - config = CategoricalPolicyConfig.model_validate(payload["config"]) - json.dumps(payload.get("metadata", {}), allow_nan=False) - with torch.random.fork_rng(devices=[]): - policy = CategoricalPolicy(config) - policy.load_state_dict(payload["state_dict"], strict=True) + save_model_artifact( + build_model_artifact( + policy.network, + policy.execution_config, + provenance=metadata, + state_dict={ + name: value.detach().cpu() + for name, value in policy.network.state_dict().items() + }, + ), + path, + ) + + +def load_policy( + path: str | Path, *, device: str = "cpu", interface_name: str | None = None +) -> CategoricalPolicy: + network, config, _ = load_execution_model( + path, device=device, interface_name=interface_name + ) + if interface_name is None: + candidates = [ + name + for name, dataset in config.dataset_training.items() + if dataset.interface.policy is not None + ] + if len(candidates) != 1: + raise ValueError("Select one policy model_interface") + interface_name = candidates[0] + assert interface_name is not None + policy = CategoricalPolicy.from_network(network, config, interface_name) validate_policy_weights(policy) policy.eval() return policy diff --git a/src/sequifier/rl/distributions.py b/src/sequifier/rl/distributions.py index 62a9641a..f32c0623 100644 --- a/src/sequifier/rl/distributions.py +++ b/src/sequifier/rl/distributions.py @@ -18,9 +18,15 @@ def mode(self) -> Tensor: def sample(self, generator: torch.Generator | None = None) -> Tensor: probabilities = self.distribution.probs - sampled = torch.multinomial( - probabilities.reshape(-1, probabilities.shape[-1]), 1, generator=generator + # MPS has no device-specific Generator; sample on its generator's device. + sampling_device = ( + generator.device if generator is not None else probabilities.device ) + sampled = torch.multinomial( + probabilities.to(sampling_device).reshape(-1, probabilities.shape[-1]), + 1, + generator=generator, + ).to(probabilities.device) return sampled.reshape(probabilities.shape[:-1]) def log_prob(self, actions: Tensor) -> Tensor: diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py index 270a4274..c85350aa 100644 --- a/src/sequifier/rl/history.py +++ b/src/sequifier/rl/history.py @@ -18,6 +18,44 @@ class PolicyInputs: static_inputs: Tensor desired_returns: Tensor | None + def to(self, device: str | torch.device) -> "PolicyInputs": + return PolicyInputs( + self.observations.to(device), + self.previous_actions.to(device), + self.attention_valid_mask.to(device), + self.static_inputs.to(device), + self.desired_returns.to(device) + if self.desired_returns is not None + else None, + ) + + +def collate_policy_inputs(inputs: Sequence[PolicyInputs]) -> PolicyInputs: + if not inputs: + raise ValueError("cannot collate empty policy inputs") + conditioned = inputs[0].desired_returns is not None + if any((value.desired_returns is not None) != conditioned for value in inputs): + raise ValueError("policy batches cannot mix conditioning modes") + return PolicyInputs( + observations=torch.cat([value.observations for value in inputs]), + previous_actions=torch.cat([value.previous_actions for value in inputs]), + attention_valid_mask=torch.cat( + [value.attention_valid_mask for value in inputs] + ), + static_inputs=torch.cat([value.static_inputs for value in inputs]), + desired_returns=( + torch.cat( + [ + value.desired_returns + for value in inputs + if value.desired_returns is not None + ] + ) + if conditioned + else None + ), + ) + def action_index(value: float, config: CategoricalPolicyConfig) -> int: value = numeric_scalar(value, "recorded action") @@ -64,6 +102,12 @@ def build_policy_inputs( values = numeric_fields( observation, set(config.observation_columns), "policy observation" ) + for column, count in config.observation_n_classes.items(): + value = values[column] + if value != int(value) or not 0 <= value < count: + raise ValueError( + f"{column}: observation must be a categorical ID in [0, {count})" + ) feature_rows.append([values[column] for column in config.observation_columns]) action_ids = [len(config.action_values)] * padding for action in previous_actions[start:]: diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py new file mode 100644 index 00000000..38f7852a --- /dev/null +++ b/src/sequifier/rl/inference.py @@ -0,0 +1,246 @@ +"""CLI policy prediction and environment execution using the shared model lifecycle.""" + +from collections import deque +from pathlib import Path + +import polars as pl +import torch +from loguru import logger + +from sequifier.config.policy_infer_config import PolicyInferenceConfig +from sequifier.helpers import configure_determinism, normalize_path, write_data +from sequifier.rl.checkpoint import load_policy +from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.rl.history import ( + action_index, + build_policy_inputs, + collate_policy_inputs, +) +from sequifier.rl.runtime import collect_policy_rollout +from sequifier.rollout.factories import load_adapter +from sequifier.rollout.history import numeric_fields, numeric_scalar +from sequifier.trajectory.schema import SeedHistory +from sequifier.trajectory.store import save_trajectory + + +def infer_policy(config: PolicyInferenceConfig) -> None: + configure_determinism(config.seed, config.deterministic) + policy = load_policy( + normalize_path(config.model_path, config.project_root), + device=config.device, + interface_name=config.model_interface, + ) + spec = policy.config + path = normalize_path(config.data_path, config.project_root) + data = pl.read_csv(path) if config.read_format == "csv" else pl.read_parquet(path) + if not len(data): + raise ValueError("policy inference input must be nonempty") + required = set(spec.observation_columns + spec.static_columns) + if config.previous_action_column is not None and config.rollout is None: + required.add(config.previous_action_column) + if spec.return_conditioned and config.desired_return is None: + required.add(config.desired_return_column) + if not spec.return_conditioned and config.desired_return is not None: + raise ValueError("behavior cloning has no desired-return input") + if required - set(data.columns): + raise ValueError( + f"policy inference is missing columns: {sorted(required - set(data.columns))}" + ) + + def budget(row): + return ( + numeric_scalar( + config.desired_return + if config.desired_return is not None + else row[config.desired_return_column], + "desired_return", + ) + if spec.return_conditioned + else None + ) + + if config.rollout is not None: + assert config.generation_steps is not None + rollout = config.rollout + if rollout.seed_index >= len(data): + raise ValueError("policy rollout seed_index is outside the input rows") + row = data.row(rollout.seed_index, named=True) + seed = SeedHistory( + observations={ + column: [0.0] * (spec.context_length - 1) + + [numeric_scalar(row[column], column)] + for column in spec.observation_columns + }, + attention_valid_mask=[False] * (spec.context_length - 1) + [True], + static_inputs={ + column: numeric_scalar(row[column], column) + for column in spec.static_columns + }, + ) + desired_return = budget(row) + trajectory = collect_policy_rollout( + policy, + load_adapter(rollout.adapter), + seed, + rollout, + config.generation_steps, + desired_return=desired_return, + deterministic=config.deterministic, + random_seed=config.seed, + ) + trajectory.metadata.update( + { + "model_path": config.model_path, + "model_interface": config.model_interface, + "seed_index": rollout.seed_index, + } + ) + destination = normalize_path(rollout.trajectory_path, config.project_root) + save_trajectory(trajectory, destination) + logger.info( + f"Wrote {len(trajectory.transitions)} policy transitions to '{destination}'" + ) + return + + # Rows without a sequence ID are independent single-observation decisions. + # Within sequences, recorded actions determine the causal previous-action input. + indexed = data.with_row_index("__policy_row") + groups = ( + indexed.partition_by(config.sequence_id_column, maintain_order=True) + if config.sequence_id_column in data.columns + else [indexed.slice(i, 1) for i in range(len(indexed))] + ) + outputs = [] + generator = torch.Generator( + device=policy.device if policy.device.type == "cuda" else "cpu" + ).manual_seed(config.seed) + pending_inputs = [] + pending_rows = [] + + def flush(): + if not pending_inputs: + return + with torch.no_grad(): + inputs = collate_policy_inputs(pending_inputs).to(policy.device) + decision = CategoricalActionDistribution(policy(inputs)[:, -1, :]) + actions = ( + ( + decision.mode() + if config.deterministic + else decision.sample(generator) + ) + .cpu() + .tolist() + ) + probabilities = ( + decision.distribution.probs.cpu().tolist() + if config.output_probabilities + else None + ) + for index, (row_index, action) in enumerate(zip(pending_rows, actions)): + output = { + "__policy_row": row_index, + spec.action_column: spec.action_values[action], + } + if probabilities is not None: + output.update( + { + f"{spec.action_column}__probability_{action_id}": value + for action_id, value in enumerate(probabilities[index]) + } + ) + outputs.append(output) + pending_inputs.clear() + pending_rows.clear() + + with torch.no_grad(): + for group in groups: + observations, previous_actions, budgets = ( + deque(maxlen=spec.context_length) for _ in range(3) + ) + previous_action = len(spec.action_values) + initial_static = None + if ( + len(group) > 1 + and config.previous_action_column is None + and spec.action_column not in group.columns + ): + raise ValueError( + "sequential policy inference requires recorded actions or previous_action_column" + ) + if ( + len(group) > 1 + and config.previous_action_column is None + and spec.action_column in spec.observation_columns + spec.static_columns + ): + raise ValueError( + "previous_action_column is required when observations and actions share a name" + ) + for position, row in enumerate(group.iter_rows(named=True)): + static = numeric_fields( + {column: row[column] for column in spec.static_columns}, + set(spec.static_columns), + "policy static inputs", + ) + if initial_static is not None and static != initial_static: + raise ValueError( + "policy static inputs must remain constant within a sequence" + ) + initial_static = static + observations.append( + {column: row[column] for column in spec.observation_columns} + ) + if config.previous_action_column is not None: + value = row[config.previous_action_column] + if position > 0 and value is None: + raise ValueError( + "null previous actions are allowed only at sequence start" + ) + previous_action = ( + len(spec.action_values) + if value is None + else action_index(value, spec) + ) + previous_actions.append(previous_action) + budgets.append(budget(row)) + pending_inputs.append( + build_policy_inputs( + spec, + list(observations), + list(previous_actions), + list(budgets) if spec.return_conditioned else None, + static, + ) + ) + pending_rows.append(row["__policy_row"]) + if len(pending_inputs) >= config.inference_batch_size: + flush() + if config.previous_action_column is None and position + 1 < len(group): + previous_action = action_index(row[spec.action_column], spec) + flush() + predictions = pl.DataFrame(outputs).sort("__policy_row").drop("__policy_row") + identity = [ + column + for column in (config.sequence_id_column, "itemPosition") + if column in data.columns and column not in predictions.columns + ] + predictions = ( + pl.concat([data.select(identity), predictions], how="horizontal") + if identity + else predictions + ) + if config.output_path is None: + from sequifier.infer import inference_output_path + + destination = inference_output_path( + config.project_root, + config.write_format, + "predictions", + Path(config.model_path).stem, + 0, + ) + else: + destination = normalize_path(config.output_path, config.project_root) + Path(destination).parent.mkdir(parents=True, exist_ok=True) + write_data(predictions, destination, config.write_format) + logger.info(f"Wrote {len(predictions)} policy predictions to '{destination}'") diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 661e087e..6293ed5e 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -3,8 +3,9 @@ import json from dataclasses import dataclass, field from pathlib import Path -from typing import Protocol +from typing import Literal, Protocol +from sequifier.artifacts.run_checkpoint import checkpoint_path from sequifier.config.policy_config import PolicyTrainingConfig from sequifier.config.policy_train_config import ( ResolvedPolicyTrainConfig, @@ -12,11 +13,7 @@ ) from sequifier.rl.checkpoint import save_policy from sequifier.rl.training import PolicyTrainingResult, train_policy -from sequifier.trajectory.dataset import ( - TrajectoryDataset, - episode_content_key, - split_by_episode, -) +from sequifier.trajectory.dataset import TrajectoryDataset, split_by_episode from sequifier.trajectory.store import load_trajectory @@ -78,32 +75,36 @@ def prepare(self) -> None: manifest = self.config.trajectory_sources[name].manifest if manifest is not None: protected_paths.add(Path(manifest).resolve()) - if { + output_paths = [ Path(self.config.actor_path), Path(self.config.report_path), - } & protected_paths: + checkpoint_path(self.config), + ] + if len(output_paths) != len(set(output_paths)): + raise ValueError("model, report, and run checkpoint paths must differ") + if set(output_paths) & protected_paths: raise ValueError("training outputs must not overwrite trajectory inputs") if len(flattened) != len(set(flattened)): raise ValueError( "trajectory files cannot appear in multiple selected sources" ) - episodes = {path: load_trajectory(path) for path in flattened} - # Include episodes excluded from exact return labeling in identity checks. - ids = [episode.episode_id for episode in episodes.values()] + # Validate identities incrementally, including episodes without exact returns. + ids = [load_trajectory(path).episode_id for path in flattened] if len(ids) != len(set(ids)): raise ValueError("duplicate episode IDs across trajectory sources") def dataset(names: list[str]) -> TrajectoryDataset: - semantics = { + semantics: set[Literal["policy", "executed_predictions"]] = { self.config.trajectory_sources[name].action_semantics for name in names } if len(semantics) != 1: raise ValueError("selected sources must share action_semantics") return TrajectoryDataset( - [episodes[path] for name in names for path in all_paths[name]], + [path for name in names for path in all_paths[name]], spec, action_semantics=next(iter(semantics)), + episode_cache_size=self.config.optimization.episode_cache_size, ) corpus = dataset(phase.trajectory_sources) @@ -119,9 +120,7 @@ def dataset(names: list[str]) -> TrajectoryDataset: self.training = corpus self.validation = dataset(phase.validation_sources) self.excluded_episode_ids += self.validation.excluded_episode_ids - if {episode_content_key(e) for e in self.training.episodes} & { - episode_content_key(e) for e in self.validation.episodes - }: + if set(self.training.content_keys) & set(self.validation.content_keys): raise ValueError( "identical trajectories cannot cross training/validation partitions" ) @@ -136,7 +135,9 @@ def update(self) -> PolicyTrainingResult: epochs=phase.epochs, seed=self.config.seed, ) - self.result = train_policy(self.training, self.validation, optimization) + self.result = train_policy( + self.training, self.validation, optimization, run_config=self.config + ) return self.result def evaluate(self) -> dict: @@ -160,7 +161,7 @@ def evaluate(self) -> dict: "direction": "min", "value": result.metrics[result.best_epoch]["validation_nll"], }, - "artifact_kind": "actor_only", + "artifact_kind": "sequifier_model", "actor_path": self.config.actor_path, } diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 3b03050f..96c177a9 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -1,94 +1,237 @@ -"""A causal numeric transformer actor with one categorical head and no value network.""" +"""Policy alignment adapter over Sequifier's configurable model system.""" + +import math +from collections.abc import Mapping import torch from torch import Tensor, nn +from sequifier.artifacts.model_config import resolved_config_from_model_config from sequifier.config.policy_config import CategoricalPolicyConfig -from sequifier.model.backbone import TransformerBackbone -from sequifier.objectives import CausalObjective +from sequifier.model.factory import build_transformer_network +from sequifier.model.ingestions import get_feature_embedding_dims from sequifier.rl.distributions import CategoricalActionDistribution from sequifier.rl.history import PolicyInputs +PREVIOUS_ACTION = "__policy_previous_action" +DESIRED_RETURN = "__policy_return" + + +def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu"): + """Resolve policy features through the same execution schema as forecasts. + + Actions are decoder IDs; the extra input-only ID is the episode-start token. + Returns and previous actions remain distinct from the current action target. + """ + columns = config.observation_columns + config.static_columns + [PREVIOUS_ACTION] + if config.return_conditioned: + columns.append(DESIRED_RETURN) + target = ( + "__policy_action" if config.action_column in columns else config.action_column + ) + categorical = list(config.observation_n_classes) + [PREVIOUS_ACTION, target] + real = [column for column in columns if column not in categorical] + count = len(config.action_values) + n_classes = { + **config.observation_n_classes, + PREVIOUS_ACTION: count + 1, + target: count, + } + categorical_inputs = [column for column in columns if column in categorical] + width = max(config.backbone.dim_model, len(real) + 2 * len(categorical_inputs)) + default_ingestion = { + "type": "embedding", + "output_dim": width, + "feature_embedding_dims": { + **{column: 1 for column in real}, + **get_feature_embedding_dims( + width - len(real), categorical_inputs, [], n_classes + ), + }, + } + values = { + "training_objective": "causal", + "context_length": config.context_length, + "target_offset": 1, + "backbone": {"architecture": config.backbone.model_dump(mode="json")}, + "interfaces": { + "policy": { + "policy": config.model_dump(mode="json"), + "input_columns": columns, + "target_columns": [target], + "column_data_types": { + column: "Int64" if column in categorical else "Float32" + for column in columns + [target] + }, + "categorical_columns": categorical, + "real_columns": real, + "target_column_types": {target: "categorical"}, + "n_classes": n_classes, + "target_decoder_ids": {target: list(range(count))}, + "target_global_to_decoder": {target: list(range(count))}, + "normalize_real_columns": True, + "selected_columns_statistics": { + column: {"mean": 0.0, "std": 1.0} + for column in real + if column != DESIRED_RETURN + }, + "ingestion": ( + config.ingestion.model_dump(mode="json") + if config.ingestion + else default_ingestion + ), + "decoder": ( + config.decoder.model_dump(mode="json") + if config.decoder + else { + "type": "linear", + "prediction_length": 1, + "support": 1, + } + ), + } + }, + } + return resolved_config_from_model_config(values, device=device)[0] + class CategoricalPolicy(nn.Module): - def __init__(self, config: CategoricalPolicyConfig): + """A convenience view; the network owns all trainable weights and execution.""" + + def __init__(self, config: CategoricalPolicyConfig, *, device: str = "cpu"): super().__init__() self.config = config - self.backbone = TransformerBackbone(config.backbone) - width = self.backbone.input_dim - numeric_width = len(config.observation_columns) + len(config.static_columns) + 1 - self.ingestion = nn.Linear(numeric_width, width) - self.previous_action_embedding = nn.Embedding( - len(config.action_values) + 1, width - ) - self.decoder = nn.Linear(config.backbone.dim_model, len(config.action_values)) - self.register_buffer( - "causal_attention_mask", - CausalObjective.build_attention_mask_policy(config.context_length), - persistent=False, + self.execution_config = policy_execution_config(config, device) + self.network = build_transformer_network( + self.execution_config, device=torch.device(device) + ).network + self.interface_name = "policy" + + @classmethod + def from_network(cls, network, execution_config, interface_name: str): + interface = next( + dataset.interface + for dataset in execution_config.dataset_training.values() + if dataset.model_interface == interface_name ) - self.register_buffer( - "observation_mean", torch.zeros(len(config.observation_columns)) + if interface.policy is None: + raise ValueError( + f"Model interface {interface_name!r} has no policy alignment" + ) + policy = cls.__new__(cls) + nn.Module.__init__(policy) + policy.config = interface.policy + policy.execution_config = execution_config + policy.interface_name = interface_name + policy.network = network + return policy + + @property + def interface(self): + return next( + dataset.interface + for dataset in self.execution_config.dataset_training.values() + if dataset.model_interface == self.interface_name ) - self.register_buffer( - "observation_scale", torch.ones(len(config.observation_columns)) + + @property + def device(self) -> torch.device: + return next(self.network.parameters()).device + + @property + def backbone(self): + return self.network.backbone + + def set_normalization_statistics( + self, statistics: Mapping[str, Mapping[str, float]] + ) -> None: + expected = ( + set(self.config.observation_columns + self.config.static_columns) + - self.config.observation_n_classes.keys() ) - self.register_buffer("static_mean", torch.zeros(len(config.static_columns))) - self.register_buffer("static_scale", torch.ones(len(config.static_columns))) + if set(statistics) != expected: + raise ValueError("normalization statistics must match real policy columns") + fitted = {} + for column, values in statistics.items(): + mean, std = float(values["mean"]), float(values["std"]) + if not math.isfinite(mean) or not math.isfinite(std) or std <= 0: + raise ValueError( + "policy normalization requires finite means and positive scales" + ) + fitted[column] = {"mean": mean, "std": std} + self.interface.selected_columns_statistics = fitted def set_normalization(self, observations: Tensor, static_inputs: Tensor) -> None: - """Fit once on the training episode partition; persist as model buffers.""" - for values, mean, scale in ( - (observations, self.observation_mean, self.observation_scale), - (static_inputs, self.static_mean, self.static_scale), + """Convenience API for in-memory callers; file training fits incrementally.""" + statistics = {} + for values, columns in ( + (observations, self.config.observation_columns), + (static_inputs, self.config.static_columns), ): if ( values.ndim != 2 - or values.shape[1:] != mean.shape + or values.shape[1] != len(columns) or not len(values) or not torch.isfinite(values).all() ): raise ValueError( - "normalization requires finite training rows matching policy columns" - ) - if not mean.numel(): - continue - fitted = values.to(torch.float64) - mean.copy_(fitted.mean(dim=0)) - scale.copy_(fitted.std(dim=0, unbiased=False).clamp_min(1e-6)) - if not torch.isfinite(mean).all() or not torch.isfinite(scale).all(): - raise ValueError( - "training normalization exceeds the model's numeric range" + "normalization requires finite rows matching policy columns" ) + for index, column in enumerate(columns): + if column not in self.config.observation_n_classes: + fitted = values[:, index].to(device="cpu", dtype=torch.float64) + statistics[column] = { + "mean": float(fitted.mean()), + "std": float(fitted.std(unbiased=False).clamp_min(1e-6)), + } + self.set_normalization_statistics(statistics) - def forward(self, inputs: PolicyInputs) -> Tensor: + def model_inputs( + self, inputs: PolicyInputs + ) -> tuple[dict[str, Tensor], dict[str, Tensor]]: + """Translate aligned raw histories into standard named model tensors.""" self.validate_inputs(inputs) - mask = inputs.attention_valid_mask - batch, context = mask.shape - observations = ( - inputs.observations - self.observation_mean - ) / self.observation_scale - static = (inputs.static_inputs - self.static_mean) / self.static_scale - static = static[:, None, :].expand(batch, context, -1) + features = { + column: inputs.observations[:, :, index] + for index, column in enumerate(self.config.observation_columns) + } + features.update( + { + column: inputs.static_inputs[:, index, None].expand_as( + inputs.previous_actions + ) + for index, column in enumerate(self.config.static_columns) + } + ) + for column, count in self.config.observation_n_classes.items(): + values = features[column] + if ( + (values != values.round()).any() + or (values < 0).any() + or (values >= count).any() + ): + raise ValueError( + f"{column}: observations must be categorical IDs in [0, {count})" + ) + features[column] = values.long() + for column, statistics in self.interface.selected_columns_statistics.items(): + features[column] = (features[column] - statistics["mean"]) / ( + statistics["std"] + 1e-9 + ) + features[PREVIOUS_ACTION] = inputs.previous_actions if self.config.return_conditioned: assert inputs.desired_returns is not None - returns = inputs.desired_returns / self.config.returns.scale - else: - # Same parameters and architecture as RC; the return channel is disabled. - returns = observations.new_zeros((batch, context)) - numeric = torch.cat((observations, static, returns.unsqueeze(-1)), dim=-1) - encoded = torch.tanh( - self.ingestion(numeric) - + self.previous_action_embedding(inputs.previous_actions) - ) - # As in ModelInterfaceModule, combine the objective's causal mask with - # finite padding penalties so left-padded query rows remain well-defined. - attention_mask = self.causal_attention_mask[None, None, :, :] + torch.zeros( - batch, 1, 1, context, dtype=encoded.dtype, device=encoded.device - ).masked_fill(~mask[:, None, None, :], torch.finfo(encoded.dtype).min) - hidden = self.backbone(encoded, attention_mask) - logits = self.decoder(hidden) - return logits.masked_fill(~mask[:, :, None], 0.0) + features[DESIRED_RETURN] = ( + inputs.desired_returns / self.config.returns.scale + ) + return features, {"attention_valid_mask": inputs.attention_valid_mask} + + def forward(self, inputs: PolicyInputs) -> Tensor: + features, metadata = self.model_inputs(inputs) + logits = self.network( + features, metadata, interface_name=self.interface_name + ).logits[self.interface.target_columns[0]] + return logits.masked_fill(~inputs.attention_valid_mask[:, :, None], 0.0) def distribution(self, inputs: PolicyInputs) -> CategoricalActionDistribution: return CategoricalActionDistribution(self(inputs)) @@ -139,11 +282,13 @@ def validate_inputs(self, inputs: PolicyInputs) -> None: ) numeric.append(inputs.desired_returns) if any( - value.device.type != "cpu" + value.device != self.device or value.dtype != torch.float32 or not torch.isfinite(value).all() for value in numeric ): - raise ValueError("numeric policy inputs must be finite CPU float32 tensors") - if mask.device.type != "cpu" or actions.device.type != "cpu": - raise ValueError("numeric policies currently support CPU execution only") + raise ValueError( + "numeric policy inputs must be finite float32 tensors on the model device" + ) + if mask.device != self.device or actions.device != self.device: + raise ValueError("policy masks and actions must be on the model device") diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index 22325a15..fcdab143 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -55,7 +55,9 @@ def reset(self, request: ResetRequest) -> None: self.return_trace = [self.budget] if spec.return_conditioned else [] self.previous_action_trace = [] self.log_probabilities = [] - self.generator = torch.Generator().manual_seed(request.seed) + self.generator = torch.Generator( + device=self.policy.device if self.policy.device.type == "cuda" else "cpu" + ).manual_seed(request.seed) self.pending = None self.ended = False self.active = True @@ -79,7 +81,9 @@ def __call__(self, history: SeedHistory) -> dict[str, float]: history.static_inputs, ) with torch.no_grad(): - distribution = CategoricalActionDistribution(self.policy(inputs)[:, -1, :]) + distribution = CategoricalActionDistribution( + self.policy(inputs.to(self.policy.device))[:, -1, :] + ) action = ( distribution.mode() if self.deterministic @@ -143,7 +147,7 @@ def collect_policy_rollout( def validate_history(history): if config.mode != "policy": raise ValueError("categorical execution requires rollout mode: policy") - horizon = spec.returns.finite_horizon + horizon = spec.returns.finite_horizon if spec.return_conditioned else None if horizon is not None and config.max_episode_steps != horizon: raise ValueError( "finite-horizon policy execution requires max_episode_steps equal to its return horizon" @@ -162,7 +166,7 @@ def validate_history(history): else None, history.static_inputs, ) - policy.validate_inputs(inputs) + policy.validate_inputs(inputs.to(policy.device)) was_training = policy.training policy.eval() diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index c1ddd37d..61d554ec 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -1,19 +1,37 @@ """Supervised masked action likelihood using Sequifier's optimizer transaction.""" -from copy import deepcopy from dataclasses import dataclass import torch from loguru import logger from torch.utils.data import DataLoader +from sequifier.artifacts.model_artifact import ( + build_model_artifact, + load_execution_model, +) +from sequifier.artifacts.run_checkpoint import ( + RUN_CHECKPOINT_FORMAT_VERSION, + DistributedRandomState, + IntegrationState, + LoaderState, + RunCheckpoint, + RunCheckpointStore, + select_run_checkpoint, +) from sequifier.config.policy_config import PolicyTrainingConfig +from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig from sequifier.integration.callbacks import IntegrationManager from sequifier.integration.contexts import StepIdentity from sequifier.rl.distributions import masked_action_likelihood from sequifier.rl.policy import CategoricalPolicy +from sequifier.runtime.random_state import RandomState, RandomStateManager from sequifier.training.optimization import OptimizationRuntime, UpdatePolicy -from sequifier.trajectory.dataset import TrajectoryDataset, collate_policy_examples +from sequifier.trajectory.dataset import ( + EpisodeWindowSampler, + TrajectoryDataset, + collate_policy_examples, +) @dataclass(frozen=True) @@ -30,6 +48,9 @@ def train_policy( training: TrajectoryDataset, validation: TrajectoryDataset, config: PolicyTrainingConfig, + *, + run_config: ResolvedPolicyTrainConfig | None = None, + integrations: IntegrationManager | None = None, ) -> PolicyTrainingResult: if not len(training) or not len(validation): raise ValueError("policy training and validation partitions must be nonempty") @@ -40,46 +61,140 @@ def train_policy( raise ValueError( "training and validation must share the policy/return definition" ) - train_ids = tuple(episode.episode_id for episode in training.episodes) - validation_ids = tuple(episode.episode_id for episode in validation.episodes) + if run_config is not None: + phase = run_config.training_plan.phases[0] + expected = PolicyTrainingConfig( + **run_config.optimization.model_dump(), + epochs=phase.epochs, + seed=run_config.seed, + ) + if config != expected or training.config != run_config.policies[phase.policy]: + raise ValueError( + "policy training arguments must match the resolved run configuration" + ) + train_ids = tuple(training.episode_ids) + validation_ids = tuple(validation.episode_ids) if set(train_ids) & set(validation_ids): raise ValueError( "training and validation partitions must not share episode IDs" ) - # Keep initialization and batch order equal across the RC and BC comparison. - with torch.random.fork_rng(devices=[]): + # Seed model initialization and data order independently. + device = run_config.device if run_config is not None else "cpu" + cuda_index = torch.device(device).index + cuda_devices = ( + [cuda_index if cuda_index is not None else torch.cuda.current_device()] + if device.startswith("cuda") + else [] + ) + with torch.random.fork_rng(devices=cuda_devices): torch.manual_seed(config.seed) - policy = CategoricalPolicy(training.config) - policy.set_normalization(*training.normalization_rows()) + policy = CategoricalPolicy(training.config, device=device) + policy.set_normalization_statistics(training.normalization_statistics()) + resume_path = ( + select_run_checkpoint(run_config) if run_config is not None else None + ) + if ( + run_config is not None + and run_config.initialize_from is not None + and resume_path is None + ): + source, source_config, _ = load_execution_model( + run_config.initialize_from, device=device + ) + if ( + source_config.model.backbone.architecture + != policy.execution_config.model.backbone.architecture + ): + raise ValueError( + "initialize_from requires the same backbone architecture" + ) + policy.backbone.load_state_dict(source.backbone.state_dict(), strict=True) + del source optimization = OptimizationRuntime.create( - config, "cpu", policy.parameters(), phase_epochs=config.epochs + config, device, policy.network.parameters(), phase_epochs=config.epochs ) - integrations = IntegrationManager() + integrations = integrations or IntegrationManager() generator = torch.Generator().manual_seed(config.seed) train_loader = DataLoader( training, batch_size=config.batch_size, - shuffle=True, + sampler=EpisodeWindowSampler(training, generator), + num_workers=config.num_workers, generator=generator, collate_fn=collate_policy_examples, ) validation_loader = DataLoader( - validation, batch_size=config.batch_size, collate_fn=collate_policy_examples + validation, + batch_size=config.batch_size, + collate_fn=collate_policy_examples, + num_workers=config.num_workers, ) metrics = [] best_loss = float("inf") best_epoch = 0 best_weights = None global_batch = 0 + first_epoch = 0 + random_state = RandomStateManager(torch.device(device)) + store = ( + RunCheckpointStore(run_config, run_config.model_name) + if run_config is not None + else None + ) + if resume_path is not None: + assert store is not None and run_config is not None + checkpoint = store.load(resume_path) + previous = checkpoint.training_config + if not isinstance(previous, ResolvedPolicyTrainConfig): + raise ValueError("policy resume requires a policy training checkpoint") + + if _resume_contract(previous) != _resume_contract(run_config): + raise ValueError( + "policy resume configuration changes the training contract" + ) + expected_data = _loader_contract(training, validation) + if checkpoint.loader_state.parts != expected_data: + raise ValueError( + "policy resume trajectory content or episode order changed" + ) + policy.network.load_state_dict( + checkpoint.model.model_state_dict, strict=True + ) + saved_interface = checkpoint.model.model_config.values["interfaces"][ + policy.interface_name + ] + policy.set_normalization_statistics( + saved_interface["selected_columns_statistics"] + ) + optimization.optimizer.load_state_dict(checkpoint.optimization.optimizer) + optimization.load_non_optimizer_state(checkpoint.optimization) + state = checkpoint.run_state + first_epoch = state["next_epoch"] + metrics = list(state["metrics"]) + best_loss, best_epoch, best_weights = ( + state["best_loss"], + state["best_epoch"], + state["best_weights"], + ) + global_batch = state["global_batch"] + generator.set_state(state["data_generator"]) + random_state.restore( + RandomState.from_state_dict(checkpoint.random_state.states[0]) + ) + if integrations.enabled: + integrations.load_state_dict(checkpoint.integration_state.values) + del checkpoint, state optimization.optimizer.zero_grad(set_to_none=True) - for epoch in range(config.epochs): + for epoch in range(first_epoch, config.epochs): policy.train() total_loss = 0.0 decisions = 0 for batch_index, batch in enumerate(train_loader): - distribution = policy.distribution(batch.inputs) + distribution = policy.distribution(batch.inputs.to(device)) loss = masked_action_likelihood( - distribution, batch.actions, batch.action_valid_mask + distribution, + batch.actions.to(device), + batch.action_valid_mask.to(device), ) if not torch.isfinite(loss): raise ValueError("policy action likelihood became non-finite") @@ -96,9 +211,12 @@ def train_policy( 0, 1, ) - optimization.accumulate(loss, identity, integrations, policy) + loss = loss + policy.network.regularization_loss(policy.interface_name) + if not torch.isfinite(loss): + raise ValueError("policy regularization became non-finite") + optimization.accumulate(loss, identity, integrations, policy.network) result = optimization.complete_step( - policy, identity, integrations, UpdatePolicy() + policy.network, identity, integrations, UpdatePolicy() ) if result.stop_requested or not result.applied: raise RuntimeError( @@ -111,9 +229,9 @@ def train_policy( with torch.no_grad(): for batch in validation_loader: loss = masked_action_likelihood( - policy.distribution(batch.inputs), - batch.actions, - batch.action_valid_mask, + policy.distribution(batch.inputs.to(device)), + batch.actions.to(device), + batch.action_valid_mask.to(device), ) if not torch.isfinite(loss): raise ValueError( @@ -138,13 +256,51 @@ def train_policy( if validation_loss < best_loss: best_loss = validation_loss best_epoch = epoch - best_weights = deepcopy(policy.state_dict()) + best_weights = { + name: value.detach().cpu().clone() + for name, value in policy.network.state_dict().items() + } optimization.step_scheduler(validation_loss) + if store is not None and run_config is not None: + checkpoint = RunCheckpoint( + format_version=RUN_CHECKPOINT_FORMAT_VERSION, + model=build_model_artifact( + policy.network, + policy.execution_config, + state_dict={ + name: value.detach().cpu().clone() + for name, value in policy.network.state_dict().items() + }, + ), + optimization=optimization.state_dict(), + run_state={ + "algorithm": "return_conditioned", + "next_epoch": epoch + 1, + "metrics": metrics, + "best_loss": best_loss, + "best_epoch": best_epoch, + "best_weights": best_weights, + "global_batch": global_batch, + "data_generator": generator.get_state(), + }, + random_state=DistributedRandomState( + states=(random_state.capture_local().state_dict(),) + ), + loader_state=LoaderState( + parts=_loader_contract(training, validation) + ), + integration_state=IntegrationState( + values=integrations.state_dict() + ), + training_config=run_config, + ) + store.save(checkpoint, "latest") + del checkpoint if best_weights is None: raise ValueError( "policy training did not produce finite validation likelihood" ) - policy.load_state_dict(best_weights, strict=True) + policy.network.load_state_dict(best_weights, strict=True) policy.eval() return PolicyTrainingResult( policy, @@ -154,3 +310,34 @@ def train_policy( train_ids, validation_ids, ) + + +def _loader_contract( + training: TrajectoryDataset, validation: TrajectoryDataset +) -> dict: + return { + "policy": { + "training": training.content_keys, + "validation": validation.content_keys, + "training_episode_ids": training.episode_ids, + "validation_episode_ids": validation.episode_ids, + } + } + + +def _resume_contract(config: ResolvedPolicyTrainConfig) -> dict: + # Device, output paths, workers, and epoch budget may change; learning semantics may not. + values = config.model_dump(mode="json") + for key in ( + "device", + "actor_path", + "report_path", + "initialize_from", + "project_root", + ): + values.pop(key) + for phase in values["training_plan"]["phases"]: + phase.pop("epochs") + for key in ("resume", "num_workers", "episode_cache_size"): + values["optimization"].pop(key) + return values diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index e2e2f484..f93c2c8e 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -6,6 +6,7 @@ PolicyTrainingConfig, ReturnDefinition, ) +from sequifier.config.policy_infer_config import PolicyInferenceConfig from sequifier.config.policy_train_config import ( PolicyTrainConfig, ResolvedPolicyTrainConfig, @@ -35,6 +36,7 @@ from sequifier.trajectory.returns import remaining_return, return_to_go __all__ = [ + "PolicyInferenceConfig", "PolicyTrainConfig", "PolicyOptimizationConfig", "TrajectorySourceConfig", diff --git a/src/sequifier/train.py b/src/sequifier/train.py index f0720c69..1f7e8ada 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -15,10 +15,7 @@ from loguru import logger as loguru_logger # noqa: E402 from torch import Tensor, nn # noqa: E402 -from sequifier.artifacts.model_artifact import ( # noqa: E402 - load_model_artifact, - load_weights_from_run_checkpoint, -) +from sequifier.artifacts.model_artifact import load_execution_model # noqa: E402 from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig # noqa: E402 from sequifier.config.train_config import ( # noqa: E402 ResolvedSequifierConfig as TrainModel, @@ -178,6 +175,8 @@ def run_training( ) from sequifier.rl.learner import run_policy_training + configure_logger(config.project_root, config.model_name, 0) + configure_determinism(config.seed) run_policy_training(config) return if not isinstance(config, TrainModel): @@ -241,22 +240,10 @@ def load_inference_model( """Load only the new portable-model or exact-run artifact formats.""" del training_config_path - payload = torch.load(model_path, map_location="cpu", weights_only=False) - artifact_type = payload.get("artifact_type") if isinstance(payload, dict) else None interface_name = args_config.get("model_interface") or args_config.get("dataset") - if artifact_type == "sequifier_model": - network, config, _ = load_model_artifact( - model_path, device=device, interface_name=interface_name - ) - elif artifact_type == "sequifier_run_checkpoint": - network, config, _ = load_weights_from_run_checkpoint( - model_path, device=device, interface_name=interface_name - ) - else: - raise ValueError( - "Unsupported PyTorch artifact. Sequifier accepts only the current " - "portable model or run-checkpoint format." - ) + network, config, _ = load_execution_model( + model_path, device=device, interface_name=interface_name + ) if interface_name is None: if len(network.interfaces) != 1: raise ValueError( diff --git a/src/sequifier/trajectory/dataset.py b/src/sequifier/trajectory/dataset.py index b737c418..fe0ee3de 100644 --- a/src/sequifier/trajectory/dataset.py +++ b/src/sequifier/trajectory/dataset.py @@ -3,16 +3,23 @@ import hashlib import json import random +from bisect import bisect_right +from collections import OrderedDict from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import Literal, overload import torch -from torch.utils.data import Dataset +from torch.utils.data import Dataset, Sampler from sequifier.config.policy_config import CategoricalPolicyConfig -from sequifier.rl.history import PolicyInputs, action_index, build_policy_inputs +from sequifier.rl.history import ( + PolicyInputs, + action_index, + build_policy_inputs, + collate_policy_inputs, +) from sequifier.trajectory.returns import labeled_episode_length, return_to_go from sequifier.trajectory.schema import Trajectory, validate_trajectory from sequifier.trajectory.store import load_trajectory @@ -30,39 +37,8 @@ class PolicyExample: def collate_policy_examples(examples: Sequence[PolicyExample]) -> PolicyExample: if not examples: raise ValueError("cannot collate an empty policy batch") - conditioned = examples[0].inputs.desired_returns is not None - if any( - (example.inputs.desired_returns is not None) != conditioned - for example in examples - ): - raise ValueError("policy batches cannot mix conditioning modes") - returns = ( - torch.cat( - [ - example.inputs.desired_returns - for example in examples - if example.inputs.desired_returns is not None - ] - ) - if conditioned - else None - ) return PolicyExample( - inputs=PolicyInputs( - observations=torch.cat( - [example.inputs.observations for example in examples] - ), - previous_actions=torch.cat( - [example.inputs.previous_actions for example in examples] - ), - attention_valid_mask=torch.cat( - [example.inputs.attention_valid_mask for example in examples] - ), - static_inputs=torch.cat( - [example.inputs.static_inputs for example in examples] - ), - desired_returns=returns, - ), + inputs=collate_policy_inputs([example.inputs for example in examples]), actions=torch.cat([example.actions for example in examples]), action_valid_mask=torch.cat( [example.action_valid_mask for example in examples] @@ -74,13 +50,42 @@ def collate_policy_examples(examples: Sequence[PolicyExample]) -> PolicyExample: ) +class EpisodeCollection(Sequence[Trajectory]): + """Keep path-backed episodes on disk, with a bounded per-worker cache.""" + + def __init__(self, sources, cache_size: int): + self.sources = list(sources) + self.cache_size = cache_size + self.cache = OrderedDict() + + def __len__(self): + return len(self.sources) + + @overload + def __getitem__(self, index: int) -> Trajectory: ... + + @overload + def __getitem__(self, index: slice) -> Sequence[Trajectory]: ... + + def __getitem__(self, index: int | slice) -> Trajectory | Sequence[Trajectory]: + if isinstance(index, slice): + return [self[i] for i in range(*index.indices(len(self)))] + source = self.sources[index] + if isinstance(source, Trajectory): + return source + if index not in self.cache: + self.cache[index] = load_trajectory(source) + if len(self.cache) > self.cache_size: + self.cache.popitem(last=False) + self.cache.move_to_end(index) + return self.cache[index] + + class TrajectoryDataset(Dataset): - """One labeled decision per example; context contains only its causal prefix. + """One labeled decision per example with a bounded causal context. - Context positions have observations and previous actions, while only the - current (last) position contributes action likelihood. No future observations, - current actions, or current rewards are arguments to the history builder. - Return-to-go is an intentional conditioning label computed from rewards. + File-backed corpora retain only episode metadata and a small episode cache. + Return labels are cached per episode, never reconstructed for every prefix. """ def __init__( @@ -89,21 +94,27 @@ def __init__( config: CategoricalPolicyConfig, *, action_semantics: Literal["policy", "executed_predictions"] = "policy", + episode_cache_size: int = 8, ): if action_semantics not in {"policy", "executed_predictions"}: raise ValueError( "trajectory action_semantics must be policy or executed_predictions" ) + if episode_cache_size < 1: + raise ValueError("episode_cache_size must be positive") self.config = config self.action_semantics: Literal["policy", "executed_predictions"] = ( action_semantics ) - self.episodes: list[Trajectory] = [] - self.excluded_episode_ids: list[str] = [] - self.lengths: list[int] = [] - self.returns: list[list[float]] = [] - self.actions: list[list[int]] = [] - self.index: list[tuple[int, int]] = [] + self.excluded_episode_ids = [] + self.all_episode_ids = [] + self.episode_ids = [] + self.content_keys = [] + self.lengths = [] + self.cumulative_lengths = [] + self.episode_cache_size = episode_cache_size + self._labels = OrderedDict() + eligible = [] seen = set() for source in sources: episode = ( @@ -114,6 +125,7 @@ def __init__( if episode.episode_id in seen: raise ValueError(f"duplicate episode ID {episode.episode_id!r}") seen.add(episode.episode_id) + self.all_episode_ids.append(episode.episode_id) if ( episode.format != "sequifier.numeric-policy-trajectory" and action_semantics != "executed_predictions" @@ -138,48 +150,120 @@ def __init__( raise ValueError( "policy trajectories must contain the configured single action head" ) - length = labeled_episode_length(episode, config.returns) + length = ( + labeled_episode_length(episode, config.returns) + if config.return_conditioned + else len(episode.transitions) + ) if length is None: self.excluded_episode_ids.append(episode.episode_id) continue - actions = [ + for transition in episode.transitions[:length]: action_index(transition.predictions[config.action_column], config) - for transition in episode.transitions[:length] - ] - returns = return_to_go( - [transition.reward for transition in episode.transitions[:length]], - config.returns.gamma, - ) - episode_index = len(self.episodes) - self.episodes.append(episode) + eligible.append(source) + self.episode_ids.append(episode.episode_id) + self.content_keys.append(episode_content_key(episode)) self.lengths.append(length) - self.actions.append(actions) - self.returns.append(returns) - self.index.extend((episode_index, step) for step in range(length)) + self.cumulative_lengths.append( + length + (self.cumulative_lengths[-1] if self.cumulative_lengths else 0) + ) + self.episodes = EpisodeCollection(eligible, episode_cache_size) def __len__(self): - return len(self.index) + return self.cumulative_lengths[-1] if self.cumulative_lengths else 0 + + def _episode_labels(self, index: int): + if index not in self._labels: + transitions = self.episodes[index].transitions[: self.lengths[index]] + actions = [ + action_index( + transition.predictions[self.config.action_column], self.config + ) + for transition in transitions + ] + returns = ( + return_to_go( + [transition.reward for transition in transitions], + self.config.returns.gamma, + ) + if self.config.return_conditioned + else None + ) + self._labels[index] = actions, returns + if len(self._labels) > self.episode_cache_size: + self._labels.popitem(last=False) + self._labels.move_to_end(index) + return self._labels[index] def __getitem__(self, index: int) -> PolicyExample: - episode_index, step = self.index[index] + if index < 0: + index += len(self) + if not 0 <= index < len(self): + raise IndexError(index) + episode_index = bisect_right(self.cumulative_lengths, index) + step = index - ( + self.cumulative_lengths[episode_index - 1] if episode_index else 0 + ) episode = self.episodes[episode_index] + action_ids, returns = self._episode_labels(episode_index) + start = max(0, step + 1 - self.config.context_length) + previous = ( + [len(self.config.action_values)] if start == 0 else [] + ) + action_ids[max(0, start - 1) : step] inputs = build_policy_inputs( self.config, - [transition.observation for transition in episode.transitions[: step + 1]], - [len(self.config.action_values)] + self.actions[episode_index][:step], - self.returns[episode_index][: step + 1] - if self.config.return_conditioned - else None, + [ + transition.observation + for transition in episode.transitions[start : step + 1] + ], + previous, + returns[start : step + 1] if returns is not None else None, episode.seed.static_inputs, ) - # Labels have their own mask; they are never passed to policy.forward(). actions = torch.zeros((1, self.config.context_length), dtype=torch.int64) - actions[0, -1] = self.actions[episode_index][step] + actions[0, -1] = action_ids[step] valid = torch.zeros_like(actions, dtype=torch.bool) valid[0, -1] = True return PolicyExample(inputs, actions, valid, (episode.episode_id,), (step,)) + def normalization_statistics(self) -> dict[str, dict[str, float]]: + """Combine episode moments without materializing all training rows.""" + columns = self.config.observation_columns + self.config.static_columns + moments = { + column: (0, 0.0, 0.0) + for column in columns + if column not in self.config.observation_n_classes + } + for episode, length in zip(self.episodes, self.lengths): + for column in moments: + values = ( + [ + transition.observation[column] + for transition in episode.transitions[:length] + ] + if column in self.config.observation_columns + else [episode.seed.static_inputs[column]] + ) + fitted = torch.tensor(values, dtype=torch.float64) + count, mean, m2 = moments[column] + size = len(values) + batch_mean = float(fitted.mean()) + delta = batch_mean - mean + total = count + size + moments[column] = ( + total, + mean + delta * size / total, + m2 + + float(((fitted - batch_mean) ** 2).sum()) + + delta**2 * count * size / total, + ) + return { + column: {"mean": mean, "std": max((m2 / count) ** 0.5, 1e-6)} + for column, (count, mean, m2) in moments.items() + } + def normalization_rows(self) -> tuple[torch.Tensor, torch.Tensor]: + """Materialized convenience API retained for Python callers.""" observations = [ [ transition.observation[column] @@ -200,6 +284,27 @@ def normalization_rows(self) -> tuple[torch.Tensor, torch.Tensor]: ) +class EpisodeWindowSampler(Sampler): + """Shuffle episodes and their decisions while keeping disk access local.""" + + def __init__(self, dataset: TrajectoryDataset, generator: torch.Generator): + self.dataset = dataset + self.generator = generator + + def __len__(self): + return len(self.dataset) + + def __iter__(self): + for episode in torch.randperm( + len(self.dataset.lengths), generator=self.generator + ).tolist(): + offset = self.dataset.cumulative_lengths[episode - 1] if episode else 0 + for step in torch.randperm( + self.dataset.lengths[episode], generator=self.generator + ).tolist(): + yield offset + step + + def episode_content_key(episode: Trajectory) -> str: """Group repeated demonstrations independently of generated episode IDs.""" values = episode.model_dump(mode="json") @@ -222,9 +327,13 @@ def split_by_episode( "episode splitting requires at least two eligible episodes and validation_fraction in (0,1)" ) groups = {} - for episode in dataset.episodes: - key = episode_content_key(episode) if group_identical else episode.episode_id - groups.setdefault(key, []).append(episode) + for index, source in enumerate(dataset.episodes.sources): + key = ( + dataset.content_keys[index] + if group_identical + else dataset.episode_ids[index] + ) + groups.setdefault(key, []).append(source) partitions = list(groups.values()) if len(partitions) < 2: raise ValueError("splitting requires at least two distinct trajectory groups") @@ -234,9 +343,15 @@ def split_by_episode( training = [episode for group in partitions[count:] for episode in group] return ( TrajectoryDataset( - training, dataset.config, action_semantics=dataset.action_semantics + training, + dataset.config, + action_semantics=dataset.action_semantics, + episode_cache_size=dataset.episode_cache_size, ), TrajectoryDataset( - validation, dataset.config, action_semantics=dataset.action_semantics + validation, + dataset.config, + action_semantics=dataset.action_semantics, + episode_cache_size=dataset.episode_cache_size, ), ) From 8995be260904c01fce37c9152f6966413ca1d4f0 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 14:28:18 +0100 Subject: [PATCH 06/71] unify --- README.md | 2 +- documentation/configs/preprocess.md | 14 + documentation/configs/train.md | 18 + .../configs/import-policy-sequences.yaml | 11 + .../configs/train-policy-from-table.yaml | 56 ++ .../configs/train-return-conditioned.yaml | 11 +- ...nforcement-learning-implementation-plan.md | 12 + documentation/policy-learning.md | 222 ++++- release-notes/unreleased-policy-training.md | 16 + src/sequifier/artifacts/model_config.py | 21 +- src/sequifier/config/identifiers.py | 12 + src/sequifier/config/policy_config.py | 70 +- src/sequifier/config/policy_infer_config.py | 1 + src/sequifier/config/policy_train_config.py | 255 +++++- src/sequifier/config/preprocess_config.py | 9 +- .../config/sequence_import_config.py | 23 + src/sequifier/config/train_config.py | 102 +-- src/sequifier/config/training_plan.py | 156 ++++ src/sequifier/consolidated-docs.md | 256 +++++- src/sequifier/io/batch.py | 92 +++ src/sequifier/io/policy_dataset.py | 466 +++++++++++ src/sequifier/io/policy_window.py | 158 ++++ src/sequifier/io/preprocessing.py | 121 +++ src/sequifier/io/sequence.py | 147 ++++ src/sequifier/io/values.py | 49 ++ src/sequifier/make.py | 6 +- src/sequifier/objectives.py | 33 +- src/sequifier/preprocess.py | 124 +-- src/sequifier/rl/history.py | 118 +-- src/sequifier/rl/inference.py | 22 +- src/sequifier/rl/learner.py | 81 +- src/sequifier/rl/policy.py | 196 +++-- src/sequifier/rl/runtime.py | 7 +- src/sequifier/rl/training.py | 84 +- src/sequifier/rl_api.py | 21 + src/sequifier/rollout/contracts.py | 16 +- src/sequifier/rollout/coordinator.py | 22 +- src/sequifier/rollout/history.py | 35 +- src/sequifier/trajectory/dataset.py | 367 +-------- src/sequifier/trajectory/importers.py | 762 ++++++++++++++++++ src/sequifier/trajectory/schema.py | 15 +- 41 files changed, 3220 insertions(+), 989 deletions(-) create mode 100644 documentation/example_inputs/configs/import-policy-sequences.yaml create mode 100644 documentation/example_inputs/configs/train-policy-from-table.yaml create mode 100644 src/sequifier/config/identifiers.py create mode 100644 src/sequifier/config/sequence_import_config.py create mode 100644 src/sequifier/config/training_plan.py create mode 100644 src/sequifier/io/policy_dataset.py create mode 100644 src/sequifier/io/policy_window.py create mode 100644 src/sequifier/io/preprocessing.py create mode 100644 src/sequifier/io/sequence.py create mode 100644 src/sequifier/io/values.py create mode 100644 src/sequifier/trajectory/importers.py diff --git a/README.md b/README.md index 443d3cf6..3be2ce79 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ There are seven standalone commands within sequifier: `make`, `preprocess`, `tra | --- | --- | | `make` | Create a new sequifier project with config templates. | | `preprocess` | Convert input data into fixed-length subsequences. | -| `train` | Train a supervised model or a trajectory-backed categorical policy. | +| `train` | Train a supervised model or a categorical policy from sequence tables or rollout recordings. | | `infer` | Generate predictions, probabilities, embeddings, or policy actions and rollouts. | | `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | | `visualize-training` | Generate interactive HTML plots from structured training metrics. | diff --git a/documentation/configs/preprocess.md b/documentation/configs/preprocess.md index 26a33274..56541404 100644 --- a/documentation/configs/preprocess.md +++ b/documentation/configs/preprocess.md @@ -507,3 +507,17 @@ layout properties, so unused stored layouts/features can be added independently. To consume a named layout during training, configure a `depth_transformer` ingestion branch whose `layout` names this entry and whose `columns` are selected features from it. See [the training guide](train.md#depth-encoders-and-nested-composites). + +## Import sequence demonstrations + +`operation: import_sequences` selects the explicit policy-data import operation. +It accepts a `source`, a model-independent `policy` data schema, an `output_path` +(new dataset directory) and optional `episodes_per_shard` (default 64). Sources +can be raw CSV/Parquet tables, existing scalar prepared histories with their +metadata, or JSON rollout recordings. Import preserves domain values and missing +transition information; it does not fit vocabularies/statistics or create padded +windows. Raw tables can also be used directly by policy training without this step. + +The operation atomically publishes Parquet shards and a checksummed manifest. +See [policy learning](../policy-learning.md#optional-committed-import) and +[import-policy-sequences.yaml](../example_inputs/configs/import-policy-sequences.yaml). diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 0c4e5f53..d2275d7a 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -782,3 +782,21 @@ IQL/PPO remain planned. Policy models use the shared configurable network and portable artifacts. They support CPU/CUDA/MPS, backbone transfer with `initialize_from`, bounded file-backed episode loading, and epoch-boundary resume through `optimization.resume`. + +## Policy sequence sources and shared authoring + +Policy phases accept ordinary CSV/Parquet demonstrations with `sequenceId` and +`itemPosition`. Use `data_sources` and phase `sources: [{source: demonstrations}]`; +the previous `trajectory_sources` spellings remain accepted. BC does not require +rewards or successor states. Return-conditioned training requires rewards and a +known return endpoint. JSON rollout logs are adapted into the same sequence data. + +Policy training can use the ordinary `model.backbone`, `model.interfaces` and +`dataset_training..parts` authoring patterns. Put the model-independent +observation/action schema in an interface's `policy` field and a sequence source +in each dataset part. The algorithm supplies action NLL. `global_training` can +supply the policy optimization settings and `context_length`, with +`training_objective: policy`; the compact `policies`/`optimization` surface remains +supported. One policy phase is currently supported. See the complete +[table training example](../example_inputs/configs/train-policy-from-table.yaml) +and [policy learning guide](../policy-learning.md). diff --git a/documentation/example_inputs/configs/import-policy-sequences.yaml b/documentation/example_inputs/configs/import-policy-sequences.yaml new file mode 100644 index 00000000..77a35285 --- /dev/null +++ b/documentation/example_inputs/configs/import-policy-sequences.yaml @@ -0,0 +1,11 @@ +operation: import_sequences +project_root: . +source: + paths: [data/demonstrations.csv] +policy: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + return_conditioned: false +output_path: data/imported-demonstrations +episodes_per_shard: 64 diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml new file mode 100644 index 00000000..d7703648 --- /dev/null +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -0,0 +1,56 @@ +project_root: . +model_name: table-policy +device: cpu +seed: 1010 + +# The ordinary model component schemas are also used for policy training. +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128} + positional_encoding_scope: global + interfaces: + actor: + input_columns: [position, velocity] + target_columns: [action] + policy: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: + position: 1 + velocity: 1 + __policy_previous_action: 6 + decoder: {type: linear, prediction_length: 1, support: 1} + +dataset_training: + demonstrations: + model_interface: actor + parts: + recorded: + paths: [data/demonstrations.parquet] + # BC requires sequenceId, itemPosition, position, velocity and action. + # Rewards, boundaries and simulator recording metadata are optional. + +global_training: + training_objective: policy + context_length: 16 + batch_size: 64 + learning_rate: 0.005 + validation_fraction: 0.2 + +training_plan: + phases: + - name: behavior_cloning + algorithm: return_conditioned + policy: actor + epochs: 80 + sources: [{source: demonstrations.recorded}] diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index 14e55b34..cb595fc1 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -3,13 +3,12 @@ model_name: policy device: cpu seed: 1010 -trajectory_sources: +data_sources: demonstrations: - # Paths inside the manifest resolve relative to the manifest directory. - manifest: data/trajectories/manifest.json - action_semantics: policy + # One row per current observation/executed action/resulting reward. + paths: [data/demonstrations.parquet] -# Set observation columns and action values to match your recorded trajectories. +# Set observation columns and action values to match your table. policies: actor: observation_columns: [OBSERVATION_COLUMN] @@ -32,7 +31,7 @@ training_plan: - name: offline_policy algorithm: return_conditioned epochs: 80 - trajectory_sources: [demonstrations] + sources: [{source: demonstrations}] policy: actor optimization: diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md index 5be2c7fd..23bf55ae 100644 --- a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md +++ b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md @@ -35,6 +35,18 @@ caching, context-sized windows, and BC eligibility without return labels are implemented. The broader stages below remain planned. No tests or training runs were executed for this follow-up. +Representation unification follow-up: raw CSV/Parquet demonstrations, scalar +prepared histories and JSON recordings now adapt into a common columnar sequence +representation with optional transition capabilities. Policy views use explicit +same-position causal alignment, named inputs, standard sequence batches, shared +vocabulary/statistics fitting and supervised loss calculation. Shared model, +dataset-part and phase authoring is available alongside the compact compatibility +surface. Preprocess has an explicit optional raw sequence import operation with +atomic Parquet/manifest publication. BC requires no synthetic rewards, successors, +bindings or seed histories. Policy image/depth inputs, mixed-algorithm execution, +IQL/PPO and their algorithm-specific buffers remain planned. No tests or training +runs were written or executed for this follow-up, as requested. + Deliver this as one coordinated update, built and verified in the three dependency-ordered stages below. Completion means all three stages work together; stage boundaries are engineering checkpoints, not separate proposed releases. This document specifies new behavior and proposed API names, not functionality already present. ## 1. Outcome and scope diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index b9145e7d..aa4b11cc 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -6,47 +6,181 @@ A policy adds a decision-alignment contract to a model interface: at time `t`, the inputs contain the current observation, the **previous** action, and, optionally, a desired return. The current action is the supervised target. -## Train from trajectories +## Train from ordinary sequence tables -```console -sequifier train --config-path configs/train-policy.yaml -sequifier infer --config-path configs/infer-policy.yaml +Use the existing `sequenceId` and `itemPosition` conventions. Each row describes +**the current observation, the action executed at that observation, and the +reward/boundary resulting from that action**. One sequence is one episode or an +explicit episode fragment. CSV and Parquet files can contain many episodes; +files are not assumed to be episode boundaries. Positions must be nonnegative, +unique and contiguous within each sequence, including across input shards. + +```yaml +project_root: . +model_name: policy +data_sources: + demonstrations: + paths: [data/demonstrations.parquet] +policies: + actor: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + context_length: 16 + return_conditioned: false +training_plan: + algorithm: return_conditioned + epochs: 80 + sources: [{source: demonstrations}] + policy: actor +``` + +Run `sequifier train --config-path configs/train-policy.yaml`. BC +(`return_conditioned: false`) needs observations and action labels, without +rewards, successor observations, bindings, simulator metadata or fabricated seed +windows. A recorded current action is never also accepted as an observation from +the same source column. Return-conditioned modeling additionally needs recorded +rewards and a known terminal outcome or an explicitly configured finite horizon. +Unknown/truncated reward tails are excluded and reported; missing rewards never +become zero. Setting `return_conditioned: true` selects this mode. + +Defaults use the policy feature names and the columns `sequenceId`, +`itemPosition`, `reward`, `terminated` and `truncated`. Map different source names +explicitly: + +```yaml +data_sources: + demonstrations: + paths: [data/recorded.csv] + columns: + sequence_id: episode + item_position: step + observations: {position: measured_position, velocity: measured_velocity} + action: command + reward: score + terminated: task_finished + truncated: time_limit + previous_action: preceding_command + group: collection_session +``` + +Boundary columns, when present, contain booleans. No row may follow a boundary +within the same episode. Both termination and truncation may be true; termination +disables bootstrap. `episode_end: terminated` or `truncated` is an explicit claim +about each sequence's final row and must agree with recorded flags. The default +is `unknown`, never an inferred terminal outcome at file end. + +Position zero with no preceding action is an episode start. Fragments require +`columns.previous_action` at their first position; subsequent entries must lag +current actions by exactly one decision. At window boundaries, the preceding +action is fetched from the sequence rather than replaced with a start token. +Static features can be constant inline columns or supplied in a separate +`static_data_path` CSV/Parquet table keyed by the sequence ID. The table must cover +every selected ID without duplicates; fitting reads only training episodes. + +Optional `columns.successors` maps every observation name to its next-observation +column. An all-null successor is explicitly missing; partial successors are +invalid. Adjacent observations must agree with supplied successors. Optional +`columns.bootstrap_allowed` cannot permit bootstrap after termination or at a +final decision without an observed successor. BC and return-conditioned modeling +do not require successors. Task outcomes and collection groups remain data +metadata rather than model features. + +Without `validation_sources`, training splits whole episodes deterministically, +keeping repeated demonstrations and declared collection groups together. At +least two independent groups are required. Alternatively declare a held-out source +and set `validation_sources: [held_out]` on the phase. Duplicate paths and episode +IDs, overlapping collection groups and repeated demonstrations across partitions +are rejected. Normalization and categorical vocabulary fitting use training data. + +Declare raw categorical observation/static fields with `categorical_columns` on +the policy. They use the original preprocessing vocabulary fitter and reserved +IDs. Alternatively supply fixed `observation_n_classes` / `static_n_classes` for +already encoded integer IDs and optional `id_maps` for fixed domain labels. +String action labels are supported through `action_values`. Executable action +IDs use the usual global-to-decoder mapping; the start marker is input-only and +cannot be sampled as an action. Previously saved policy artifacts retain their +legacy vocabulary layout. + +Sources accept `format: auto` (default), `table`, `trajectory` or `prepared`. +Automatic selection recognizes CSV/Parquet tables, JSON rollout logs and PT +prepared histories. Prepared CSV/Parquet windows require `format: prepared`. +Prepared sources require `metadata_config_path`; overlapping windows are checked +and reconstructed into unique, split-owned decisions. A preceding action can be +recovered from stored context for a fragment. Categorical IDs and normalized +values are decoded using the original metadata, then fitted for the selected +training partition. Missing reward/boundary information remains unknown. +For a metadata-referenced prepared split, use `metadata_config_path` and +`split_index` instead of `paths`. Scalar prepared histories are currently supported; +policy image/depth ingestion and IQL/PPO remain planned. + +A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy +`manifest` listing version-1 JSON episodes. These are validated rollout recordings +adapted into the same sequence representation, with bindings retained in their +recording format. Forecast recordings require +`action_semantics: executed_predictions`; this does not convert forecast checkpoint +alignment into policy alignment. The old `trajectory_sources` registry and phase +field remain aliases for `data_sources` and `sources`. + +Run paths resolve relative to `project_root`; manifest entries resolve relative +to the manifest directory. `optimization.num_workers` controls loader workers; +`episode_cache_size` bounds cached episodes per worker. Table-backed episodes use +filtered lazy scans. Prepared window reconstruction is eager per source and JSON +records still require one episode to fit in memory. + +The best model is selected by held-out action NLL. `model_path` defaults to +`models//.pt` and contains an ordinary `sequifier_model`. +`actor_path` remains an alias. The report records partitions, exclusions, the fitted +policy codec, decision counts and per-epoch NLL. Epochs belong to the phase, seed +and device to the run, and optimizer/scheduler settings to `optimization`. + +## Optional committed import + +Direct training from a table needs no conversion. For reusable imported data, +`sequifier preprocess` supports a separate operation: + +```yaml +operation: import_sequences +project_root: . +source: + paths: [data/demonstrations.csv] +policy: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + return_conditioned: false +output_path: data/imported-demonstrations +episodes_per_shard: 64 ``` -A complete training configuration is in -[train-return-conditioned.yaml](example_inputs/configs/train-return-conditioned.yaml). -`sequifier make` also creates `configs/train-policy.yaml`. Define named -`trajectory_sources`, named `policies`, and one `training_plan` phase with -`algorithm: return_conditioned`. Set `return_conditioned: false` on the policy -for behavior cloning (BC). Offline training does not start an environment. - -Each source specifies either `paths: [episode-a.json, episode-b.json]` or -`manifest: data/trajectories/manifest.json`. Version-1 manifests contain an -`episodes` list of objects with `path` and optional `episode_id`. Only listed -files are selected. Run paths resolve relative to `project_root`; manifest -entries resolve relative to the manifest directory. - -Set `epochs` on the phase, `seed` and `device` on the run, and update settings -in `optimization`. Devices follow PyTorch's CPU, CUDA, and MPS conventions. -`optimization.num_workers` controls data-loader workers and -`optimization.episode_cache_size` bounds the number of cached file-backed -episodes per worker (default 8). The learner retains episode metadata and loads -episodes on demand. It shuffles episodes and the decisions within each episode -to keep disk access local. Normalization combines episode statistics -incrementally. JSON storage still requires an individual episode to fit in memory. - -Without `validation_sources`, the runner splits whole episodes deterministically, -keeping identical demonstrations together. At least two distinct trajectory -groups are required. To supply held-out data, declare a separate source and add -`validation_sources: [held_out]` to the phase. Duplicate paths and episode IDs, -and identical content across training and validation, are rejected. -Normalization fits only the training partition. - -The best model is selected by held-out action negative log likelihood (NLL). -`model_path` defaults to `models//.pt` and contains an -ordinary `sequifier_model` artifact. `actor_path` remains an alias for `model_path` for existing configurations. `report_path` -defaults to `outputs/training//report.json`; it records sources, -partition IDs, exclusions, decision counts, per-epoch NLL, and optimizer steps. +The output directory must be new. Import writes raw Parquet shards and a +checksummed `manifest.json`, then atomically publishes the directory. It does not +fit statistics, pad contexts or invent missing transitions. Train with +`data_sources.demonstrations.manifest: data/imported-demonstrations/manifest.json`; +the recorded column mappings are loaded automatically. The manifest retains +source provenance, episode identities, optional successors and collection groups. + +## Shared model and dataset authoring + +The compact `policies` surface is optional. Policy training also accepts the +ordinary `model.backbone` and `model.interfaces` schemas, with a `policy` data +schema on each interface. Ingestion and decoder components keep their existing +syntax; the policy adapter adds the previous-action/return features. Use top-level +`context_length`, or `global_training.context_length`, to select a context smaller +than the backbone maximum. `global_training` accepts the policy optimization +settings as an alternative to `optimization` and uses `training_objective: policy`. + +Likewise, `dataset_training.` accepts `model_interface` and `parts`, with each +part containing a sequence source specification. Reference it with the ordinary +`{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these +parts do not need fabricated criteria. Model/interface references are validated. +The shared phase schema preserves supervised defaults; this learner still supports +one `return_conditioned` phase, with BC selected on its policy. Mixed algorithms, +backbone repositories, image/depth policies and online learning remain planned. +Use `initialize_from` for backbone artifact transfer. + +See [train-policy-from-table.yaml](example_inputs/configs/train-policy-from-table.yaml) +for a complete example using the shared model, dataset and phase authoring surface. ## Configure the shared model @@ -80,7 +214,7 @@ have `support: 1` and `prediction_length: 1`. The internal input names `__policy_previous_action` and `__policy_return` are reserved. The latter exists only for return-conditioned policies. Static columns -are broadcast over the decision window. Real observation and static statistics +are broadcast over the decision window. Real observation and static sample statistics are persisted in the model interface's preprocessing metadata. Encoded categorical observations can be declared with `observation_n_classes: {state_id: 8}`; their recorded values must be integral IDs in `[0, 8)` and are not normalized. @@ -88,8 +222,9 @@ When authoring a mixed real/categorical embedding ingestion, provide `feature_embedding_dims` under the normal ingestion rules, including the internal previous-action column and, when enabled, return column. -Actions use the ordered IDs of `action_values`; an extra input-only ID equal to -the action count represents episode start. It is excluded from decoder outputs. +Actions use decoder-local IDs in the order of `action_values`, mapped to the +standard global category IDs. A separate input-only ID represents episode start +and is excluded from decoder outputs. An observation may have the same external name as the action; in that case the model uses `__policy_action` as its separate target name. Default decoders infer the target automatically. @@ -150,7 +285,8 @@ desired_return: 10.0 See [infer-policy.yaml](example_inputs/configs/infer-policy.yaml). CSV and Parquet inputs contain observations and any static columns in original units. Without a `sequenceId` column, each row is an independent first decision. Within each -sequence, row order defines decision order and static inputs must be constant. +sequence, `itemPosition` defines decision order when present and must be unique +and contiguous; otherwise row order is used. Static inputs must be constant. Provide the recorded `action_column` to derive previous actions by shifting one row, or set `previous_action_column` to a column containing previous **domain action values**, with null for episode start. Use a new sequence ID for each @@ -166,7 +302,7 @@ supplied by the caller; offline inference does not update budgets from rewards. BC requires no return input and rejects a scalar `desired_return`. `inference_batch_size` bounds batched model calls (default 64). Predictions go to `outputs/predictions//part-000.csv` by default. -`output_path` overrides that location. Sequence IDs and `itemPosition`, when +`output_path` overrides that location. Sequence IDs and the configured `item_position_column` (default `itemPosition`), when present, are retained in input order. `output_probabilities: true` adds one `__probability_` column per action. Set `deterministic: false` for seeded categorical sampling. Select `model_interface` diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index 1e356682..0721cd33 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -25,3 +25,19 @@ and CLI ONNX policy execution remain future work. No tests or training runs were executed, as requested. Validation used static lint, formatting, and type checks. + +Policy sequence representation follow-up: + +- Train BC directly from CSV/Parquet observation/action tables. Rewards, + successors, bindings and seed histories are not required for BC. +- JSON recordings and existing scalar prepared histories adapt into the same + columnar episode representation, with optional transition capabilities. +- Policy windows use explicit same-position labels and causal attention, named + tensors, standard sequence batches and the supervised loss service. +- Categorical maps and sample statistics use shared preprocessing functions and + fit only training episodes. New action vocabularies use global/decoder mappings; + existing policy artifacts retain their legacy layout. +- Shared model/dataset/phase authoring is available alongside compact policy + configuration. Preprocess can optionally commit raw Parquet sequence shards + with a checksummed manifest. +- No tests or training runs were written or executed for this change, as requested. diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index d6dabb98..d73033dc 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -4,6 +4,7 @@ from pydantic import TypeAdapter +from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig from sequifier.config.train_config import ( BackboneComponentConfig, DatasetFreezingSpecModel, @@ -96,6 +97,17 @@ def resolved_config_from_model_config( else None ) authored_interfaces[name] = ModelInterfaceSpecModel( + policy=( + PolicySchemaConfig.model_validate( + { + key: value + for key, value in interface["policy"].items() + if key in PolicySchemaConfig.model_fields + } + ) + if interface.get("policy") is not None + else None + ), input_columns=interface["input_columns"], static_conditioning=interface.get("static_conditioning"), preconditioning_image_encoder=interface.get( @@ -116,8 +128,15 @@ def resolved_config_from_model_config( {column: len(ids) for column, ids in target_decoder_ids.items()}, ) global_to_decoder = interface.get("target_global_to_decoder", {}) + policy_spec = interface.get("policy") + if policy_spec is not None: + policy_spec = dict(policy_spec) + # Existing policy artifacts used decoder IDs as input vocabulary IDs. + policy_spec.setdefault("categorical_encoding", "legacy") resolved = ResolvedModelInterface( - policy=interface.get("policy"), + policy=CategoricalPolicyConfig.model_validate(policy_spec) + if policy_spec is not None + else None, depth_layouts=interface.get("depth_layouts", {}), tensor_payload_version=interface.get("tensor_payload_version", 1), split_paths=interface.get("split_paths", []), diff --git a/src/sequifier/config/identifiers.py b/src/sequifier/config/identifiers.py new file mode 100644 index 00000000..6de312de --- /dev/null +++ b/src/sequifier/config/identifiers.py @@ -0,0 +1,12 @@ +import keyword + +from sequifier.typechecking import beartype + + +@beartype +def identifier(value: str, usage: str) -> str: + if "." in value or not value.isidentifier() or keyword.iskeyword(value): + raise ValueError( + f"{usage} {value!r} must be a valid identifier and cannot contain '.'." + ) + return value diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index fc8b9c4b..13851f96 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -2,15 +2,17 @@ from typing import Annotated, Literal -from pydantic import BaseModel, ConfigDict, Field, FiniteFloat, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator from sequifier.config.components import ( BackboneArchitectureConfig, + BackboneComponentConfig, ComponentSpec, DecoderComponentConfig, IngestionComponentConfig, ResumeConfig, ) +from sequifier.io.values import DomainValue class PolicyConfigBase(BaseModel): @@ -25,7 +27,9 @@ class ReturnDefinition(PolicyConfigBase): class PolicyModelConfig(PolicyConfigBase): context_length: int = Field(default=4, gt=0) + interface_name: str = Field(default="policy", min_length=1) backbone: BackboneArchitectureConfig + backbone_component: BackboneComponentConfig | None = None ingestion: IngestionComponentConfig | None = None decoder: DecoderComponentConfig | None = None @@ -56,21 +60,37 @@ def validate_backbone(self): raise ValueError( "policy context_length exceeds backbone max_context_length" ) + if self.backbone_component is not None: + if self.backbone_component.architecture != self.backbone: + raise ValueError( + "backbone component architecture must match the policy backbone" + ) + if self.backbone_component.repository is not None: + raise ValueError( + "policy phases currently use initialize_from instead of backbone repositories" + ) return self -class CategoricalPolicyConfig(PolicyModelConfig): +class PolicySchemaConfig(PolicyConfigBase): alignment: Literal["current_observation_previous_action"] = ( "current_observation_previous_action" ) observation_columns: list[str] = Field(min_length=1) static_columns: list[str] = Field(default_factory=list) + categorical_columns: list[str] = Field(default_factory=list) # Numeric trajectory fields may also carry already encoded categorical IDs. - observation_n_classes: dict[ - str, Annotated[int, Field(strict=True, ge=2, le=2**24)] - ] = Field(default_factory=dict) + observation_n_classes: dict[str, Annotated[int, Field(strict=True, ge=2)]] = Field( + default_factory=dict + ) + static_n_classes: dict[str, Annotated[int, Field(strict=True, ge=2)]] = Field( + default_factory=dict + ) + # The same label -> global ID convention as preprocessing metadata. + id_maps: dict[str, dict[str | int, int]] = Field(default_factory=dict) + categorical_encoding: Literal["global", "legacy"] = "global" action_column: str = Field(default="action", min_length=1) - action_values: list[FiniteFloat] = Field( + action_values: list[DomainValue] = Field( default_factory=lambda: [0.0, 1.0], min_length=2 ) return_conditioned: bool = True @@ -92,14 +112,48 @@ def validate_names(self): raise ValueError( "observation_n_classes requires observation columns with at least two classes" ) + if set(self.static_n_classes) - set(self.static_columns): + raise ValueError("static_n_classes must name static columns") + counts = {**self.observation_n_classes, **self.static_n_classes} + if len(set(self.categorical_columns)) != len(self.categorical_columns) or set( + self.categorical_columns + ) - set(names): + raise ValueError( + "categorical_columns must name unique observation/static features" + ) + if set(self.id_maps) - set(counts): + raise ValueError("id_maps must name declared categorical inputs") + for column, mapping in self.id_maps.items(): + if ( + not mapping + or len(set(mapping.values())) != len(mapping) + or any( + value < 0 or value >= counts[column] for value in mapping.values() + ) + ): + raise ValueError(f"{column}: invalid categorical ID map") + if len(self.action_values) != len(set(self.action_values)): + raise ValueError("policy action_values must be unique") + if len({isinstance(value, str) for value in self.action_values}) != 1: + raise ValueError( + "action_values must use consistently numeric or string labels" + ) + return self + + @property + def categorical_n_classes(self) -> dict[str, int]: + return {**self.observation_n_classes, **self.static_n_classes} + + +class CategoricalPolicyConfig(PolicySchemaConfig, PolicyModelConfig): + @model_validator(mode="after") + def validate_decoder(self): if self.decoder is not None and ( self.decoder.support != 1 or self.decoder.prediction_length != 1 ): raise ValueError( "policy decoders require support=1 and prediction_length=1" ) - if len(self.action_values) != len(set(self.action_values)): - raise ValueError("policy action_values must be unique") return self diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index ac5c6e2b..91a6205d 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -28,6 +28,7 @@ class PolicyInferenceConfig(PolicyConfigBase): desired_return_column: str = "desired_return" previous_action_column: str | None = None sequence_id_column: str = "sequenceId" + item_position_column: str = "itemPosition" rollout: RolloutConfig | None = None autoregressive: bool = False generation_steps: int | None = Field(default=None, gt=0) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 252a0a9e..b1b99f79 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -3,40 +3,139 @@ from pathlib import Path from typing import Literal -from pydantic import AliasChoices, Field, field_validator, model_validator +from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator from sequifier.config.policy_config import ( CategoricalPolicyConfig, PolicyConfigBase, PolicyOptimizationConfig, ) +from sequifier.config.training_plan import ( + PolicyTrainingPlanModel, + ReturnConditionedPhaseConfig, +) + + +class PolicyTableColumns(PolicyConfigBase): + """Explicit row timing: current observation/action, then resulting reward.""" + + sequence_id: str = "sequenceId" + item_position: str = "itemPosition" + observations: dict[str, str] = Field(default_factory=dict) + static_inputs: dict[str, str] = Field(default_factory=dict) + action: str | None = None + reward: str | None = "reward" + terminated: str | None = "terminated" + truncated: str | None = "truncated" + previous_action: str | None = None + successors: dict[str, str] = Field(default_factory=dict) + bootstrap_allowed: str | None = None + group: str | None = None + episode_id: str | None = None + + @model_validator(mode="after") + def validate_columns(self): + names = [ + value for value in self.model_dump().values() if isinstance(value, str) + ] + names += ( + list(self.observations.values()) + + list(self.static_inputs.values()) + + list(self.successors.values()) + ) + if any(not value for value in names): + raise ValueError("table column names must be nonempty") + return self -class TrajectorySourceConfig(PolicyConfigBase): +class SequenceSourceConfig(PolicyConfigBase): paths: list[str] = Field(default_factory=list) manifest: str | None = None + format: Literal["auto", "trajectory", "table", "prepared"] = "auto" + columns: PolicyTableColumns = Field(default_factory=PolicyTableColumns) + # A file boundary alone says nothing about task completion. + episode_end: Literal["unknown", "terminated", "truncated"] = "unknown" + episode_namespace: str = "" + metadata_config_path: str | None = None + encoding: Literal["raw", "model"] = "raw" + static_data_path: str | None = None + split_index: int | None = Field(default=None, ge=0) action_semantics: Literal["policy", "executed_predictions"] = "policy" @model_validator(mode="after") def validate_location(self): - if bool(self.paths) == (self.manifest is not None): + metadata_split = ( + self.metadata_config_path is not None and self.split_index is not None + ) + if sum((bool(self.paths), self.manifest is not None, metadata_split)) != 1: raise ValueError("trajectory source requires either paths or one manifest") if any(not path for path in self.paths) or self.manifest == "": raise ValueError("trajectory paths must be nonempty") + if self.split_index is not None and self.metadata_config_path is None: + raise ValueError("split_index requires metadata_config_path") + if self.encoding == "model" and self.metadata_config_path is None: + raise ValueError("model-encoded sources require metadata_config_path") + if self.format == "prepared" and self.metadata_config_path is None: + raise ValueError("prepared sources require metadata_config_path") return self -class ReturnConditionedPhaseConfig(PolicyConfigBase): - algorithm: Literal["return_conditioned"] - name: str = Field(default="policy", min_length=1) - epochs: int = Field(gt=0) - trajectory_sources: list[str] = Field(min_length=1) - validation_sources: list[str] | None = Field(default=None, min_length=1) - policy: str = Field(min_length=1) +# Existing Python callers and authored trajectory_sources remain supported. +TrajectorySourceConfig = SequenceSourceConfig -class PolicyTrainingPlanConfig(PolicyConfigBase): - phases: list[ReturnConditionedPhaseConfig] = Field(min_length=1, max_length=1) +PolicyTrainingPlanConfig = PolicyTrainingPlanModel + + +def resolve_sequence_source( + source: SequenceSourceConfig, root: Path +) -> SequenceSourceConfig: + values = source.model_dump() + values["paths"] = [str((root / path).resolve()) for path in source.paths] + for key in ("manifest", "metadata_config_path", "static_data_path"): + if values[key] is not None: + values[key] = str((root / values[key]).resolve()) + if ( + values["metadata_config_path"] is not None + and values["static_data_path"] is None + ): + from sequifier.config.metadata import load_dataset_metadata + + metadata = load_dataset_metadata(values["metadata_config_path"]) + prepared = ( + source.format == "prepared" + or source.split_index is not None + or any(Path(path).suffix.lower() == ".pt" for path in source.paths) + ) + if prepared and metadata.static_features_path: + values["static_data_path"] = str( + (root / metadata.static_features_path).resolve() + ) + if source.split_index is not None: + from sequifier.config.metadata import load_dataset_metadata + + metadata_path = values["metadata_config_path"] + if not isinstance(metadata_path, str): + raise ValueError("prepared split sources require metadata_config_path") + metadata = load_dataset_metadata(metadata_path) + if source.split_index >= len(metadata.split_paths): + raise ValueError( + "source split_index is outside the prepared metadata splits" + ) + path = (root / metadata.split_paths[source.split_index]).resolve() + values["paths"] = ( + sorted( + str(item) + for item in path.glob( + "*.pt" if path.is_dir() and any(path.glob("*.pt")) else "*.parquet" + ) + ) + if path.is_dir() + else [str(path)] + ) + values["format"] = "prepared" + values["split_index"] = None + return SequenceSourceConfig.model_validate(values) class _PolicyTrainConfigBase(PolicyConfigBase): @@ -45,7 +144,10 @@ class _PolicyTrainConfigBase(PolicyConfigBase): device: str = "cpu" initialize_from: str | None = None seed: int = Field(default=1010, ge=0) - trajectory_sources: dict[str, TrajectorySourceConfig] = Field(min_length=1) + trajectory_sources: dict[str, SequenceSourceConfig] = Field( + min_length=1, + validation_alias=AliasChoices("data_sources", "trajectory_sources"), + ) policies: dict[str, CategoricalPolicyConfig] = Field(min_length=1) training_plan: PolicyTrainingPlanConfig # Epochs and seed belong to the phase and run respectively. @@ -71,9 +173,124 @@ def normalize_plan(cls, values): if not isinstance(values, dict): return values values = dict(values) + source_interfaces = {} + if "global_training" in values: + if "optimization" in values: + raise ValueError("configure global_training or optimization, not both") + training = dict(values.pop("global_training")) + context = training.pop("context_length", None) + if context is not None: + values["context_length"] = context + if training.pop("training_objective", "policy") != "policy": + raise ValueError("policy phases require training_objective: policy") + values["optimization"] = training + if "dataset_training" in values: + if "data_sources" in values or "trajectory_sources" in values: + raise ValueError("configure dataset_training or data_sources, not both") + sources = {} + for name, dataset in values.pop("dataset_training").items(): + if set(dataset) - {"model_interface", "parts"}: + raise ValueError( + "policy datasets configure model_interface and source parts; action NLL is supplied by the algorithm" + ) + if not dataset.get("parts"): + raise ValueError("policy datasets require nonempty source parts") + for part, source in dataset["parts"].items(): + sources[f"{name}.{part}"] = source + source_interfaces[f"{name}.{part}"] = dataset.get("model_interface") + values["data_sources"] = sources + # Canonical model authoring is shared with supervised learning. The + # compact policies registry remains a compatibility/convenience surface. + if "model" in values: + if "policies" in values: + raise ValueError("configure model or policies, not both") + from sequifier.config.train_config import ModelSpecModel + + model = ModelSpecModel.model_validate(values.pop("model")) + policies = {} + for name, interface in model.interfaces.items(): + if interface.policy is None: + raise ValueError( + "policy training model interfaces require a policy schema" + ) + schema = interface.policy.model_dump() + expected = schema["observation_columns"] + schema["static_columns"] + if interface.input_columns and interface.input_columns != expected: + raise ValueError( + "policy interface inputs must name its observation and static columns in order" + ) + target = schema["action_column"] + if interface.target_columns != [target]: + raise ValueError( + "policy interface targets must name its action column" + ) + if ( + interface.categorical_hashing + or interface.static_conditioning + or interface.preconditioning_image_encoder + or interface.feature_layout + ): + raise ValueError( + "this policy learner currently supports scalar policy inputs" + ) + policies[name] = { + **schema, + "interface_name": name, + "backbone": model.backbone.architecture.model_dump(), + "backbone_component": model.backbone.model_dump(), + "context_length": values.get( + "context_length", model.backbone.architecture.max_context_length + ), + "ingestion": interface.ingestion.model_dump(), + "decoder": interface.decoder.model_dump(), + } + values.pop("context_length", None) + values["policies"] = policies + elif "context_length" in values: + context = values.pop("context_length") + values["policies"] = { + name: {**policy, "context_length": context} + for name, policy in values.get("policies", {}).items() + } plan = values.get("training_plan") + if plan is not None and isinstance(plan, BaseModel): + plan = plan.model_dump() + values["training_plan"] = plan if isinstance(plan, dict) and "phases" not in plan: values["training_plan"] = {"phases": [plan]} + plan = values.get("training_plan", {}) + if not isinstance(plan, dict): + plan = {} + normalized_phases = [] + for phase in plan.get("phases", []): + if isinstance(phase, BaseModel): + phase = phase.model_dump() + if not isinstance(phase, dict): + normalized_phases.append(phase) + continue + phase = dict(phase) + if "policy" not in phase and len(values.get("policies", {})) == 1: + phase["policy"] = next(iter(values["policies"])) + for reference in phase.get("sources", phase.get("trajectory_sources", [])): + if isinstance(reference, BaseModel): + reference = reference.model_dump() + name = ( + reference + if isinstance(reference, str) + else reference.get("source") + if isinstance(reference, dict) + else None + ) + if source_interfaces.get(name) is not None and source_interfaces[ + name + ] != phase.get("policy"): + raise ValueError( + "policy dataset model_interface disagrees with its phase policy" + ) + # Work on copied phases, never mutate the caller's authored mapping. + normalized_phases.append(phase) + if plan.get("phases"): + values["training_plan"] = {**plan, "phases": normalized_phases} optimization = values.get("optimization", {}) if isinstance(optimization, dict) and {"epochs", "seed"} & optimization.keys(): raise ValueError("set epochs on the phase and seed on the run") @@ -81,6 +298,10 @@ def normalize_plan(cls, values): @model_validator(mode="after") def validate_references(self): + if len(self.training_plan.phases) != 1 or not isinstance( + self.training_plan.phases[0], ReturnConditionedPhaseConfig + ): + raise ValueError("this learner supports one return_conditioned phase") phase = self.training_plan.phases[0] if phase.policy not in self.policies: raise ValueError(f"unknown policy {phase.policy!r}") @@ -134,10 +355,10 @@ def absolute(path): values = config.model_dump(mode="json") values["project_root"] = str(root) - for source in values["trajectory_sources"].values(): - source["paths"] = [absolute(path) for path in source["paths"]] - if source["manifest"] is not None: - source["manifest"] = absolute(source["manifest"]) + values["trajectory_sources"] = { + name: resolve_sequence_source(source, root).model_dump() + for name, source in config.trajectory_sources.items() + } if config.initialize_from is not None: values["initialize_from"] = absolute(config.initialize_from) values["actor_path"] = absolute( diff --git a/src/sequifier/config/preprocess_config.py b/src/sequifier/config/preprocess_config.py index f1ecc247..a8ba01c9 100644 --- a/src/sequifier/config/preprocess_config.py +++ b/src/sequifier/config/preprocess_config.py @@ -19,6 +19,7 @@ ) from sequifier.config.depth_layout import DepthLayoutRegistryModel from sequifier.config.image_config import ImageSourceConfig, PreconditioningImageConfig +from sequifier.config.sequence_import_config import SequenceImportConfig from sequifier.helpers import ( canonicalize_polars_dtype_name, is_float_dtype_name, @@ -31,12 +32,14 @@ @beartype def load_preprocessor_config( config_path: str, args_config: dict -) -> "PreprocessorModel": +) -> "PreprocessorModel | SequenceImportConfig": """Load preprocessing YAML plus CLI overrides.""" config_values = load_composed_yaml_config(config_path) config_values = merge_config_fragments((config_values, args_config)) + if config_values.get("operation") == "import_sequences": + return SequenceImportConfig.model_validate(config_values) return try_catch_excess_keys(config_path, PreprocessorModel, config_values) @@ -350,7 +353,7 @@ def validate_mask_column_requires_metadata(self) -> "PreprocessorModel": overlap = categorical & real if overlap: raise ValueError( - "Columns cannot be both categorical and real: " f"{sorted(overlap)}" + f"Columns cannot be both categorical and real: {sorted(overlap)}" ) declared = categorical | real @@ -434,7 +437,7 @@ def validate_mask_column_requires_metadata(self) -> "PreprocessorModel": raise ValueError("split_column cannot be sequenceId or itemPosition") if self.split_column == "__sequifier_split_value": raise ValueError( - "split_column cannot use reserved name " "'__sequifier_split_value'" + "split_column cannot use reserved name '__sequifier_split_value'" ) if self.split_column == self.mask_column: raise ValueError("split_column cannot also be mask_column") diff --git a/src/sequifier/config/sequence_import_config.py b/src/sequifier/config/sequence_import_config.py new file mode 100644 index 00000000..ec6c0b49 --- /dev/null +++ b/src/sequifier/config/sequence_import_config.py @@ -0,0 +1,23 @@ +"""The explicit sequence import operation of sequifier preprocess.""" + +from pathlib import Path +from typing import Literal + +from pydantic import Field + +from sequifier.config.policy_config import PolicyConfigBase, PolicySchemaConfig +from sequifier.config.policy_train_config import SequenceSourceConfig + + +class SequenceImportConfig(PolicyConfigBase): + operation: Literal["import_sequences"] + project_root: str = "." + source: SequenceSourceConfig + policy: PolicySchemaConfig + output_path: str + episodes_per_shard: int = Field(default=64, gt=0) + + def resolved_source(self) -> SequenceSourceConfig: + from sequifier.config.policy_train_config import resolve_sequence_source + + return resolve_sequence_source(self.source, Path(self.project_root).resolve()) diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 96cef23e..299281e4 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -10,7 +10,6 @@ import copy import inspect -import keyword import math import os import warnings @@ -47,6 +46,7 @@ IngestionFreezingConfig, LayerFreezingConfigFields, ) +from sequifier.config.identifiers import identifier as _identifier from sequifier.config.image_config import ( ImageSourceConfig, MediaPath, @@ -59,12 +59,19 @@ OptimizerPlan, resolve_plan_scheduler_arguments, ) -from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig from sequifier.config.policy_train_config import ( PolicyTrainConfig, ResolvedPolicyTrainConfig, resolve_policy_train_config, ) +from sequifier.config.training_plan import ( + SupervisedTrainingPlanModel as TrainingPlanModel, +) +from sequifier.config.training_plan import ( + TrainingPhaseSpecModel as TrainingPhaseSpecModel, +) +from sequifier.config.training_plan import TrainingSourceSpecModel from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -111,15 +118,6 @@ def _validate_constructor_arguments( raise ValueError(f"Invalid {component} {name} arguments: {error}") from error -@beartype -def _identifier(value: str, usage: str) -> str: - if "." in value or not value.isidentifier() or keyword.iskeyword(value): - raise ValueError( - f"{usage} {value!r} must be a valid identifier and cannot contain '.'." - ) - return value - - @beartype def _unique_columns(value: list[str], usage: str) -> list[str]: if len(value) != len(set(value)): @@ -604,6 +602,7 @@ class ModelInterfaceSpecModel(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") input_columns: list[str] = Field(default_factory=list) + policy: PolicySchemaConfig | None = None static_conditioning: StaticConditioningConfig | None = None preconditioning_image_encoder: ImageEncoderConfig | None = None target_columns: list[str] = Field(..., min_length=1) @@ -818,87 +817,6 @@ def validate_loss_weights(cls, value): return value -class TrainingSourceSpecModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - source: str - weight: Optional[float] = Field(default=None, gt=0) - batches_per_selection: Optional[int] = Field(default=None, gt=0) - - @field_validator("source") - @classmethod - @beartype - def validate_source(cls, value): - parts = value.split(".") - if len(parts) > 2: - raise ValueError("Source refs use only dataset or dataset.part") - for part in parts: - _identifier(part, "Source reference component") - return value - - -class TrainingPhaseSpecModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - algorithm: Literal["supervised"] = "supervised" - - name: str - epochs: int = Field(gt=0) - mode: Literal["sequential", "interleaved"] - selection: Optional[Literal["round_robin", "weighted_random"]] = None - sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) - - @field_validator("name") - @classmethod - @beartype - def validate_name(cls, value): - return _identifier(value, "Training phase name") - - @model_validator(mode="after") - @beartype - def validate_mode_fields(self): - if self.mode == "sequential": - if self.selection is not None: - raise ValueError("Sequential phases cannot configure selection") - if any(source.batches_per_selection is not None for source in self.sources): - raise ValueError( - "batches_per_selection is invalid in sequential phases" - ) - if any(source.weight is not None for source in self.sources): - raise ValueError("weight is invalid in sequential phases") - else: - self.selection = self.selection or "round_robin" - if self.selection == "round_robin" and any( - source.weight is not None for source in self.sources - ): - raise ValueError("weight is only valid for weighted_random selection") - return self - - -class TrainingPlanModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - curriculum_training: bool = False - curriculum_column: Optional[str] = Field(default=None, min_length=1) - phases: list[TrainingPhaseSpecModel] = Field(..., min_length=1) - - @model_validator(mode="after") - @beartype - def validate_unique_names(self): - if self.curriculum_column is not None and not self.curriculum_training: - raise ValueError( - "curriculum_column may only be set when curriculum_training is true" - ) - if self.curriculum_training and self.curriculum_column is None: - raise ValueError( - "curriculum_column must be set when curriculum_training is true" - ) - names = [phase.name for phase in self.phases] - if len(names) != len(set(names)): - raise ValueError("Training phase names must be unique") - return self - - class EvaluationMonitorSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py new file mode 100644 index 00000000..c92c46aa --- /dev/null +++ b/src/sequifier/config/training_plan.py @@ -0,0 +1,156 @@ +"""Shared authored phase contracts; algorithm-specific budgets remain explicit.""" + +from typing import Annotated, Generic, Literal, Optional, TypeVar + +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, +) + +from sequifier.config.identifiers import identifier as _identifier +from sequifier.typechecking import beartype + + +class TrainingSourceSpecModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + source: str + weight: Optional[float] = Field(default=None, gt=0) + batches_per_selection: Optional[int] = Field(default=None, gt=0) + + @field_validator("source") + @classmethod + @beartype + def validate_source(cls, value): + parts = value.split(".") + if len(parts) > 2: + raise ValueError("Source refs use only dataset or dataset.part") + for part in parts: + _identifier(part, "Source reference component") + return value + + +class TrainingPhaseSpecModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + algorithm: Literal["supervised"] = "supervised" + + name: str + epochs: int = Field(gt=0) + mode: Literal["sequential", "interleaved"] + selection: Optional[Literal["round_robin", "weighted_random"]] = None + sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) + + @field_validator("name") + @classmethod + @beartype + def validate_name(cls, value): + return _identifier(value, "Training phase name") + + @model_validator(mode="after") + @beartype + def validate_mode_fields(self): + if self.mode == "sequential": + if self.selection is not None: + raise ValueError("Sequential phases cannot configure selection") + if any(source.batches_per_selection is not None for source in self.sources): + raise ValueError( + "batches_per_selection is invalid in sequential phases" + ) + if any(source.weight is not None for source in self.sources): + raise ValueError("weight is invalid in sequential phases") + else: + self.selection = self.selection or "round_robin" + if self.selection == "round_robin" and any( + source.weight is not None for source in self.sources + ): + raise ValueError("weight is only valid for weighted_random selection") + return self + + +class ReturnConditionedPhaseConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + algorithm: Literal["return_conditioned"] + name: str = Field(default="policy", min_length=1) + epochs: int = Field(gt=0) + sources: list[TrainingSourceSpecModel] = Field( + min_length=1, validation_alias=AliasChoices("sources", "trajectory_sources") + ) + + @field_validator("sources", mode="before") + @classmethod + def normalize_sources(cls, values): + return [ + {"source": value} if isinstance(value, str) else value for value in values + ] + + @model_validator(mode="after") + def validate_sources(self): + if any( + value.weight is not None or value.batches_per_selection is not None + for value in self.sources + ): + raise ValueError( + "policy sources do not support supervised source scheduling" + ) + return self + + @property + def trajectory_sources(self) -> list[str]: + return [value.source for value in self.sources] + + validation_sources: list[str] | None = Field(default=None, min_length=1) + policy: str = Field(min_length=1) + + +Phase = TypeVar("Phase", TrainingPhaseSpecModel, ReturnConditionedPhaseConfig) + + +class TrainingPlanModel(BaseModel, Generic[Phase]): + model_config = ConfigDict(extra="forbid") + + curriculum_training: bool = False + curriculum_column: Optional[str] = Field(default=None, min_length=1) + phases: list[Phase] = Field(min_length=1) + + @field_validator("phases", mode="before") + @classmethod + def default_algorithm(cls, values): + return [ + {"algorithm": "supervised", **value} if isinstance(value, dict) else value + for value in values + ] + + @model_validator(mode="after") + @beartype + def validate_unique_names(self): + if self.curriculum_column is not None and not self.curriculum_training: + raise ValueError( + "curriculum_column may only be set when curriculum_training is true" + ) + if self.curriculum_training and self.curriculum_column is None: + raise ValueError( + "curriculum_column must be set when curriculum_training is true" + ) + if self.curriculum_training and any( + phase.algorithm != "supervised" for phase in self.phases + ): + raise ValueError( + "curriculum scheduling currently requires supervised phases" + ) + names = [phase.name for phase in self.phases] + if len(names) != len(set(names)): + raise ValueError("Training phase names must be unique") + return self + + +SupervisedTrainingPlanModel = TrainingPlanModel[TrainingPhaseSpecModel] +PolicyTrainingPlanModel = TrainingPlanModel[ReturnConditionedPhaseConfig] +AlgorithmPhase = Annotated[ + TrainingPhaseSpecModel | ReturnConditionedPhaseConfig, + Field(discriminator="algorithm"), +] diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index b2b0e6d4..909fc9a2 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -104,7 +104,7 @@ There are seven standalone commands within sequifier: `make`, `preprocess`, `tra | --- | --- | | `make` | Create a new sequifier project with config templates. | | `preprocess` | Convert input data into fixed-length subsequences. | -| `train` | Train a supervised model or a trajectory-backed categorical policy. | +| `train` | Train a supervised model or a categorical policy from sequence tables or rollout recordings. | | `infer` | Generate predictions, probabilities, embeddings, or policy actions and rollouts. | | `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | | `visualize-training` | Generate interactive HTML plots from structured training metrics. | @@ -934,6 +934,20 @@ To consume a named layout during training, configure a `depth_transformer` ingestion branch whose `layout` names this entry and whose `columns` are selected features from it. See [the training guide](train.md#depth-encoders-and-nested-composites). +## Import sequence demonstrations + +`operation: import_sequences` selects the explicit policy-data import operation. +It accepts a `source`, a model-independent `policy` data schema, an `output_path` +(new dataset directory) and optional `episodes_per_shard` (default 64). Sources +can be raw CSV/Parquet tables, existing scalar prepared histories with their +metadata, or JSON rollout recordings. Import preserves domain values and missing +transition information; it does not fit vocabularies/statistics or create padded +windows. Raw tables can also be used directly by policy training without this step. + +The operation atomically publishes Parquet shards and a checksummed manifest. +See [policy learning](../policy-learning.md#optional-committed-import) and +[import-policy-sequences.yaml](../example_inputs/configs/import-policy-sequences.yaml). + # Train Command Guide @@ -1720,6 +1734,24 @@ Policy models use the shared configurable network and portable artifacts. They support CPU/CUDA/MPS, backbone transfer with `initialize_from`, bounded file-backed episode loading, and epoch-boundary resume through `optimization.resume`. +## Policy sequence sources and shared authoring + +Policy phases accept ordinary CSV/Parquet demonstrations with `sequenceId` and +`itemPosition`. Use `data_sources` and phase `sources: [{source: demonstrations}]`; +the previous `trajectory_sources` spellings remain accepted. BC does not require +rewards or successor states. Return-conditioned training requires rewards and a +known return endpoint. JSON rollout logs are adapted into the same sequence data. + +Policy training can use the ordinary `model.backbone`, `model.interfaces` and +`dataset_training..parts` authoring patterns. Put the model-independent +observation/action schema in an interface's `policy` field and a sequence source +in each dataset part. The algorithm supplies action NLL. `global_training` can +supply the policy optimization settings and `context_length`, with +`training_objective: policy`; the compact `policies`/`optimization` surface remains +supported. One policy phase is currently supported. See the complete +[table training example](../example_inputs/configs/train-policy-from-table.yaml) +and [policy learning guide](../policy-learning.md). + # Infer Command Guide @@ -2325,47 +2357,181 @@ A policy adds a decision-alignment contract to a model interface: at time `t`, the inputs contain the current observation, the **previous** action, and, optionally, a desired return. The current action is the supervised target. -## Train from trajectories +## Train from ordinary sequence tables -```console -sequifier train --config-path configs/train-policy.yaml -sequifier infer --config-path configs/infer-policy.yaml +Use the existing `sequenceId` and `itemPosition` conventions. Each row describes +**the current observation, the action executed at that observation, and the +reward/boundary resulting from that action**. One sequence is one episode or an +explicit episode fragment. CSV and Parquet files can contain many episodes; +files are not assumed to be episode boundaries. Positions must be nonnegative, +unique and contiguous within each sequence, including across input shards. + +```yaml +project_root: . +model_name: policy +data_sources: + demonstrations: + paths: [data/demonstrations.parquet] +policies: + actor: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + context_length: 16 + return_conditioned: false +training_plan: + algorithm: return_conditioned + epochs: 80 + sources: [{source: demonstrations}] + policy: actor +``` + +Run `sequifier train --config-path configs/train-policy.yaml`. BC +(`return_conditioned: false`) needs observations and action labels, without +rewards, successor observations, bindings, simulator metadata or fabricated seed +windows. A recorded current action is never also accepted as an observation from +the same source column. Return-conditioned modeling additionally needs recorded +rewards and a known terminal outcome or an explicitly configured finite horizon. +Unknown/truncated reward tails are excluded and reported; missing rewards never +become zero. Setting `return_conditioned: true` selects this mode. + +Defaults use the policy feature names and the columns `sequenceId`, +`itemPosition`, `reward`, `terminated` and `truncated`. Map different source names +explicitly: + +```yaml +data_sources: + demonstrations: + paths: [data/recorded.csv] + columns: + sequence_id: episode + item_position: step + observations: {position: measured_position, velocity: measured_velocity} + action: command + reward: score + terminated: task_finished + truncated: time_limit + previous_action: preceding_command + group: collection_session +``` + +Boundary columns, when present, contain booleans. No row may follow a boundary +within the same episode. Both termination and truncation may be true; termination +disables bootstrap. `episode_end: terminated` or `truncated` is an explicit claim +about each sequence's final row and must agree with recorded flags. The default +is `unknown`, never an inferred terminal outcome at file end. + +Position zero with no preceding action is an episode start. Fragments require +`columns.previous_action` at their first position; subsequent entries must lag +current actions by exactly one decision. At window boundaries, the preceding +action is fetched from the sequence rather than replaced with a start token. +Static features can be constant inline columns or supplied in a separate +`static_data_path` CSV/Parquet table keyed by the sequence ID. The table must cover +every selected ID without duplicates; fitting reads only training episodes. + +Optional `columns.successors` maps every observation name to its next-observation +column. An all-null successor is explicitly missing; partial successors are +invalid. Adjacent observations must agree with supplied successors. Optional +`columns.bootstrap_allowed` cannot permit bootstrap after termination or at a +final decision without an observed successor. BC and return-conditioned modeling +do not require successors. Task outcomes and collection groups remain data +metadata rather than model features. + +Without `validation_sources`, training splits whole episodes deterministically, +keeping repeated demonstrations and declared collection groups together. At +least two independent groups are required. Alternatively declare a held-out source +and set `validation_sources: [held_out]` on the phase. Duplicate paths and episode +IDs, overlapping collection groups and repeated demonstrations across partitions +are rejected. Normalization and categorical vocabulary fitting use training data. + +Declare raw categorical observation/static fields with `categorical_columns` on +the policy. They use the original preprocessing vocabulary fitter and reserved +IDs. Alternatively supply fixed `observation_n_classes` / `static_n_classes` for +already encoded integer IDs and optional `id_maps` for fixed domain labels. +String action labels are supported through `action_values`. Executable action +IDs use the usual global-to-decoder mapping; the start marker is input-only and +cannot be sampled as an action. Previously saved policy artifacts retain their +legacy vocabulary layout. + +Sources accept `format: auto` (default), `table`, `trajectory` or `prepared`. +Automatic selection recognizes CSV/Parquet tables, JSON rollout logs and PT +prepared histories. Prepared CSV/Parquet windows require `format: prepared`. +Prepared sources require `metadata_config_path`; overlapping windows are checked +and reconstructed into unique, split-owned decisions. A preceding action can be +recovered from stored context for a fragment. Categorical IDs and normalized +values are decoded using the original metadata, then fitted for the selected +training partition. Missing reward/boundary information remains unknown. +For a metadata-referenced prepared split, use `metadata_config_path` and +`split_index` instead of `paths`. Scalar prepared histories are currently supported; +policy image/depth ingestion and IQL/PPO remain planned. + +A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy +`manifest` listing version-1 JSON episodes. These are validated rollout recordings +adapted into the same sequence representation, with bindings retained in their +recording format. Forecast recordings require +`action_semantics: executed_predictions`; this does not convert forecast checkpoint +alignment into policy alignment. The old `trajectory_sources` registry and phase +field remain aliases for `data_sources` and `sources`. + +Run paths resolve relative to `project_root`; manifest entries resolve relative +to the manifest directory. `optimization.num_workers` controls loader workers; +`episode_cache_size` bounds cached episodes per worker. Table-backed episodes use +filtered lazy scans. Prepared window reconstruction is eager per source and JSON +records still require one episode to fit in memory. + +The best model is selected by held-out action NLL. `model_path` defaults to +`models//.pt` and contains an ordinary `sequifier_model`. +`actor_path` remains an alias. The report records partitions, exclusions, the fitted +policy codec, decision counts and per-epoch NLL. Epochs belong to the phase, seed +and device to the run, and optimizer/scheduler settings to `optimization`. + +## Optional committed import + +Direct training from a table needs no conversion. For reusable imported data, +`sequifier preprocess` supports a separate operation: + +```yaml +operation: import_sequences +project_root: . +source: + paths: [data/demonstrations.csv] +policy: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + return_conditioned: false +output_path: data/imported-demonstrations +episodes_per_shard: 64 ``` -A complete training configuration is in -[train-return-conditioned.yaml](example_inputs/configs/train-return-conditioned.yaml). -`sequifier make` also creates `configs/train-policy.yaml`. Define named -`trajectory_sources`, named `policies`, and one `training_plan` phase with -`algorithm: return_conditioned`. Set `return_conditioned: false` on the policy -for behavior cloning (BC). Offline training does not start an environment. - -Each source specifies either `paths: [episode-a.json, episode-b.json]` or -`manifest: data/trajectories/manifest.json`. Version-1 manifests contain an -`episodes` list of objects with `path` and optional `episode_id`. Only listed -files are selected. Run paths resolve relative to `project_root`; manifest -entries resolve relative to the manifest directory. - -Set `epochs` on the phase, `seed` and `device` on the run, and update settings -in `optimization`. Devices follow PyTorch's CPU, CUDA, and MPS conventions. -`optimization.num_workers` controls data-loader workers and -`optimization.episode_cache_size` bounds the number of cached file-backed -episodes per worker (default 8). The learner retains episode metadata and loads -episodes on demand. It shuffles episodes and the decisions within each episode -to keep disk access local. Normalization combines episode statistics -incrementally. JSON storage still requires an individual episode to fit in memory. - -Without `validation_sources`, the runner splits whole episodes deterministically, -keeping identical demonstrations together. At least two distinct trajectory -groups are required. To supply held-out data, declare a separate source and add -`validation_sources: [held_out]` to the phase. Duplicate paths and episode IDs, -and identical content across training and validation, are rejected. -Normalization fits only the training partition. - -The best model is selected by held-out action negative log likelihood (NLL). -`model_path` defaults to `models//.pt` and contains an -ordinary `sequifier_model` artifact. `actor_path` remains an alias for `model_path` for existing configurations. `report_path` -defaults to `outputs/training//report.json`; it records sources, -partition IDs, exclusions, decision counts, per-epoch NLL, and optimizer steps. +The output directory must be new. Import writes raw Parquet shards and a +checksummed `manifest.json`, then atomically publishes the directory. It does not +fit statistics, pad contexts or invent missing transitions. Train with +`data_sources.demonstrations.manifest: data/imported-demonstrations/manifest.json`; +the recorded column mappings are loaded automatically. The manifest retains +source provenance, episode identities, optional successors and collection groups. + +## Shared model and dataset authoring + +The compact `policies` surface is optional. Policy training also accepts the +ordinary `model.backbone` and `model.interfaces` schemas, with a `policy` data +schema on each interface. Ingestion and decoder components keep their existing +syntax; the policy adapter adds the previous-action/return features. Use top-level +`context_length`, or `global_training.context_length`, to select a context smaller +than the backbone maximum. `global_training` accepts the policy optimization +settings as an alternative to `optimization` and uses `training_objective: policy`. + +Likewise, `dataset_training.` accepts `model_interface` and `parts`, with each +part containing a sequence source specification. Reference it with the ordinary +`{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these +parts do not need fabricated criteria. Model/interface references are validated. +The shared phase schema preserves supervised defaults; this learner still supports +one `return_conditioned` phase, with BC selected on its policy. Mixed algorithms, +backbone repositories, image/depth policies and online learning remain planned. +Use `initialize_from` for backbone artifact transfer. + +See [train-policy-from-table.yaml](example_inputs/configs/train-policy-from-table.yaml) +for a complete example using the shared model, dataset and phase authoring surface. ## Configure the shared model @@ -2399,7 +2565,7 @@ have `support: 1` and `prediction_length: 1`. The internal input names `__policy_previous_action` and `__policy_return` are reserved. The latter exists only for return-conditioned policies. Static columns -are broadcast over the decision window. Real observation and static statistics +are broadcast over the decision window. Real observation and static sample statistics are persisted in the model interface's preprocessing metadata. Encoded categorical observations can be declared with `observation_n_classes: {state_id: 8}`; their recorded values must be integral IDs in `[0, 8)` and are not normalized. @@ -2407,8 +2573,9 @@ When authoring a mixed real/categorical embedding ingestion, provide `feature_embedding_dims` under the normal ingestion rules, including the internal previous-action column and, when enabled, return column. -Actions use the ordered IDs of `action_values`; an extra input-only ID equal to -the action count represents episode start. It is excluded from decoder outputs. +Actions use decoder-local IDs in the order of `action_values`, mapped to the +standard global category IDs. A separate input-only ID represents episode start +and is excluded from decoder outputs. An observation may have the same external name as the action; in that case the model uses `__policy_action` as its separate target name. Default decoders infer the target automatically. @@ -2469,7 +2636,8 @@ desired_return: 10.0 See [infer-policy.yaml](example_inputs/configs/infer-policy.yaml). CSV and Parquet inputs contain observations and any static columns in original units. Without a `sequenceId` column, each row is an independent first decision. Within each -sequence, row order defines decision order and static inputs must be constant. +sequence, `itemPosition` defines decision order when present and must be unique +and contiguous; otherwise row order is used. Static inputs must be constant. Provide the recorded `action_column` to derive previous actions by shifting one row, or set `previous_action_column` to a column containing previous **domain action values**, with null for episode start. Use a new sequence ID for each @@ -2485,7 +2653,7 @@ supplied by the caller; offline inference does not update budgets from rewards. BC requires no return input and rejects a scalar `desired_return`. `inference_batch_size` bounds batched model calls (default 64). Predictions go to `outputs/predictions//part-000.csv` by default. -`output_path` overrides that location. Sequence IDs and `itemPosition`, when +`output_path` overrides that location. Sequence IDs and the configured `item_position_column` (default `itemPosition`), when present, are retained in input order. `output_probabilities: true` adds one `__probability_` column per action. Set `deterministic: false` for seeded categorical sampling. Select `model_interface` diff --git a/src/sequifier/io/batch.py b/src/sequifier/io/batch.py index 76a2845a..1af074e8 100644 --- a/src/sequifier/io/batch.py +++ b/src/sequifier/io/batch.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from dataclasses import dataclass from typing import Optional @@ -8,6 +9,52 @@ REQUIRED_METADATA_KEYS = frozenset({"attention_valid_mask", "target_valid_mask"}) +@dataclass(frozen=True) +class SequifierInputs: + """Named model tensors, independent of labels or learning algorithm.""" + + inputs: dict[str, torch.Tensor] + metadata: dict[str, torch.Tensor] + + @property + def attention_valid_mask(self) -> torch.Tensor: + return self.metadata["attention_valid_mask"] + + def to(self, device: str | torch.device) -> "SequifierInputs": + return SequifierInputs( + { + key: value.to(device, non_blocking=True) + for key, value in self.inputs.items() + }, + { + key: value.to(device, non_blocking=True) + for key, value in self.metadata.items() + }, + ) + + +def collate_model_inputs(examples: Sequence[SequifierInputs]) -> SequifierInputs: + if not examples: + raise ValueError("cannot collate empty model inputs") + first = examples[0] + if any( + value.inputs.keys() != first.inputs.keys() + or value.metadata.keys() != first.metadata.keys() + for value in examples + ): + raise ValueError("model inputs must share feature and metadata schemas") + return SequifierInputs( + { + key: torch.cat([value.inputs[key] for value in examples]) + for key in first.inputs + }, + { + key: torch.cat([value.metadata[key] for value in examples]) + for key in first.metadata + }, + ) + + @dataclass(frozen=True) class SequifierBatch: inputs: dict[str, torch.Tensor] @@ -16,6 +63,25 @@ class SequifierBatch: sequence_ids: Optional[torch.Tensor] = None subsequence_ids: Optional[torch.Tensor] = None + @property + def model_inputs(self) -> SequifierInputs: + return SequifierInputs(self.inputs, self.metadata) + + def to(self, device: str | torch.device) -> "SequifierBatch": + model_inputs = self.model_inputs.to(device) + return SequifierBatch( + model_inputs.inputs, + { + key: value.to(device, non_blocking=True) + for key, value in self.targets.items() + }, + model_inputs.metadata, + self.sequence_ids.to(device) if self.sequence_ids is not None else None, + self.subsequence_ids.to(device) + if self.subsequence_ids is not None + else None, + ) + @beartype def __post_init__(self) -> None: metadata_keys = self.metadata.keys() if self.metadata is not None else set() @@ -43,3 +109,29 @@ def pin_memory(self) -> "SequifierBatch": else None ), ) + + +def collate_sequence_batches(examples: Sequence[SequifierBatch]) -> SequifierBatch: + model_inputs = collate_model_inputs([value.model_inputs for value in examples]) + first = examples[0] + if any(value.targets.keys() != first.targets.keys() for value in examples): + raise ValueError("sequence batches must share target schemas") + + def identities(name): + values = [getattr(value, name) for value in examples] + if all(value is None for value in values): + return None + if any(value is None for value in values): + raise ValueError(f"sequence batches must consistently supply {name}") + return torch.cat(values) + + return SequifierBatch( + model_inputs.inputs, + { + key: torch.cat([value.targets[key] for value in examples]) + for key in first.targets + }, + model_inputs.metadata, + identities("sequence_ids"), + identities("subsequence_ids"), + ) diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py new file mode 100644 index 00000000..54522712 --- /dev/null +++ b/src/sequifier/io/policy_dataset.py @@ -0,0 +1,466 @@ +"""Episode-safe policy views over ordinary sequences and imported rollout logs.""" + +import hashlib +import json +import random +from bisect import bisect_right +from collections import OrderedDict +from collections.abc import Sequence +from decimal import Decimal +from pathlib import Path +from typing import Literal, overload + +import polars as pl +import torch +from torch.utils.data import Dataset, Sampler + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.io.batch import SequifierBatch, collate_sequence_batches +from sequifier.io.policy_window import ( + PolicyWindowView, + action_global_ids, + action_index, + action_target, + encode_feature, +) +from sequifier.io.preprocessing import ( + _finite_mean_and_std, + create_id_map, + get_combined_statistics, +) +from sequifier.io.sequence import SequenceEpisode +from sequifier.io.values import numeric_scalar +from sequifier.trajectory.importers import ( + TrajectoryReference, + sequence_from_trajectory, + source_episodes, +) +from sequifier.trajectory.returns import return_to_go +from sequifier.trajectory.schema import Trajectory + +# Compatibility names now refer to the standard supervised batch contract. +PolicyExample = SequifierBatch +collate_policy_examples = collate_sequence_batches + + +def _load(source, config, action_semantics) -> SequenceEpisode: + if isinstance(source, SequenceEpisode): + return SequenceEpisode.model_validate(source.model_dump()) + if isinstance(source, Trajectory): + return sequence_from_trajectory(source, config, action_semantics) + if isinstance(source, (str, Path)): + return TrajectoryReference(Path(source), config, action_semantics).load() + return source.load() + + +class EpisodeCollection(Sequence[SequenceEpisode]): + """Keep source-backed episodes on disk, with a bounded per-worker cache.""" + + def __init__(self, sources, cache_size, config, action_semantics): + self.sources = list(sources) + self.cache_size = cache_size + self.config = config + self.action_semantics = action_semantics + self.cache: OrderedDict[int, SequenceEpisode] = OrderedDict() + + def __len__(self): + return len(self.sources) + + @overload + def __getitem__(self, index: int) -> SequenceEpisode: ... + + @overload + def __getitem__(self, index: slice) -> Sequence[SequenceEpisode]: ... + + def __getitem__( + self, index: int | slice + ) -> SequenceEpisode | Sequence[SequenceEpisode]: + if isinstance(index, slice): + return [self[i] for i in range(*index.indices(len(self)))] + if index not in self.cache: + self.cache[index] = _load( + self.sources[index], self.config, self.action_semantics + ) + if len(self.cache) > self.cache_size: + self.cache.popitem(last=False) + self.cache.move_to_end(index) + return self.cache[index] + + +def _canonical(value): + if isinstance(value, dict): + return {key: _canonical(item) for key, item in value.items()} + if isinstance(value, (tuple, list)): + return [_canonical(item) for item in value] + if isinstance(value, (int, float)) and not isinstance(value, bool): + return ["number", str(Decimal(str(value)).normalize())] + return value + + +def episode_content_key(episode, *, length=None, returns=None) -> str: + """Hash learning content, independently of log format, IDs and provenance.""" + if isinstance(episode, Trajectory): + values = { + "observations": [step.observation for step in episode.transitions], + "actions": [step.predictions for step in episode.transitions], + "static_inputs": episode.seed.static_inputs, + } + else: + length = len(episode.actions) if length is None else length + values = { + "observations": { + column: entries[:length] + for column, entries in episode.observations.items() + }, + "actions": episode.actions[:length], + "static_inputs": episode.static_inputs, + "previous_action": episode.previous_action, + "starts_episode": episode.starts_episode, + "returns": returns, + } + return hashlib.sha256( + json.dumps(_canonical(values), sort_keys=True, allow_nan=False).encode() + ).hexdigest() + + +def _return_labels(episode: SequenceEpisode, length: int, gamma: float): + if episode.transitions is None or episode.transitions.rewards is None: + raise ValueError("return-conditioned examples require recorded rewards") + return return_to_go(episode.transitions.rewards[:length], gamma) + + +class PolicySequenceDataset(Dataset): + """One decision per standard SequifierBatch; the storage format is incidental.""" + + def __init__( + self, + sources: Sequence, + config: CategoricalPolicyConfig, + *, + action_semantics: Literal["policy", "executed_predictions"] = "policy", + episode_cache_size: int = 8, + ): + if action_semantics not in {"policy", "executed_predictions"}: + raise ValueError("action_semantics must be policy or executed_predictions") + if episode_cache_size < 1: + raise ValueError("episode_cache_size must be positive") + self.config = config + self.action_semantics = action_semantics + self.episode_cache_size = episode_cache_size + self.excluded_episode_ids = [] + self.exclusions = {} + self.all_episode_ids = [] + self.episode_ids = [] + self.content_keys = [] + self.demonstration_keys = [] + self.group_ids = [] + self.lengths = [] + self.cumulative_lengths = [] + self._labels = OrderedDict() + eligible = [] + seen = set() + tables = [ + str(source) + for source in sources + if isinstance(source, (str, Path)) + and Path(source).suffix.lower() in {".csv", ".parquet"} + ] + expanded = [] + for source in sources: + if isinstance(source, SequenceSourceConfig): + expanded.extend(source_episodes(source, config)) + elif not (isinstance(source, (str, Path)) and str(source) in tables): + expanded.append(source) + if tables: + expanded.extend(source_episodes(SequenceSourceConfig(paths=tables), config)) + categorical = ( + set(config.categorical_columns) | config.categorical_n_classes.keys() + ) + for source in expanded: + episode = _load(source, config, action_semantics) + if episode.episode_id in seen: + raise ValueError(f"duplicate episode ID {episode.episode_id!r}") + seen.add(episode.episode_id) + self.all_episode_ids.append(episode.episode_id) + if set(episode.observations) != set(config.observation_columns) or set( + episode.static_inputs + ) != set(config.static_columns): + raise ValueError( + "sequence observations/static inputs do not match the policy schema" + ) + # Validate all labels, even in an excluded unknown return tail. + for value in episode.actions: + action_index(value, config) + if episode.previous_action is not None: + action_index(episode.previous_action, config) + for column, values in episode.observations.items(): + for value in values: + if column not in categorical: + numeric_scalar(value, column) + for column, value in episode.static_inputs.items(): + if column not in categorical: + numeric_scalar(value, column) + length = ( + episode.return_length(config.returns.finite_horizon) + if config.return_conditioned + else len(episode.actions) + ) + if length is None: + self.excluded_episode_ids.append(episode.episode_id) + self.exclusions[episode.episode_id] = ( + "missing_rewards" + if episode.transitions is None + or episode.transitions.rewards is None + else "unknown_return_tail" + ) + continue + returns = ( + _return_labels(episode, length, config.returns.gamma) + if config.return_conditioned + else None + ) + eligible.append(source) + self.episode_ids.append(episode.episode_id) + self.content_keys.append( + episode_content_key(episode, length=length, returns=returns) + ) + self.demonstration_keys.append(episode_content_key(episode, length=length)) + self.group_ids.append(episode.group_id) + self.lengths.append(length) + self.cumulative_lengths.append( + length + (self.cumulative_lengths[-1] if self.cumulative_lengths else 0) + ) + self.episodes = EpisodeCollection( + eligible, episode_cache_size, config, action_semantics + ) + + def __len__(self): + return self.cumulative_lengths[-1] if self.cumulative_lengths else 0 + + def fit_categorical_config(self) -> CategoricalPolicyConfig: + """Fit only declared raw categorical features, using the training episodes.""" + values = self.config.model_dump() + for column in self.config.categorical_columns: + if ( + column in self.config.id_maps + or column in self.config.categorical_n_classes + ): + continue + vocabulary = set() + for episode, length in zip(self.episodes, self.lengths): + vocabulary.update( + episode.observations[column][:length] + if column in episode.observations + else [episode.static_inputs[column]] + ) + if not vocabulary: + raise ValueError(f"cannot fit empty categorical column {column!r}") + if len({isinstance(value, str) for value in vocabulary}) > 1: + raise ValueError( + f"{column}: mixed string/numeric category labels are ambiguous" + ) + if any( + not isinstance(value, str) and value != int(value) + for value in vocabulary + ): + raise ValueError( + f"{column}: categorical numeric labels must be integers" + ) + mapping = create_id_map(pl.DataFrame({column: sorted(vocabulary)}), column) + values["id_maps"][column] = mapping + key = ( + "observation_n_classes" + if column in self.config.observation_columns + else "static_n_classes" + ) + values[key][column] = max(mapping.values()) + 1 + return CategoricalPolicyConfig.model_validate(values) + + def use_config(self, config: CategoricalPolicyConfig) -> None: + self.config = config + self.episodes.config = config + # Validate held-out values against the fixed training codec before updates. + for episode, length in zip(self.episodes, self.lengths): + for column, values in episode.observations.items(): + for value in values[:length]: + encode_feature(value, column, config) + for column, value in episode.static_inputs.items(): + encode_feature(value, column, config) + + def _episode_labels(self, index: int): + if index not in self._labels: + episode = self.episodes[index] + length = self.lengths[index] + actions = [ + action_index(value, self.config) for value in episode.actions[:length] + ] + returns = ( + _return_labels(episode, length, self.config.returns.gamma) + if self.config.return_conditioned + else None + ) + self._labels[index] = actions, returns + if len(self._labels) > self.episode_cache_size: + self._labels.popitem(last=False) + self._labels.move_to_end(index) + return self._labels[index] + + def __getitem__(self, index: int) -> SequifierBatch: + if index < 0: + index += len(self) + if not 0 <= index < len(self): + raise IndexError(index) + episode_index = bisect_right(self.cumulative_lengths, index) + step = index - ( + self.cumulative_lengths[episode_index - 1] if episode_index else 0 + ) + episode = self.episodes[episode_index] + action_ids, returns = self._episode_labels(episode_index) + start = max(0, step + 1 - self.config.context_length) + initial = ( + len(self.config.action_values) + if episode.starts_episode + else action_index(episode.previous_action, self.config) + ) + previous = ([initial] if start == 0 else []) + action_ids[ + max(0, start - 1) : step + ] + inputs = PolicyWindowView(self.config).inputs( + [episode.observation(position) for position in range(start, step + 1)], + previous, + returns[start : step + 1] if returns is not None else None, + episode.static_inputs, + ) + targets = torch.zeros((1, self.config.context_length), dtype=torch.int64) + targets[0, -1] = action_global_ids(self.config)[action_ids[step]] + metadata = dict(inputs.metadata) + metadata["target_valid_mask"] = torch.zeros_like(inputs.attention_valid_mask) + metadata["target_valid_mask"][0, -1] = True + return SequifierBatch( + inputs.inputs, + {action_target(self.config): targets}, + metadata, + torch.tensor([episode_index], dtype=torch.int64), + torch.tensor([episode.item_positions[step]], dtype=torch.int64), + ) + + def normalization_statistics(self): + """Use the same stable sample moments as supervised preprocessing.""" + moments = {} + columns = self.config.observation_columns + self.config.static_columns + for episode, length in zip(self.episodes, self.lengths): + for column in columns: + if column in self.config.categorical_n_classes: + continue + values = ( + episode.observations[column][:length] + if column in episode.observations + else [episode.static_inputs[column]] + ) + mean, std = _finite_mean_and_std(pl.Series(column, values), column) + size = len(values) + if column in moments: + count, old_mean, old_std = moments[column] + mean, std = get_combined_statistics( + size, mean, std, count, old_mean, old_std + ) + size += count + moments[column] = size, mean, std + return { + column: {"mean": mean, "std": std} + for column, (_, mean, std) in moments.items() + } + + def normalization_rows(self): + observations, static = [], [] + for episode, length in zip(self.episodes, self.lengths): + observations += [ + [ + encode_feature( + episode.observations[column][position], column, self.config + ) + for column in self.config.observation_columns + ] + for position in range(length) + ] + static.append( + [ + encode_feature(episode.static_inputs[column], column, self.config) + for column in self.config.static_columns + ] + ) + return torch.tensor(observations, dtype=torch.float32), torch.tensor( + static, dtype=torch.float32 + ) + + +class EpisodeWindowSampler(Sampler): + """Shuffle episodes and their decisions while keeping disk access local.""" + + def __init__(self, dataset, generator): + self.dataset, self.generator = dataset, generator + + def __len__(self): + return len(self.dataset) + + def __iter__(self): + for episode in torch.randperm( + len(self.dataset.lengths), generator=self.generator + ).tolist(): + offset = self.dataset.cumulative_lengths[episode - 1] if episode else 0 + for step in torch.randperm( + self.dataset.lengths[episode], generator=self.generator + ).tolist(): + yield offset + step + + +def split_by_episode( + dataset, validation_fraction=0.2, seed=1010, *, group_identical=False +): + if not 0 < validation_fraction < 1 or len(dataset.episodes) < 2: + raise ValueError( + "episode splitting requires at least two eligible episodes and validation_fraction in (0,1)" + ) + # Union both collection groups and repeated content, including transitive ties. + parents = list(range(len(dataset.episodes))) + + def root(index): + while parents[index] != index: + parents[index] = parents[parents[index]] + index = parents[index] + return index + + seen = {} + for index in range(len(parents)): + keys = ( + [("group", dataset.group_ids[index])] + if dataset.group_ids[index] is not None + else [] + ) + if group_identical: + keys.append(("content", dataset.demonstration_keys[index])) + for key in keys: + if key in seen: + parents[root(index)] = root(seen[key]) + seen[key] = index + groups = {} + for index, source in enumerate(dataset.episodes.sources): + groups.setdefault(root(index), []).append(source) + partitions = list(groups.values()) + if len(partitions) < 2: + raise ValueError("splitting requires at least two independent episode groups") + random.Random(seed).shuffle(partitions) + count = min(len(partitions) - 1, max(1, int(len(partitions) * validation_fraction))) + return tuple( + PolicySequenceDataset( + [source for group in partition for source in group], + dataset.config, + action_semantics=dataset.action_semantics, + episode_cache_size=dataset.episode_cache_size, + ) + for partition in (partitions[count:], partitions[:count]) + ) + + +TrajectoryDataset = PolicySequenceDataset diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py new file mode 100644 index 00000000..1ffc700e --- /dev/null +++ b/src/sequifier/io/policy_window.py @@ -0,0 +1,158 @@ +"""Policy alignment over the shared named-tensor and model-window contracts.""" + +from collections.abc import Mapping, Sequence + +import torch + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view +from sequifier.io.batch import SequifierInputs +from sequifier.io.values import domain_fields, domain_scalar, numeric_scalar +from sequifier.special_tokens import SPECIAL_TOKEN_IDS + +PREVIOUS_ACTION = "__policy_previous_action" +DESIRED_RETURN = "__policy_return" + + +def action_index(value, config: CategoricalPolicyConfig) -> int: + value = domain_scalar(value, "recorded action") + try: + return config.action_values.index(value) + except ValueError as exc: + raise ValueError( + f"action {value!r} is outside action_values {config.action_values!r}" + ) from exc + + +def action_global_ids(config: CategoricalPolicyConfig) -> list[int]: + offset = ( + SPECIAL_TOKEN_IDS.user_start if config.categorical_encoding == "global" else 0 + ) + return list(range(offset, offset + len(config.action_values))) + + +def action_target(config: CategoricalPolicyConfig) -> str: + names = config.observation_columns + config.static_columns + return "__policy_action" if config.action_column in names else config.action_column + + +def encode_feature(value, column: str, config: CategoricalPolicyConfig): + mapping = config.id_maps.get(column) + if mapping is not None: + try: + # JSON object keys are strings; preprocessing accepts both forms. + value = mapping[value] if value in mapping else mapping[str(value)] + except (KeyError, TypeError) as exc: + unknown = SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.unknown] + if unknown not in mapping: + if ( + config.categorical_encoding == "global" + and SPECIAL_TOKEN_IDS.unknown not in mapping.values() + ): + return SPECIAL_TOKEN_IDS.unknown + raise ValueError( + f"{column}: value {value!r} is outside the fixed vocabulary" + ) from exc + value = mapping[unknown] + count = config.categorical_n_classes.get(column) + if count is not None: + domain_scalar(value, column) + if isinstance(value, str) or value != int(value) or not 0 <= value < count: + raise ValueError(f"{column}: categorical ID must be in [0, {count})") + return int(value) + return numeric_scalar(value, column) + + +class PolicyWindowView: + """At decision t, expose o_t and a_(t-1), and supervise a_t separately. + + Previous actions are decoder-local IDs at this boundary. They are converted + to the persisted input vocabulary here; the start token is input-only. + """ + + def __init__(self, config: CategoricalPolicyConfig): + self.config = config + self.view = ModelWindowView(config.context_length, "policy", 0) + self.resolved = resolve_window_view( + StoredWindowLayout(config.context_length, 0, 2), self.view + ) + + def inputs( + self, + observations: Sequence[Mapping], + previous_actions: Sequence[int], + desired_returns: Sequence[float] | None, + static_inputs: Mapping, + ) -> SequifierInputs: + config = self.config + length = len(observations) + if not length or len(previous_actions) != length: + raise ValueError( + "policy observations and previous actions must have matching nonempty lengths" + ) + if config.return_conditioned: + if desired_returns is None or len(desired_returns) != length: + raise ValueError( + "return-conditioned windows require one return per decision" + ) + elif desired_returns is not None: + raise ValueError("behavior cloning must not consume return inputs") + start = max(0, length - config.context_length) + padding = config.context_length - (length - start) + rows = [ + domain_fields(row, set(config.observation_columns), "policy observation") + for row in observations[start:] + ] + static = domain_fields( + static_inputs, set(config.static_columns), "policy static inputs" + ) + features = {} + for column in config.observation_columns + config.static_columns: + values = ( + [row[column] for row in rows] + if column in config.observation_columns + else [static[column]] * len(rows) + ) + features[column] = torch.tensor( + [ + [0] * padding + + [encode_feature(value, column, config) for value in values] + ], + dtype=torch.int64 + if column in config.categorical_n_classes + else torch.float32, + ) + ids = action_global_ids(config) + bos = ids[-1] + 1 + preceding = [] + for value in previous_actions[start:]: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 0 <= value <= len(ids) + ): + raise ValueError( + "previous actions require decoder IDs or the explicit start token" + ) + preceding.append(bos if value == len(ids) else ids[value]) + features[PREVIOUS_ACTION] = torch.tensor( + [[bos] * padding + preceding], dtype=torch.int64 + ) + if desired_returns is not None: + features[DESIRED_RETURN] = torch.tensor( + [ + [0.0] * padding + + [ + numeric_scalar(value, "desired_return") + for value in desired_returns[start:] + ] + ], + dtype=torch.float32, + ) + if any( + value.is_floating_point() and not torch.isfinite(value).all() + for value in features.values() + ): + raise ValueError("policy inputs exceed the finite float32 range") + metadata = self.resolved.build_masks(torch.tensor([padding], dtype=torch.int64)) + return SequifierInputs(features, metadata) diff --git a/src/sequifier/io/preprocessing.py b/src/sequifier/io/preprocessing.py new file mode 100644 index 00000000..39099a8e --- /dev/null +++ b/src/sequifier/io/preprocessing.py @@ -0,0 +1,121 @@ +"""Shared categorical vocabulary and stable sample-statistics fitting.""" + +import math +import warnings +from typing import Union + +import numpy as np +import polars as pl + +from sequifier.special_tokens import SPECIAL_TOKEN_IDS +from sequifier.typechecking import beartype + + +@beartype +def _finite_mean_and_std(column: pl.Series, column_name: str) -> tuple[float, float]: + """Compute sample statistics without overflowing intermediate squares.""" + values = column.to_numpy().astype(np.float64, copy=False) + if values.size == 0: + raise ValueError(f"Cannot compute statistics for empty column {column_name!r}.") + if not np.isfinite(values).all(): + raise ValueError(f"Column {column_name!r} contains non-finite values.") + + scale = float(np.max(np.abs(values))) + if scale == 0.0: + return 0.0, 0.0 + + scaled_values = values / scale + mean = float(np.mean(scaled_values) * scale) + std = 0.0 if values.size <= 1 else float(np.std(scaled_values, ddof=1) * scale) + if not math.isfinite(mean) or not math.isfinite(std): + raise ValueError( + f"Column {column_name!r} has a numeric range too large for finite " + "normalization statistics." + ) + return mean, std + + +@beartype +def get_combined_statistics( + n1: int, mean1: float, std1: float, n2: int, mean2: float, std2: float +) -> tuple[float, float]: + """Combine two mean/std summaries.""" + if n1 == 0: + return mean2, std2 + if n2 == 0: + return mean1, std1 + + scale = max(abs(mean1), abs(mean2), abs(std1), abs(std2)) + if not math.isfinite(scale): + raise ValueError("Cannot combine non-finite normalization statistics.") + if scale == 0.0: + return 0.0, 0.0 + + scaled_mean1 = mean1 / scale + scaled_mean2 = mean2 / scale + scaled_std1 = std1 / scale + scaled_std2 = std2 / scale + combined_mean_scaled = (n1 * scaled_mean1 + n2 * scaled_mean2) / (n1 + n2) + combined_mean = combined_mean_scaled * scale + + if n1 + n2 <= 1: + return combined_mean, 0.0 + + sum_of_squares1 = (n1 - 1) * scaled_std1**2 + n1 * ( + scaled_mean1 - combined_mean_scaled + ) ** 2 + sum_of_squares2 = (n2 - 1) * scaled_std2**2 + n2 * ( + scaled_mean2 - combined_mean_scaled + ) ** 2 + + combined_std = ( + math.sqrt((sum_of_squares1 + sum_of_squares2) / (n1 + n2 - 1)) * scale + ) + if not math.isfinite(combined_mean) or not math.isfinite(combined_std): + raise ValueError( + "Combined numeric range is too large for finite normalization statistics." + ) + + return combined_mean, combined_std + + +@beartype +def create_id_map(data: pl.DataFrame, column: str) -> dict[Union[str, int], int]: + """Map sorted user values to IDs after reserved tokens.""" + ids = sorted( + [int(x) if not isinstance(x, str) else x for x in np.unique(data[column])] + ) # type: ignore + + if isinstance(ids[0], str): + if SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.mask] in ids: + raise ValueError( + f"Found value '{SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.mask]}' in {column}, this is invalid" + ) + + for special_val in [ + SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.unknown], + SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.other], + ]: + if special_val in ids: + warnings.warn( + f"Found special value {special_val} in {column}, these will be combined with the sequifier-internal special value {special_val}" + ) + ids = [ + id_ + for id_ in ids + if id_ + not in [ + SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.unknown], + SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.other], + ] + ] + id_map = {id_: i + SPECIAL_TOKEN_IDS.user_start for i, id_ in enumerate(ids)} + id_map[SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.unknown]] = ( + SPECIAL_TOKEN_IDS.unknown + ) + id_map[SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.other]] = ( + SPECIAL_TOKEN_IDS.other + ) + else: + id_map = {id_: i + SPECIAL_TOKEN_IDS.user_start for i, id_ in enumerate(ids)} + return dict(id_map) diff --git a/src/sequifier/io/sequence.py b/src/sequifier/io/sequence.py new file mode 100644 index 00000000..4f3ccd2f --- /dev/null +++ b/src/sequifier/io/sequence.py @@ -0,0 +1,147 @@ +"""Columnar demonstrations with optional transition capabilities. + +These records describe data, independently of the simulator, collection context +length, prediction bindings, preprocessing or the algorithm that will consume it. +""" + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictBool, + StrictInt, + model_validator, +) + +from sequifier.io.values import DomainValue + + +def ordered_sequence_rows(frame, position_column="itemPosition"): + """Share explicit temporal ordering and gap validation across table consumers.""" + if position_column not in frame.columns: + return frame + positions = frame[position_column] + if not positions.dtype.is_integer() or positions.null_count(): + raise ValueError(f"{position_column} must contain non-null integers") + frame = frame.sort(position_column) + values = frame[position_column].to_list() + if values and ( + values[0] < 0 + or any(right != left + 1 for left, right in zip(values, values[1:])) + ): + raise ValueError( + f"{position_column} must be nonnegative, unique and contiguous within a sequence" + ) + return frame + + +class TransitionColumns(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + rewards: tuple[float, ...] | None = None + terminated: tuple[StrictBool, ...] | None = None + truncated: tuple[StrictBool, ...] | None = None + # None at a position means the true successor was not observed. Never a reset. + successors: tuple[dict[str, DomainValue] | None, ...] | None = None + bootstrap_allowed: tuple[StrictBool, ...] | None = None + + +class SequenceEpisode(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + episode_id: str = Field(min_length=1) + item_positions: tuple[StrictInt, ...] = Field(min_length=1) + observations: dict[str, tuple[DomainValue, ...]] = Field(min_length=1) + actions: tuple[DomainValue, ...] + static_inputs: dict[str, DomainValue] = Field(default_factory=dict) + starts_episode: StrictBool = True + previous_action: DomainValue | None = None + group_id: str | None = None + transitions: TransitionColumns | None = None + + @model_validator(mode="after") + def validate_sequence(self): + length = len(self.item_positions) + if self.item_positions[0] < 0 or any( + right != left + 1 + for left, right in zip(self.item_positions, self.item_positions[1:]) + ): + raise ValueError( + "itemPosition must be nonnegative, unique and contiguous within an episode" + ) + if len(self.actions) != length or any( + len(values) != length for values in self.observations.values() + ): + raise ValueError( + "sequence feature/action lengths must match item positions" + ) + if set(self.observations) & set(self.static_inputs): + raise ValueError("static and temporal features must be disjoint") + if self.starts_episode: + if self.item_positions[0] != 0 or self.previous_action is not None: + raise ValueError( + "episode starts require position zero and no preceding action" + ) + elif self.previous_action is None: + raise ValueError("fragments require an explicit preceding action") + transitions = self.transitions + if transitions is None: + return self + for field in type(transitions).model_fields: + values = getattr(transitions, field) + if values is not None and len(values) != length: + raise ValueError(f"transition {field} must match sequence length") + for index in range(length): + terminated = ( + transitions.terminated is not None and transitions.terminated[index] + ) + truncated = ( + transitions.truncated is not None and transitions.truncated[index] + ) + if (terminated or truncated) and index != length - 1: + raise ValueError( + "a sequence cannot contain a decision after an episode boundary" + ) + successor = ( + transitions.successors[index] + if transitions.successors is not None + else None + ) + if successor is not None: + if successor.keys() != self.observations.keys(): + raise ValueError( + "successor observation fields must match observations" + ) + if index + 1 < length and successor != self.observation(index + 1): + raise ValueError("explicit successor breaks sequence continuity") + bootstrap = ( + transitions.bootstrap_allowed[index] + if transitions.bootstrap_allowed is not None + else False + ) + if terminated and bootstrap: + raise ValueError("task termination must disable bootstrap") + if bootstrap and index + 1 == length and successor is None: + raise ValueError( + "bootstrapped final decisions require an observed successor" + ) + return self + + def observation(self, index: int) -> dict: + return {column: values[index] for column, values in self.observations.items()} + + def return_length(self, finite_horizon: int | None) -> int | None: + if self.transitions is None or self.transitions.rewards is None: + return None + length = len(self.actions) + if finite_horizon is not None and ( + self.starts_episode or self.item_positions[0] > 0 + ): + remaining = finite_horizon - self.item_positions[0] + if remaining <= 0: + return None + if length >= remaining: + return remaining + if self.transitions.terminated is not None and self.transitions.terminated[-1]: + return length + return None diff --git a/src/sequifier/io/values.py b/src/sequifier/io/values.py new file mode 100644 index 00000000..ebdb8e93 --- /dev/null +++ b/src/sequifier/io/values.py @@ -0,0 +1,49 @@ +"""Domain scalars shared by table import, preprocessing and environment adapters.""" + +import math +from collections.abc import Mapping +from numbers import Integral, Real +from typing import Annotated, TypeAlias + +from pydantic import BeforeValidator, FiniteFloat, StrictInt, StrictStr + +DomainValue: TypeAlias = Annotated[ + StrictInt | FiniteFloat | StrictStr, + BeforeValidator(lambda value: domain_scalar(value, "domain value")), +] + + +def numeric_scalar(value, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{label} must be a finite numeric scalar, got {value!r}") + try: + number = float(value) + except (ValueError, OverflowError) as exc: + raise ValueError(f"{label} is outside the supported numeric range") from exc + if not math.isfinite(number): + raise ValueError(f"{label} must be finite, got {value!r}") + return number + + +def domain_scalar(value, label: str): + if isinstance(value, str): + return value + number = numeric_scalar(value, label) + return int(value) if isinstance(value, Integral) else number + + +def domain_fields(values, expected: set[str], label: str) -> dict: + if not isinstance(values, Mapping) or set(values) != expected: + raise ValueError( + f"{label} must supply exactly these fields: {sorted(expected)}" + ) + return { + key: domain_scalar(values[key], f"{label}.{key}") for key in sorted(expected) + } + + +def numeric_fields(values, expected: set[str], label: str) -> dict[str, float]: + return { + key: numeric_scalar(value, f"{label}.{key}") + for key, value in domain_fields(values, expected, label).items() + } diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 72bc935e..5bfdefc2 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -95,9 +95,9 @@ model_name: policy device: cpu seed: 1010 -trajectory_sources: +data_sources: demonstrations: - manifest: PLEASE_FILL + paths: [PLEASE_FILL.csv] policies: actor: observation_columns: [EXAMPLE_OBSERVATION] @@ -109,7 +109,7 @@ training_plan: algorithm: return_conditioned epochs: 80 - trajectory_sources: [demonstrations] + sources: [{source: demonstrations}] policy: actor optimization: batch_size: 64 diff --git a/src/sequifier/objectives.py b/src/sequifier/objectives.py index 07263d3a..dbe087ea 100644 --- a/src/sequifier/objectives.py +++ b/src/sequifier/objectives.py @@ -148,6 +148,34 @@ class CausalObjective(Objective): name = "causal" +class PolicyObjective(Objective): + """Causal attention with same-position action labels, never future labels.""" + + name = "policy" + forward_looking = False + + @classmethod + @beartype + def default_target_offset(cls) -> int: + return 0 + + @classmethod + @beartype + def validate_window_view(cls, context_length: int, target_offset: int) -> None: + if target_offset != 0: + raise ValueError("Policy views require target_offset=0") + + @classmethod + @beartype + def item_positions( + cls, start_positions: np.ndarray, context_length: int, prediction_length: int + ) -> np.ndarray: + return ( + start_positions[:, None] + + np.arange(context_length - prediction_length, context_length)[None, :] + ).reshape(-1) + + class FinalValueObjective(CausalObjective): name = "final_value" @@ -326,11 +354,14 @@ def item_positions( OBJECTIVE_REGISTRY: dict[str, type[Objective]] = { CausalObjective.name: CausalObjective, + PolicyObjective.name: PolicyObjective, BERTObjective.name: BERTObjective, FinalValueObjective.name: FinalValueObjective, NextOccurrenceObjective.name: NextOccurrenceObjective, } -OBJECTIVE_NAME_MESSAGE = "'causal', 'bert', 'final_value', and 'next_occurrence'" +OBJECTIVE_NAME_MESSAGE = ( + "'causal', 'bert', 'policy', 'final_value', and 'next_occurrence'" +) ALLOWED_OBJECTIVE_NAMES = frozenset(OBJECTIVE_REGISTRY) diff --git a/src/sequifier/preprocess.py b/src/sequifier/preprocess.py index 8bfd013b..2a994571 100644 --- a/src/sequifier/preprocess.py +++ b/src/sequifier/preprocess.py @@ -37,6 +37,11 @@ read_data, ) from sequifier.io.images import _time_ms, validate_sources +from sequifier.io.preprocessing import ( + _finite_mean_and_std, + create_id_map, + get_combined_statistics, +) from sequifier.io.pt_payload import StoredTensorBatch, load_pt_payload, save_pt_payload from sequifier.io.sample_order import ( CURRICULUM_COLUMN_PREFIX, @@ -784,7 +789,14 @@ def preprocess(args: Any, args_config: dict[str, Any]) -> None: logger.info("--- Starting Preprocessing ---") config_path = args.config_path or "configs/preprocess.yaml" config = load_preprocessor_config(config_path, args_config) - Preprocessor(**config.dict()) + from sequifier.config.sequence_import_config import SequenceImportConfig + + if isinstance(config, SequenceImportConfig): + from sequifier.trajectory.importers import import_sequences + + import_sequences(config) + else: + Preprocessor(**config.dict()) logger.info("--- Preprocessing Complete ---") @@ -3818,30 +3830,6 @@ def _get_column_statistics( return id_maps, selected_columns_statistics -@beartype -def _finite_mean_and_std(column: pl.Series, column_name: str) -> tuple[float, float]: - """Compute sample statistics without overflowing intermediate squares.""" - values = column.to_numpy().astype(np.float64, copy=False) - if values.size == 0: - raise ValueError(f"Cannot compute statistics for empty column {column_name!r}.") - if not np.isfinite(values).all(): - raise ValueError(f"Column {column_name!r} contains non-finite values.") - - scale = float(np.max(np.abs(values))) - if scale == 0.0: - return 0.0, 0.0 - - scaled_values = values / scale - mean = float(np.mean(scaled_values) * scale) - std = 0.0 if values.size <= 1 else float(np.std(scaled_values, ddof=1) * scale) - if not math.isfinite(mean) or not math.isfinite(std): - raise ValueError( - f"Column {column_name!r} has a numeric range too large for finite " - "normalization statistics." - ) - return mean, std - - @beartype def _validate_sequence_coordinates( data: pl.DataFrame, source: str, sort_output: bool = True @@ -4421,92 +4409,6 @@ def _create_preprocess_pool(processes: int) -> Any: os.environ[name] = value -@beartype -def get_combined_statistics( - n1: int, mean1: float, std1: float, n2: int, mean2: float, std2: float -) -> tuple[float, float]: - """Combine two mean/std summaries.""" - if n1 == 0: - return mean2, std2 - if n2 == 0: - return mean1, std1 - - scale = max(abs(mean1), abs(mean2), abs(std1), abs(std2)) - if not math.isfinite(scale): - raise ValueError("Cannot combine non-finite normalization statistics.") - if scale == 0.0: - return 0.0, 0.0 - - scaled_mean1 = mean1 / scale - scaled_mean2 = mean2 / scale - scaled_std1 = std1 / scale - scaled_std2 = std2 / scale - combined_mean_scaled = (n1 * scaled_mean1 + n2 * scaled_mean2) / (n1 + n2) - combined_mean = combined_mean_scaled * scale - - if n1 + n2 <= 1: - return combined_mean, 0.0 - - sum_of_squares1 = (n1 - 1) * scaled_std1**2 + n1 * ( - scaled_mean1 - combined_mean_scaled - ) ** 2 - sum_of_squares2 = (n2 - 1) * scaled_std2**2 + n2 * ( - scaled_mean2 - combined_mean_scaled - ) ** 2 - - combined_std = ( - math.sqrt((sum_of_squares1 + sum_of_squares2) / (n1 + n2 - 1)) * scale - ) - if not math.isfinite(combined_mean) or not math.isfinite(combined_std): - raise ValueError( - "Combined numeric range is too large for finite normalization statistics." - ) - - return combined_mean, combined_std - - -@beartype -def create_id_map(data: pl.DataFrame, column: str) -> dict[Union[str, int], int]: - """Map sorted user values to IDs after reserved tokens.""" - ids = sorted( - [int(x) if not isinstance(x, str) else x for x in np.unique(data[column])] - ) # type: ignore - - if isinstance(ids[0], str): - if SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.mask] in ids: - raise ValueError( - f"Found value '{SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.mask]}' in {column}, this is invalid" - ) - - for special_val in [ - SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.unknown], - SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.other], - ]: - if special_val in ids: - warnings.warn( - f"Found special value {special_val} in {column}, these will be combined with the sequifier-internal special value {special_val}" - ) - ids = [ - id_ - for id_ in ids - if id_ - not in [ - SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.unknown], - SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.other], - ] - ] - id_map = {id_: i + SPECIAL_TOKEN_IDS.user_start for i, id_ in enumerate(ids)} - id_map[SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.unknown]] = ( - SPECIAL_TOKEN_IDS.unknown - ) - id_map[SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.other]] = ( - SPECIAL_TOKEN_IDS.other - ) - else: - id_map = {id_: i + SPECIAL_TOKEN_IDS.user_start for i, id_ in enumerate(ids)} - return dict(id_map) - - @beartype def get_batch_limits( data: pl.DataFrame, n_batches: int, allow_sequence_splitting: bool diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py index c85350aa..6933dcde 100644 --- a/src/sequifier/rl/history.py +++ b/src/sequifier/rl/history.py @@ -7,7 +7,8 @@ from torch import Tensor from sequifier.config.policy_config import CategoricalPolicyConfig -from sequifier.rollout.history import numeric_fields, numeric_scalar +from sequifier.io.batch import SequifierInputs, collate_model_inputs +from sequifier.io.policy_window import PolicyWindowView, action_index # noqa: F401 @dataclass(frozen=True) @@ -30,24 +31,30 @@ def to(self, device: str | torch.device) -> "PolicyInputs": ) -def collate_policy_inputs(inputs: Sequence[PolicyInputs]) -> PolicyInputs: +def collate_policy_inputs(inputs: Sequence[PolicyInputs | SequifierInputs]): if not inputs: raise ValueError("cannot collate empty policy inputs") - conditioned = inputs[0].desired_returns is not None - if any((value.desired_returns is not None) != conditioned for value in inputs): + named = [value for value in inputs if isinstance(value, SequifierInputs)] + if named: + if len(named) != len(inputs): + raise ValueError("cannot mix named and legacy dense policy inputs") + return collate_model_inputs(named) + legacy = [value for value in inputs if isinstance(value, PolicyInputs)] + conditioned = legacy[0].desired_returns is not None + if any((value.desired_returns is not None) != conditioned for value in legacy): raise ValueError("policy batches cannot mix conditioning modes") return PolicyInputs( - observations=torch.cat([value.observations for value in inputs]), - previous_actions=torch.cat([value.previous_actions for value in inputs]), + observations=torch.cat([value.observations for value in legacy]), + previous_actions=torch.cat([value.previous_actions for value in legacy]), attention_valid_mask=torch.cat( - [value.attention_valid_mask for value in inputs] + [value.attention_valid_mask for value in legacy] ), - static_inputs=torch.cat([value.static_inputs for value in inputs]), + static_inputs=torch.cat([value.static_inputs for value in legacy]), desired_returns=( torch.cat( [ value.desired_returns - for value in inputs + for value in legacy if value.desired_returns is not None ] ) @@ -57,95 +64,14 @@ def collate_policy_inputs(inputs: Sequence[PolicyInputs]) -> PolicyInputs: ) -def action_index(value: float, config: CategoricalPolicyConfig) -> int: - value = numeric_scalar(value, "recorded action") - try: - return config.action_values.index(value) - except ValueError as exc: - raise ValueError( - f"action {value!r} is outside action_values {config.action_values!r}" - ) from exc - - def build_policy_inputs( config: CategoricalPolicyConfig, - observations: Sequence[Mapping[str, float]], + observations: Sequence[Mapping], previous_actions: Sequence[int], desired_returns: Sequence[float] | None, - static_inputs: Mapping[str, float], -) -> PolicyInputs: - """Only supplied decision prefixes are visible; targets are never an argument. - - At position t, observations[t] is o_t and previous_actions[t] is a_(t-1). - The action-count ID is the explicit start token, never an executable action. - """ - length = len(observations) - if not length or len(previous_actions) != length: - raise ValueError( - "policy observations and previous actions must have matching nonempty lengths" - ) - if config.return_conditioned and ( - desired_returns is None or len(desired_returns) != length - ): - raise ValueError( - "return-conditioned windows require one desired return per decision position" - ) - static = numeric_fields( - static_inputs, set(config.static_columns), "policy static inputs" - ) - context = config.context_length - start = max(0, length - context) - count = length - start - padding = context - count - feature_rows = [[0.0] * len(config.observation_columns) for _ in range(padding)] - for observation in observations[start:]: - values = numeric_fields( - observation, set(config.observation_columns), "policy observation" - ) - for column, count in config.observation_n_classes.items(): - value = values[column] - if value != int(value) or not 0 <= value < count: - raise ValueError( - f"{column}: observation must be a categorical ID in [0, {count})" - ) - feature_rows.append([values[column] for column in config.observation_columns]) - action_ids = [len(config.action_values)] * padding - for action in previous_actions[start:]: - if ( - isinstance(action, bool) - or not isinstance(action, int) - or not 0 <= action <= len(config.action_values) - ): - raise ValueError( - "previous actions must be categorical IDs or the start token" - ) - action_ids.append(action) - returns = None - if config.return_conditioned: - assert desired_returns is not None - returns = torch.tensor( - [ - [0.0] * padding - + [ - numeric_scalar(value, "desired_return") - for value in desired_returns[start:] - ] - ], - dtype=torch.float32, - ) - inputs = PolicyInputs( - observations=torch.tensor([feature_rows], dtype=torch.float32), - previous_actions=torch.tensor([action_ids], dtype=torch.int64), - attention_valid_mask=torch.tensor([[False] * padding + [True] * count]), - static_inputs=torch.tensor( - [[static[column] for column in config.static_columns]], dtype=torch.float32 - ), - desired_returns=returns, + static_inputs: Mapping, +) -> SequifierInputs: + """Compatibility entry point for the shared policy window view.""" + return PolicyWindowView(config).inputs( + observations, previous_actions, desired_returns, static_inputs ) - if ( - not torch.isfinite(inputs.observations).all() - or not torch.isfinite(inputs.static_inputs).all() - or (returns is not None and not torch.isfinite(returns).all()) - ): - raise ValueError("policy inputs exceed the finite float32 range") - return inputs diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 38f7852a..fec31d64 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -9,6 +9,8 @@ from sequifier.config.policy_infer_config import PolicyInferenceConfig from sequifier.helpers import configure_determinism, normalize_path, write_data +from sequifier.io.sequence import ordered_sequence_rows +from sequifier.io.values import domain_fields, domain_scalar from sequifier.rl.checkpoint import load_policy from sequifier.rl.distributions import CategoricalActionDistribution from sequifier.rl.history import ( @@ -18,7 +20,7 @@ ) from sequifier.rl.runtime import collect_policy_rollout from sequifier.rollout.factories import load_adapter -from sequifier.rollout.history import numeric_fields, numeric_scalar +from sequifier.rollout.history import numeric_scalar from sequifier.trajectory.schema import SeedHistory from sequifier.trajectory.store import save_trajectory @@ -68,12 +70,12 @@ def budget(row): seed = SeedHistory( observations={ column: [0.0] * (spec.context_length - 1) - + [numeric_scalar(row[column], column)] + + [domain_scalar(row[column], column)] for column in spec.observation_columns }, attention_valid_mask=[False] * (spec.context_length - 1) + [True], static_inputs={ - column: numeric_scalar(row[column], column) + column: domain_scalar(row[column], column) for column in spec.static_columns }, ) @@ -155,6 +157,16 @@ def flush(): with torch.no_grad(): for group in groups: + group = ordered_sequence_rows(group, config.item_position_column) + first_position = ( + group[config.item_position_column][0] + if config.item_position_column in group.columns + else 0 + ) + if first_position > 0 and config.previous_action_column is None: + raise ValueError( + "fragment inference requires an explicit preceding action" + ) observations, previous_actions, budgets = ( deque(maxlen=spec.context_length) for _ in range(3) ) @@ -177,7 +189,7 @@ def flush(): "previous_action_column is required when observations and actions share a name" ) for position, row in enumerate(group.iter_rows(named=True)): - static = numeric_fields( + static = domain_fields( {column: row[column] for column in spec.static_columns}, set(spec.static_columns), "policy static inputs", @@ -192,7 +204,7 @@ def flush(): ) if config.previous_action_column is not None: value = row[config.previous_action_column] - if position > 0 and value is None: + if (position > 0 or first_position > 0) and value is None: raise ValueError( "null previous actions are allowed only at sequence start" ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 6293ed5e..2468a904 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -3,7 +3,7 @@ import json from dataclasses import dataclass, field from pathlib import Path -from typing import Literal, Protocol +from typing import Protocol from sequifier.artifacts.run_checkpoint import checkpoint_path from sequifier.config.policy_config import PolicyTrainingConfig @@ -11,10 +11,10 @@ ResolvedPolicyTrainConfig, TrajectorySourceConfig, ) +from sequifier.io.policy_dataset import TrajectoryDataset, split_by_episode from sequifier.rl.checkpoint import save_policy from sequifier.rl.training import PolicyTrainingResult, train_policy -from sequifier.trajectory.dataset import TrajectoryDataset, split_by_episode -from sequifier.trajectory.store import load_trajectory +from sequifier.trajectory.importers import source_episodes, source_paths class AlgorithmRuntime(Protocol): @@ -25,33 +25,8 @@ def checkpoint(self) -> None: ... def trajectory_paths(source: TrajectorySourceConfig) -> list[Path]: - if source.manifest is None: - paths = [Path(path) for path in source.paths] - else: - manifest_path = Path(source.manifest) - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if not isinstance(manifest, dict) or manifest.get("version") != 1: - raise ValueError("trajectory manifest must have version 1") - episodes = manifest.get("episodes") - if not isinstance(episodes, list) or not episodes: - raise ValueError("trajectory manifest requires nonempty episodes") - paths = [] - for entry in episodes: - if ( - not isinstance(entry, dict) - or not isinstance(entry.get("path"), str) - or not entry["path"] - ): - raise ValueError("each manifest episode requires a nonempty path") - path = manifest_path.parent / entry["path"] - if "episode_id" in entry: - if load_trajectory(path).episode_id != entry["episode_id"]: - raise ValueError(f"manifest episode ID disagrees with {path}") - paths.append(path) - paths = [path.resolve() for path in paths] - if len(paths) != len(set(paths)): - raise ValueError("trajectory source contains duplicate paths") - return paths + """Compatibility alias for source path/manifest resolution.""" + return source_paths(source) @dataclass @@ -60,6 +35,7 @@ class ReturnConditionedRuntime: training: TrajectoryDataset = field(init=False) validation: TrajectoryDataset = field(init=False) excluded_episode_ids: list[str] = field(init=False) + exclusions: dict[str, str] = field(init=False) sources: dict[str, list[str]] = field(init=False) result: PolicyTrainingResult = field(init=False) @@ -75,6 +51,10 @@ def prepare(self) -> None: manifest = self.config.trajectory_sources[name].manifest if manifest is not None: protected_paths.add(Path(manifest).resolve()) + source = self.config.trajectory_sources[name] + for path in (source.metadata_config_path, source.static_data_path): + if path is not None: + protected_paths.add(Path(path).resolve()) output_paths = [ Path(self.config.actor_path), Path(self.config.report_path), @@ -89,26 +69,22 @@ def prepare(self) -> None: "trajectory files cannot appear in multiple selected sources" ) - # Validate identities incrementally, including episodes without exact returns. - ids = [load_trajectory(path).episode_id for path in flattened] - if len(ids) != len(set(ids)): - raise ValueError("duplicate episode IDs across trajectory sources") - def dataset(names: list[str]) -> TrajectoryDataset: - semantics: set[Literal["policy", "executed_predictions"]] = { - self.config.trajectory_sources[name].action_semantics for name in names - } - if len(semantics) != 1: - raise ValueError("selected sources must share action_semantics") return TrajectoryDataset( - [path for name in names for path in all_paths[name]], + [ + episode + for name in names + for episode in source_episodes( + self.config.trajectory_sources[name], spec + ) + ], spec, - action_semantics=next(iter(semantics)), episode_cache_size=self.config.optimization.episode_cache_size, ) corpus = dataset(phase.trajectory_sources) self.excluded_episode_ids = list(corpus.excluded_episode_ids) + self.exclusions = dict(corpus.exclusions) if phase.validation_sources is None: self.training, self.validation = split_by_episode( corpus, @@ -120,7 +96,24 @@ def dataset(names: list[str]) -> TrajectoryDataset: self.training = corpus self.validation = dataset(phase.validation_sources) self.excluded_episode_ids += self.validation.excluded_episode_ids - if set(self.training.content_keys) & set(self.validation.content_keys): + self.exclusions.update(self.validation.exclusions) + if set(corpus.all_episode_ids) & set(self.validation.all_episode_ids): + raise ValueError( + "duplicate episode IDs across training/validation sources" + ) + if not len(self.training) or not len(self.validation): + raise ValueError( + f"policy training needs nonempty eligible partitions; exclusions: {corpus.exclusions}" + ) + training_groups = set(self.training.group_ids) - {None} + validation_groups = set(self.validation.group_ids) - {None} + if training_groups & validation_groups: + raise ValueError( + "collection groups cannot cross training/validation partitions" + ) + if set(self.training.demonstration_keys) & set( + self.validation.demonstration_keys + ): raise ValueError( "identical trajectories cannot cross training/validation partitions" ) @@ -148,9 +141,11 @@ def evaluate(self) -> dict: "algorithm": "return_conditioned", "config": self.config.model_dump(mode="json"), "sources": self.sources, + "policy": result.policy.config.model_dump(mode="json"), "training_episode_ids": list(result.training_episode_ids), "validation_episode_ids": list(result.validation_episode_ids), "excluded_episode_ids": self.excluded_episode_ids, + "exclusions": self.exclusions, "training_decisions": len(self.training), "validation_decisions": len(self.validation), "best_epoch": result.best_epoch, diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 96c177a9..083a5e02 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -8,14 +8,19 @@ from sequifier.artifacts.model_config import resolved_config_from_model_config from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.io.batch import SequifierInputs +from sequifier.io.policy_window import ( + DESIRED_RETURN, + PREVIOUS_ACTION, + action_global_ids, + action_target, +) +from sequifier.model.execution_schema import ExecutionSchema from sequifier.model.factory import build_transformer_network from sequifier.model.ingestions import get_feature_embedding_dims from sequifier.rl.distributions import CategoricalActionDistribution from sequifier.rl.history import PolicyInputs -PREVIOUS_ACTION = "__policy_previous_action" -DESIRED_RETURN = "__policy_return" - def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu"): """Resolve policy features through the same execution schema as forecasts. @@ -26,16 +31,18 @@ def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu" columns = config.observation_columns + config.static_columns + [PREVIOUS_ACTION] if config.return_conditioned: columns.append(DESIRED_RETURN) - target = ( - "__policy_action" if config.action_column in columns else config.action_column - ) - categorical = list(config.observation_n_classes) + [PREVIOUS_ACTION, target] + target = action_target(config) + categorical = list(config.categorical_n_classes) + [PREVIOUS_ACTION, target] real = [column for column in columns if column not in categorical] - count = len(config.action_values) + decoder_ids = action_global_ids(config) + action_width = decoder_ids[-1] + 1 + lookup = [-1] * action_width + for local_id, global_id in enumerate(decoder_ids): + lookup[global_id] = local_id n_classes = { - **config.observation_n_classes, - PREVIOUS_ACTION: count + 1, - target: count, + **config.categorical_n_classes, + PREVIOUS_ACTION: action_width + 1, + target: action_width, } categorical_inputs = [column for column in columns if column in categorical] width = max(config.backbone.dim_model, len(real) + 2 * len(categorical_inputs)) @@ -50,12 +57,16 @@ def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu" }, } values = { - "training_objective": "causal", + "training_objective": "policy", "context_length": config.context_length, - "target_offset": 1, - "backbone": {"architecture": config.backbone.model_dump(mode="json")}, + "target_offset": 0, + "backbone": ( + config.backbone_component.model_dump(mode="json") + if config.backbone_component is not None + else {"architecture": config.backbone.model_dump(mode="json")} + ), "interfaces": { - "policy": { + config.interface_name: { "policy": config.model_dump(mode="json"), "input_columns": columns, "target_columns": [target], @@ -67,8 +78,15 @@ def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu" "real_columns": real, "target_column_types": {target: "categorical"}, "n_classes": n_classes, - "target_decoder_ids": {target: list(range(count))}, - "target_global_to_decoder": {target: list(range(count))}, + "target_decoder_ids": {target: decoder_ids}, + "target_global_to_decoder": {target: lookup}, + "id_maps": { + **config.id_maps, + target: { + str(value): global_id + for value, global_id in zip(config.action_values, decoder_ids) + }, + }, "normalize_real_columns": True, "selected_columns_statistics": { column: {"mean": 0.0, "std": 1.0} @@ -105,7 +123,7 @@ def __init__(self, config: CategoricalPolicyConfig, *, device: str = "cpu"): self.network = build_transformer_network( self.execution_config, device=torch.device(device) ).network - self.interface_name = "policy" + self.interface_name = config.interface_name @classmethod def from_network(cls, network, execution_config, interface_name: str): @@ -147,16 +165,16 @@ def set_normalization_statistics( ) -> None: expected = ( set(self.config.observation_columns + self.config.static_columns) - - self.config.observation_n_classes.keys() + - self.config.categorical_n_classes.keys() ) if set(statistics) != expected: raise ValueError("normalization statistics must match real policy columns") fitted = {} for column, values in statistics.items(): mean, std = float(values["mean"]), float(values["std"]) - if not math.isfinite(mean) or not math.isfinite(std) or std <= 0: + if not math.isfinite(mean) or not math.isfinite(std) or std < 0: raise ValueError( - "policy normalization requires finite means and positive scales" + "policy normalization requires finite means and nonnegative scales" ) fitted[column] = {"mean": mean, "std": std} self.interface.selected_columns_statistics = fitted @@ -178,19 +196,24 @@ def set_normalization(self, observations: Tensor, static_inputs: Tensor) -> None "normalization requires finite rows matching policy columns" ) for index, column in enumerate(columns): - if column not in self.config.observation_n_classes: - fitted = values[:, index].to(device="cpu", dtype=torch.float64) - statistics[column] = { - "mean": float(fitted.mean()), - "std": float(fitted.std(unbiased=False).clamp_min(1e-6)), - } + if column not in self.config.categorical_n_classes: + import polars as pl + + from sequifier.io.preprocessing import _finite_mean_and_std + + fitted = ( + values[:, index].detach().to(device="cpu", dtype=torch.float64) + ) + mean, std = _finite_mean_and_std( + pl.Series(column, fitted.numpy()), column + ) + statistics[column] = {"mean": mean, "std": std} self.set_normalization_statistics(statistics) - def model_inputs( - self, inputs: PolicyInputs - ) -> tuple[dict[str, Tensor], dict[str, Tensor]]: - """Translate aligned raw histories into standard named model tensors.""" - self.validate_inputs(inputs) + def named_inputs(self, inputs: PolicyInputs | SequifierInputs) -> SequifierInputs: + if isinstance(inputs, SequifierInputs): + return inputs + # Older Python callers can still supply the dense convenience object. features = { column: inputs.observations[:, :, index] for index, column in enumerate(self.config.observation_columns) @@ -203,40 +226,68 @@ def model_inputs( for index, column in enumerate(self.config.static_columns) } ) - for column, count in self.config.observation_n_classes.items(): + for column in self.config.categorical_n_classes: values = features[column] - if ( - (values != values.round()).any() - or (values < 0).any() - or (values >= count).any() - ): + if (values != values.round()).any(): raise ValueError( - f"{column}: observations must be categorical IDs in [0, {count})" + f"{column}: categorical observations must be integer IDs" ) features[column] = values.long() + ids = action_global_ids(self.config) + actions = inputs.previous_actions + if ( + actions.dtype != torch.int64 + or (actions < 0).any() + or (actions > len(ids)).any() + ): + raise ValueError("invalid previous action IDs") + lookup = torch.tensor(ids + [ids[-1] + 1], device=actions.device) + features[PREVIOUS_ACTION] = lookup[actions] + if inputs.desired_returns is not None: + features[DESIRED_RETURN] = inputs.desired_returns + return SequifierInputs( + features, {"attention_valid_mask": inputs.attention_valid_mask} + ) + + def model_inputs( + self, inputs: PolicyInputs | SequifierInputs + ) -> tuple[dict[str, Tensor], dict[str, Tensor]]: + """Encode raw named histories without mutating stored data or masks.""" + inputs = self.named_inputs(inputs) + self.validate_inputs(inputs) + features = dict(inputs.inputs) + mask = inputs.attention_valid_mask for column, statistics in self.interface.selected_columns_statistics.items(): - features[column] = (features[column] - statistics["mean"]) / ( - statistics["std"] + 1e-9 - ) - features[PREVIOUS_ACTION] = inputs.previous_actions + features[column] = ( + (features[column] - statistics["mean"]) / (statistics["std"] + 1e-9) + ).masked_fill(~mask, 0.0) if self.config.return_conditioned: - assert inputs.desired_returns is not None features[DESIRED_RETURN] = ( - inputs.desired_returns / self.config.returns.scale + features[DESIRED_RETURN] / self.config.returns.scale ) - return features, {"attention_valid_mask": inputs.attention_valid_mask} + return features, inputs.metadata - def forward(self, inputs: PolicyInputs) -> Tensor: + def forward(self, inputs: PolicyInputs | SequifierInputs) -> Tensor: features, metadata = self.model_inputs(inputs) logits = self.network( features, metadata, interface_name=self.interface_name ).logits[self.interface.target_columns[0]] - return logits.masked_fill(~inputs.attention_valid_mask[:, :, None], 0.0) + return logits.masked_fill( + ~inputs.attention_valid_mask[:, -logits.shape[1] :, None], 0.0 + ) - def distribution(self, inputs: PolicyInputs) -> CategoricalActionDistribution: + def distribution( + self, inputs: PolicyInputs | SequifierInputs + ) -> CategoricalActionDistribution: return CategoricalActionDistribution(self(inputs)) - def validate_inputs(self, inputs: PolicyInputs) -> None: + def validate_inputs(self, inputs: PolicyInputs | SequifierInputs) -> None: + inputs = self.named_inputs(inputs) + expected = set(self.interface.input_columns) + if set(inputs.inputs) != expected: + raise ValueError( + "policy inputs must match the persisted named feature schema" + ) mask = inputs.attention_valid_mask if ( mask.ndim != 2 @@ -251,44 +302,17 @@ def validate_inputs(self, inputs: PolicyInputs) -> None: raise ValueError( "policy histories require left padding followed by valid positions" ) - if inputs.observations.shape != ( - *mask.shape, - len(self.config.observation_columns), - ): - raise ValueError("policy observation shape does not match its schema") - if inputs.static_inputs.shape != ( - mask.shape[0], - len(self.config.static_columns), - ): - raise ValueError("policy static input shape does not match its schema") - actions = inputs.previous_actions - if ( - actions.shape != mask.shape - or actions.dtype != torch.int64 - or (actions < 0).any() - or (actions > len(self.config.action_values)).any() - ): - raise ValueError( - "policy previous actions must be valid int64 IDs, including the start token" - ) - numeric = [inputs.observations, inputs.static_inputs] - if self.config.return_conditioned: - if ( - inputs.desired_returns is None - or inputs.desired_returns.shape != mask.shape - ): - raise ValueError( - "policy requires a desired return at every decision position" - ) - numeric.append(inputs.desired_returns) + ExecutionSchema.from_interface( + self.interface, self.config.context_length + ).validate(inputs.inputs, inputs.metadata) if any( value.device != self.device - or value.dtype != torch.float32 - or not torch.isfinite(value).all() - for value in numeric + for value in [*inputs.inputs.values(), *inputs.metadata.values()] ): + raise ValueError("policy features and masks must be on the model device") + ids = action_global_ids(self.config) + allowed = torch.tensor(ids + [ids[-1] + 1], device=self.device) + if not torch.isin(inputs.inputs[PREVIOUS_ACTION], allowed).all(): raise ValueError( - "numeric policy inputs must be finite float32 tensors on the model device" + "previous actions must be executable action IDs or the input-only start token" ) - if mask.device != self.device or actions.device != self.device: - raise ValueError("policy masks and actions must be on the model device") diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index fcdab143..cb635d2e 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -5,12 +5,13 @@ import torch from sequifier.config.rollout_config import RolloutConfig +from sequifier.io.values import DomainValue, domain_fields from sequifier.rl.distributions import CategoricalActionDistribution from sequifier.rl.history import build_policy_inputs from sequifier.rl.policy import CategoricalPolicy from sequifier.rollout.contracts import EnvironmentAdapter, ResetRequest from sequifier.rollout.coordinator import collect_rollout -from sequifier.rollout.history import numeric_fields, numeric_scalar +from sequifier.rollout.history import numeric_scalar from sequifier.trajectory.returns import remaining_return from sequifier.trajectory.schema import ( SeedHistory, @@ -64,7 +65,7 @@ def reset(self, request: ResetRequest) -> None: self.episode_id = request.episode_id self.step_id = 0 - def __call__(self, history: SeedHistory) -> dict[str, float]: + def __call__(self, history: SeedHistory) -> dict[str, DomainValue]: if not self.active or self.pending is not None or self.ended: raise ValueError( "policy predictions require an active episode and an executed previous prediction" @@ -155,7 +156,7 @@ def validate_history(history): observation = { column: values[-1] for column, values in history.observations.items() } - numeric_fields(observation, set(spec.observation_columns), "policy observation") + domain_fields(observation, set(spec.observation_columns), "policy observation") # Validate the final successor too, even when no subsequent action is needed. inputs = build_policy_inputs( spec, diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 61d554ec..1b2fe874 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -23,15 +23,18 @@ from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig from sequifier.integration.callbacks import IntegrationManager from sequifier.integration.contexts import StepIdentity -from sequifier.rl.distributions import masked_action_likelihood -from sequifier.rl.policy import CategoricalPolicy -from sequifier.runtime.random_state import RandomState, RandomStateManager -from sequifier.training.optimization import OptimizationRuntime, UpdatePolicy -from sequifier.trajectory.dataset import ( +from sequifier.io.policy_dataset import ( EpisodeWindowSampler, TrajectoryDataset, collate_policy_examples, ) +from sequifier.model.factory import InterfaceRuntimeMetadata +from sequifier.objectives import PolicyObjective +from sequifier.rl.policy import CategoricalPolicy +from sequifier.runtime.random_state import RandomState, RandomStateManager +from sequifier.training.loss import LossService, PreparedBatch +from sequifier.training.optimization import OptimizationRuntime, UpdatePolicy +from sequifier.training.runtime import DatasetRuntime, _criterion_modules @dataclass(frozen=True) @@ -78,6 +81,13 @@ def train_policy( raise ValueError( "training and validation partitions must not share episode IDs" ) + fitted = training.fit_categorical_config() + training.use_config(fitted) + validation.use_config(fitted) + if run_config is not None: + policies = dict(run_config.policies) + policies[run_config.training_plan.phases[0].policy] = fitted + run_config = run_config.model_copy(update={"policies": policies}) # Seed model initialization and data order independently. device = run_config.device if run_config is not None else "cpu" cuda_index = torch.device(device).index @@ -110,6 +120,37 @@ def train_policy( ) policy.backbone.load_state_dict(source.backbone.state_dict(), strict=True) del source + dataset_config = policy.execution_config.dataset_training[policy.interface_name] + interface = policy.interface + dataset_runtime = DatasetRuntime( + name=policy.interface_name, + config=dataset_config, + interface_name=policy.interface_name, + objective=PolicyObjective(), + parts={}, + criteria=_criterion_modules(dataset_config, torch.device(device)), + loss_weights=None, + class_weights=None, + frozen_parameter_ids=frozenset(), + runtime_metadata=InterfaceRuntimeMetadata( + interface.target_decoder_ids, + interface.target_n_classes, + interface.target_global_to_decoder, + ), + ) + loss_service = LossService() + + def likelihood(batch): + batch = batch.to(device) + features, metadata = policy.model_inputs(batch.model_inputs) + prepared = PreparedBatch(features, batch.targets, metadata) + output = policy.network( + features, metadata, interface_name=policy.interface_name + ) + return loss_service.calculate( + output, prepared, dataset_runtime, policy.network + ) + optimization = OptimizationRuntime.create( config, device, policy.network.parameters(), phase_epochs=config.epochs ) @@ -190,16 +231,14 @@ def train_policy( total_loss = 0.0 decisions = 0 for batch_index, batch in enumerate(train_loader): - distribution = policy.distribution(batch.inputs.to(device)) - loss = masked_action_likelihood( - distribution, - batch.actions.to(device), - batch.action_valid_mask.to(device), - ) + likelihood_result = likelihood(batch) + loss = likelihood_result.backward_loss if not torch.isfinite(loss): raise ValueError("policy action likelihood became non-finite") - count = int(batch.action_valid_mask.sum()) - total_loss += float(loss.detach()) * count + count = int(likelihood_result.accounting_count) + total_loss += sum( + float(value) for value in likelihood_result.accounting_sums.values() + ) decisions += count identity = StepIdentity( epoch, @@ -211,9 +250,6 @@ def train_policy( 0, 1, ) - loss = loss + policy.network.regularization_loss(policy.interface_name) - if not torch.isfinite(loss): - raise ValueError("policy regularization became non-finite") optimization.accumulate(loss, identity, integrations, policy.network) result = optimization.complete_step( policy.network, identity, integrations, UpdatePolicy() @@ -228,18 +264,16 @@ def train_policy( validation_count = 0 with torch.no_grad(): for batch in validation_loader: - loss = masked_action_likelihood( - policy.distribution(batch.inputs.to(device)), - batch.actions.to(device), - batch.action_valid_mask.to(device), - ) - if not torch.isfinite(loss): + likelihood_result = likelihood(batch) + if not torch.isfinite(likelihood_result.backward_loss): raise ValueError( "validation action likelihood became non-finite" ) - count = int(batch.action_valid_mask.sum()) - validation_loss += float(loss) * count - validation_count += count + validation_loss += sum( + float(value) + for value in likelihood_result.accounting_sums.values() + ) + validation_count += int(likelihood_result.accounting_count) validation_loss /= validation_count metrics.append( { diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index f93c2c8e..0b3dd8c4 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -3,17 +3,25 @@ from sequifier.config.policy_config import ( CategoricalPolicyConfig, PolicyOptimizationConfig, + PolicySchemaConfig, PolicyTrainingConfig, ReturnDefinition, ) from sequifier.config.policy_infer_config import PolicyInferenceConfig from sequifier.config.policy_train_config import ( + PolicyTableColumns, PolicyTrainConfig, ResolvedPolicyTrainConfig, ReturnConditionedPhaseConfig, + SequenceSourceConfig, TrajectorySourceConfig, resolve_policy_train_config, ) +from sequifier.config.sequence_import_config import SequenceImportConfig +from sequifier.io.batch import SequifierBatch, SequifierInputs +from sequifier.io.policy_dataset import PolicySequenceDataset +from sequifier.io.policy_window import PolicyWindowView +from sequifier.io.sequence import SequenceEpisode, TransitionColumns from sequifier.rl.checkpoint import load_policy, save_policy from sequifier.rl.distributions import ( CategoricalActionDistribution, @@ -33,9 +41,22 @@ collate_policy_examples, split_by_episode, ) +from sequifier.trajectory.importers import import_sequences, source_episodes from sequifier.trajectory.returns import remaining_return, return_to_go __all__ = [ + "SequifierInputs", + "SequifierBatch", + "SequenceEpisode", + "TransitionColumns", + "PolicyWindowView", + "PolicySchemaConfig", + "PolicySequenceDataset", + "SequenceSourceConfig", + "PolicyTableColumns", + "SequenceImportConfig", + "source_episodes", + "import_sequences", "PolicyInferenceConfig", "PolicyTrainConfig", "PolicyOptimizationConfig", diff --git a/src/sequifier/rollout/contracts.py b/src/sequifier/rollout/contracts.py index e8686f61..a64af01b 100644 --- a/src/sequifier/rollout/contracts.py +++ b/src/sequifier/rollout/contracts.py @@ -1,10 +1,12 @@ -"""Adapter values are finite numeric scalars in original (unscaled) units.""" +"""Adapter values are domain labels or finite scalars in original units.""" from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, Mapping, Protocol +from sequifier.io.values import DomainValue + if TYPE_CHECKING: from sequifier.trajectory.schema import Transition @@ -12,9 +14,9 @@ @dataclass(frozen=True) class ResetRequest: episode_id: str - seed_history: Mapping[str, tuple[float, ...]] + seed_history: Mapping[str, tuple[DomainValue, ...]] attention_valid_mask: tuple[bool, ...] - static_inputs: Mapping[str, float] + static_inputs: Mapping[str, DomainValue] seed: int @@ -22,14 +24,14 @@ class ResetRequest: class StepRequest: episode_id: str step_id: int - observation: Mapping[str, float] - predictions: Mapping[str, float] - static_inputs: Mapping[str, float] + observation: Mapping[str, DomainValue] + predictions: Mapping[str, DomainValue] + static_inputs: Mapping[str, DomainValue] @dataclass(frozen=True) class StepResult: - external_inputs: Mapping[str, float] + external_inputs: Mapping[str, DomainValue] reward: float terminated: bool = False truncated: bool = False diff --git a/src/sequifier/rollout/coordinator.py b/src/sequifier/rollout/coordinator.py index 2878a6c7..d9acf932 100644 --- a/src/sequifier/rollout/coordinator.py +++ b/src/sequifier/rollout/coordinator.py @@ -7,6 +7,7 @@ from uuid import uuid4 from sequifier.config.rollout_config import RolloutConfig +from sequifier.io.values import DomainValue, domain_fields from sequifier.rollout.contracts import ( EnvironmentAdapter, ResetRequest, @@ -54,9 +55,19 @@ def collect_rollout( predictor wrapper, using fixed training metadata. """ failed = False + fields = domain_fields if config.mode == "policy" else numeric_fields try: validate_adapter(adapter) history = validate_seed(seed) + if config.mode != "policy": + for column, values in history.observations.items(): + for value in values: + numeric_scalar(value, f"forecast seed.{column}") + numeric_fields( + history.static_inputs, + set(history.static_inputs), + "forecast static inputs", + ) initial_history = history.model_copy(deep=True) if history_validator is not None: history_validator(history.model_copy(deep=True)) @@ -84,7 +95,7 @@ def collect_rollout( } def predict(): - return numeric_fields( + return fields( predictor(history.model_copy(deep=True)), set(prediction_columns), "predictions", @@ -112,9 +123,12 @@ def execute(predictions, step_id): raise ValueError( "adapter termination and truncation flags must be separate booleans" ) - external = numeric_fields( - result.external_inputs, external_keys, "adapter.external_inputs" - ) + external: dict[str, DomainValue] = { + key: value + for key, value in fields( + result.external_inputs, external_keys, "adapter.external_inputs" + ).items() + } reward = numeric_scalar(result.reward, "adapter.reward") successor = { binding.input: ( diff --git a/src/sequifier/rollout/history.py b/src/sequifier/rollout/history.py index 55f461bd..cbc90cbc 100644 --- a/src/sequifier/rollout/history.py +++ b/src/sequifier/rollout/history.py @@ -1,39 +1,20 @@ """Validated seed windows and atomic numeric feature/mask advancement.""" -import math -from collections.abc import Mapping -from numbers import Real - +from sequifier.io.values import ( # noqa: F401 + DomainValue, + numeric_fields, + numeric_scalar, +) from sequifier.trajectory.schema import SeedHistory -def numeric_scalar(value, label: str) -> float: - if isinstance(value, bool) or not isinstance(value, Real): - raise ValueError(f"{label} must be a finite numeric scalar, got {value!r}") - try: - number = float(value) - except (ValueError, OverflowError) as exc: - raise ValueError(f"{label} is outside the supported numeric range") from exc - if not math.isfinite(number): - raise ValueError(f"{label} must be finite, got {value!r}") - return number - - -def numeric_fields(values, expected: set[str], label: str) -> dict[str, float]: - if not isinstance(values, Mapping) or set(values) != expected: - raise ValueError( - f"{label} must supply exactly these fields: {sorted(expected)}" - ) - return { - key: numeric_scalar(values[key], f"{label}.{key}") for key in sorted(expected) - } - - def validate_seed(seed: SeedHistory) -> SeedHistory: return SeedHistory.model_validate(seed.model_dump()) -def advance_history(history: SeedHistory, successor: dict[str, float]) -> SeedHistory: +def advance_history( + history: SeedHistory, successor: dict[str, DomainValue] +) -> SeedHistory: return SeedHistory( observations={ key: values[1:] + [successor[key]] diff --git a/src/sequifier/trajectory/dataset.py b/src/sequifier/trajectory/dataset.py index fe0ee3de..1be229b7 100644 --- a/src/sequifier/trajectory/dataset.py +++ b/src/sequifier/trajectory/dataset.py @@ -1,357 +1,12 @@ -"""Episode-split training windows with explicit current-action policy alignment.""" - -import hashlib -import json -import random -from bisect import bisect_right -from collections import OrderedDict -from collections.abc import Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Literal, overload - -import torch -from torch.utils.data import Dataset, Sampler - -from sequifier.config.policy_config import CategoricalPolicyConfig -from sequifier.rl.history import ( - PolicyInputs, - action_index, - build_policy_inputs, - collate_policy_inputs, +"""Compatibility exports for the shared policy sequence dataset.""" + +from sequifier.io.policy_dataset import EpisodeCollection as EpisodeCollection +from sequifier.io.policy_dataset import EpisodeWindowSampler as EpisodeWindowSampler +from sequifier.io.policy_dataset import PolicyExample as PolicyExample +from sequifier.io.policy_dataset import PolicySequenceDataset as PolicySequenceDataset +from sequifier.io.policy_dataset import TrajectoryDataset as TrajectoryDataset +from sequifier.io.policy_dataset import ( + collate_policy_examples as collate_policy_examples, ) -from sequifier.trajectory.returns import labeled_episode_length, return_to_go -from sequifier.trajectory.schema import Trajectory, validate_trajectory -from sequifier.trajectory.store import load_trajectory - - -@dataclass(frozen=True) -class PolicyExample: - inputs: PolicyInputs - actions: torch.Tensor - action_valid_mask: torch.Tensor - episode_ids: tuple[str, ...] - step_ids: tuple[int, ...] - - -def collate_policy_examples(examples: Sequence[PolicyExample]) -> PolicyExample: - if not examples: - raise ValueError("cannot collate an empty policy batch") - return PolicyExample( - inputs=collate_policy_inputs([example.inputs for example in examples]), - actions=torch.cat([example.actions for example in examples]), - action_valid_mask=torch.cat( - [example.action_valid_mask for example in examples] - ), - episode_ids=tuple( - identity for example in examples for identity in example.episode_ids - ), - step_ids=tuple(step for example in examples for step in example.step_ids), - ) - - -class EpisodeCollection(Sequence[Trajectory]): - """Keep path-backed episodes on disk, with a bounded per-worker cache.""" - - def __init__(self, sources, cache_size: int): - self.sources = list(sources) - self.cache_size = cache_size - self.cache = OrderedDict() - - def __len__(self): - return len(self.sources) - - @overload - def __getitem__(self, index: int) -> Trajectory: ... - - @overload - def __getitem__(self, index: slice) -> Sequence[Trajectory]: ... - - def __getitem__(self, index: int | slice) -> Trajectory | Sequence[Trajectory]: - if isinstance(index, slice): - return [self[i] for i in range(*index.indices(len(self)))] - source = self.sources[index] - if isinstance(source, Trajectory): - return source - if index not in self.cache: - self.cache[index] = load_trajectory(source) - if len(self.cache) > self.cache_size: - self.cache.popitem(last=False) - self.cache.move_to_end(index) - return self.cache[index] - - -class TrajectoryDataset(Dataset): - """One labeled decision per example with a bounded causal context. - - File-backed corpora retain only episode metadata and a small episode cache. - Return labels are cached per episode, never reconstructed for every prefix. - """ - - def __init__( - self, - sources: Sequence[str | Path | Trajectory], - config: CategoricalPolicyConfig, - *, - action_semantics: Literal["policy", "executed_predictions"] = "policy", - episode_cache_size: int = 8, - ): - if action_semantics not in {"policy", "executed_predictions"}: - raise ValueError( - "trajectory action_semantics must be policy or executed_predictions" - ) - if episode_cache_size < 1: - raise ValueError("episode_cache_size must be positive") - self.config = config - self.action_semantics: Literal["policy", "executed_predictions"] = ( - action_semantics - ) - self.excluded_episode_ids = [] - self.all_episode_ids = [] - self.episode_ids = [] - self.content_keys = [] - self.lengths = [] - self.cumulative_lengths = [] - self.episode_cache_size = episode_cache_size - self._labels = OrderedDict() - eligible = [] - seen = set() - for source in sources: - episode = ( - validate_trajectory(source) - if isinstance(source, Trajectory) - else load_trajectory(source) - ) - if episode.episode_id in seen: - raise ValueError(f"duplicate episode ID {episode.episode_id!r}") - seen.add(episode.episode_id) - self.all_episode_ids.append(episode.episode_id) - if ( - episode.format != "sequifier.numeric-policy-trajectory" - and action_semantics != "executed_predictions" - ): - raise ValueError( - "forecast sources require explicit action_semantics='executed_predictions'; model forecast alignment is separate" - ) - if set(episode.seed.observations) != set(config.observation_columns) or set( - episode.seed.static_inputs - ) != set(config.static_columns): - raise ValueError( - "trajectory observations/static inputs do not match the policy schema" - ) - if ( - action_semantics == "policy" - and sum(episode.seed.attention_valid_mask) != 1 - ): - raise ValueError( - "policy episodes must start with one observation and an explicit previous-action start token" - ) - if episode.prediction_columns != [config.action_column]: - raise ValueError( - "policy trajectories must contain the configured single action head" - ) - length = ( - labeled_episode_length(episode, config.returns) - if config.return_conditioned - else len(episode.transitions) - ) - if length is None: - self.excluded_episode_ids.append(episode.episode_id) - continue - for transition in episode.transitions[:length]: - action_index(transition.predictions[config.action_column], config) - eligible.append(source) - self.episode_ids.append(episode.episode_id) - self.content_keys.append(episode_content_key(episode)) - self.lengths.append(length) - self.cumulative_lengths.append( - length + (self.cumulative_lengths[-1] if self.cumulative_lengths else 0) - ) - self.episodes = EpisodeCollection(eligible, episode_cache_size) - - def __len__(self): - return self.cumulative_lengths[-1] if self.cumulative_lengths else 0 - - def _episode_labels(self, index: int): - if index not in self._labels: - transitions = self.episodes[index].transitions[: self.lengths[index]] - actions = [ - action_index( - transition.predictions[self.config.action_column], self.config - ) - for transition in transitions - ] - returns = ( - return_to_go( - [transition.reward for transition in transitions], - self.config.returns.gamma, - ) - if self.config.return_conditioned - else None - ) - self._labels[index] = actions, returns - if len(self._labels) > self.episode_cache_size: - self._labels.popitem(last=False) - self._labels.move_to_end(index) - return self._labels[index] - - def __getitem__(self, index: int) -> PolicyExample: - if index < 0: - index += len(self) - if not 0 <= index < len(self): - raise IndexError(index) - episode_index = bisect_right(self.cumulative_lengths, index) - step = index - ( - self.cumulative_lengths[episode_index - 1] if episode_index else 0 - ) - episode = self.episodes[episode_index] - action_ids, returns = self._episode_labels(episode_index) - start = max(0, step + 1 - self.config.context_length) - previous = ( - [len(self.config.action_values)] if start == 0 else [] - ) + action_ids[max(0, start - 1) : step] - inputs = build_policy_inputs( - self.config, - [ - transition.observation - for transition in episode.transitions[start : step + 1] - ], - previous, - returns[start : step + 1] if returns is not None else None, - episode.seed.static_inputs, - ) - actions = torch.zeros((1, self.config.context_length), dtype=torch.int64) - actions[0, -1] = action_ids[step] - valid = torch.zeros_like(actions, dtype=torch.bool) - valid[0, -1] = True - return PolicyExample(inputs, actions, valid, (episode.episode_id,), (step,)) - - def normalization_statistics(self) -> dict[str, dict[str, float]]: - """Combine episode moments without materializing all training rows.""" - columns = self.config.observation_columns + self.config.static_columns - moments = { - column: (0, 0.0, 0.0) - for column in columns - if column not in self.config.observation_n_classes - } - for episode, length in zip(self.episodes, self.lengths): - for column in moments: - values = ( - [ - transition.observation[column] - for transition in episode.transitions[:length] - ] - if column in self.config.observation_columns - else [episode.seed.static_inputs[column]] - ) - fitted = torch.tensor(values, dtype=torch.float64) - count, mean, m2 = moments[column] - size = len(values) - batch_mean = float(fitted.mean()) - delta = batch_mean - mean - total = count + size - moments[column] = ( - total, - mean + delta * size / total, - m2 - + float(((fitted - batch_mean) ** 2).sum()) - + delta**2 * count * size / total, - ) - return { - column: {"mean": mean, "std": max((m2 / count) ** 0.5, 1e-6)} - for column, (count, mean, m2) in moments.items() - } - - def normalization_rows(self) -> tuple[torch.Tensor, torch.Tensor]: - """Materialized convenience API retained for Python callers.""" - observations = [ - [ - transition.observation[column] - for column in self.config.observation_columns - ] - for episode, length in zip(self.episodes, self.lengths) - for transition in episode.transitions[:length] - ] - static = [ - [ - episode.seed.static_inputs[column] - for column in self.config.static_columns - ] - for episode in self.episodes - ] - return torch.tensor(observations, dtype=torch.float32), torch.tensor( - static, dtype=torch.float32 - ) - - -class EpisodeWindowSampler(Sampler): - """Shuffle episodes and their decisions while keeping disk access local.""" - - def __init__(self, dataset: TrajectoryDataset, generator: torch.Generator): - self.dataset = dataset - self.generator = generator - - def __len__(self): - return len(self.dataset) - - def __iter__(self): - for episode in torch.randperm( - len(self.dataset.lengths), generator=self.generator - ).tolist(): - offset = self.dataset.cumulative_lengths[episode - 1] if episode else 0 - for step in torch.randperm( - self.dataset.lengths[episode], generator=self.generator - ).tolist(): - yield offset + step - - -def episode_content_key(episode: Trajectory) -> str: - """Group repeated demonstrations independently of generated episode IDs.""" - values = episode.model_dump(mode="json") - values.pop("episode_id") - values.pop("metadata") - for transition in values["transitions"]: - transition.pop("episode_id") - return hashlib.sha256(json.dumps(values, sort_keys=True).encode()).hexdigest() - - -def split_by_episode( - dataset: TrajectoryDataset, - validation_fraction: float = 0.2, - seed: int = 1010, - *, - group_identical: bool = False, -) -> tuple[TrajectoryDataset, TrajectoryDataset]: - if not 0 < validation_fraction < 1 or len(dataset.episodes) < 2: - raise ValueError( - "episode splitting requires at least two eligible episodes and validation_fraction in (0,1)" - ) - groups = {} - for index, source in enumerate(dataset.episodes.sources): - key = ( - dataset.content_keys[index] - if group_identical - else dataset.episode_ids[index] - ) - groups.setdefault(key, []).append(source) - partitions = list(groups.values()) - if len(partitions) < 2: - raise ValueError("splitting requires at least two distinct trajectory groups") - random.Random(seed).shuffle(partitions) - count = min(len(partitions) - 1, max(1, int(len(partitions) * validation_fraction))) - validation = [episode for group in partitions[:count] for episode in group] - training = [episode for group in partitions[count:] for episode in group] - return ( - TrajectoryDataset( - training, - dataset.config, - action_semantics=dataset.action_semantics, - episode_cache_size=dataset.episode_cache_size, - ), - TrajectoryDataset( - validation, - dataset.config, - action_semantics=dataset.action_semantics, - episode_cache_size=dataset.episode_cache_size, - ), - ) +from sequifier.io.policy_dataset import episode_content_key as episode_content_key +from sequifier.io.policy_dataset import split_by_episode as split_by_episode diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py new file mode 100644 index 00000000..3b808fc0 --- /dev/null +++ b/src/sequifier/trajectory/importers.py @@ -0,0 +1,762 @@ +"""Adapters from tables, prepared histories and rollout logs to sequence data.""" + +import hashlib +import json +import os +import shutil +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +import polars as pl + +from sequifier.config.metadata import load_dataset_metadata +from sequifier.config.policy_config import PolicySchemaConfig +from sequifier.config.policy_train_config import ( + PolicyTableColumns, + SequenceSourceConfig, +) +from sequifier.io.sequence import ( + SequenceEpisode, + TransitionColumns, + ordered_sequence_rows, +) +from sequifier.io.values import domain_scalar, numeric_scalar +from sequifier.trajectory.schema import Trajectory, validate_trajectory +from sequifier.trajectory.store import load_trajectory + + +def _checksum(path): + digest = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def source_paths(source: SequenceSourceConfig) -> list[Path]: + if source.manifest is None: + paths = [Path(path) for path in source.paths] + else: + manifest_path = Path(source.manifest) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict) or manifest.get("version") != 1: + raise ValueError("source manifest must have version 1") + sequence_manifest = manifest.get("format") == "sequifier.sequence-data" + entries = manifest.get("shards" if sequence_manifest else "episodes") + if not isinstance(entries, list) or not entries: + raise ValueError("source manifest requires a nonempty episodes list") + paths = [] + for entry in entries: + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("path"), str) + or not entry["path"] + ): + raise ValueError("each manifest entry requires a nonempty path") + path = manifest_path.parent / entry["path"] + if sequence_manifest and _checksum(path) != entry.get("sha256"): + raise ValueError(f"sequence shard checksum disagrees with {path}") + if ( + "episode_id" in entry + and load_trajectory(path).episode_id != entry["episode_id"] + ): + raise ValueError(f"manifest episode ID disagrees with {path}") + paths.append(path) + paths = [path.resolve() for path in paths] + if not paths or len(set(paths)) != len(paths): + raise ValueError("sources require nonempty, unique paths") + return paths + + +def sequence_from_trajectory( + trajectory: Trajectory, policy: PolicySchemaConfig, action_semantics="policy" +) -> SequenceEpisode: + episode = validate_trajectory(trajectory) + if ( + episode.format != "sequifier.numeric-policy-trajectory" + and action_semantics != "executed_predictions" + ): + raise ValueError( + "forecast sources require explicit action_semantics='executed_predictions'" + ) + if episode.prediction_columns != [policy.action_column]: + raise ValueError("recorded actions do not match the policy action column") + return SequenceEpisode( + episode_id=episode.episode_id, + item_positions=tuple(step.step_id for step in episode.transitions), + observations={ + column: tuple(step.observation[column] for step in episode.transitions) + for column in episode.seed.observations + }, + actions=tuple( + step.predictions[policy.action_column] for step in episode.transitions + ), + static_inputs=episode.seed.static_inputs, + transitions=TransitionColumns( + rewards=tuple(step.reward for step in episode.transitions), + terminated=tuple(step.terminated for step in episode.transitions), + truncated=tuple(step.truncated for step in episode.transitions), + successors=tuple( + step.successor_observation for step in episode.transitions + ), + bootstrap_allowed=tuple( + step.bootstrap_allowed for step in episode.transitions + ), + ), + ) + + +@dataclass(frozen=True) +class TrajectoryReference: + path: Path + policy: PolicySchemaConfig + action_semantics: str = "policy" + namespace: str = "" + + def load(self) -> SequenceEpisode: + episode = sequence_from_trajectory( + load_trajectory(self.path), self.policy, self.action_semantics + ) + return episode.model_copy( + update={"episode_id": self.namespace + episode.episode_id} + ) + + +def _scan(paths: tuple[Path, ...]): + scans = [] + for path in paths: + if path.suffix.lower() == ".csv": + scans.append(pl.scan_csv(path, infer_schema_length=None)) + elif path.suffix.lower() == ".parquet": + scans.append(pl.scan_parquet(path)) + else: + raise ValueError("table sources require CSV or Parquet files") + return pl.concat(scans, how="vertical_relaxed") + + +def _prepared_table( + paths: tuple[Path, ...], source: SequenceSourceConfig +) -> pl.DataFrame: + if source.metadata_config_path is None: + raise ValueError("prepared histories require metadata_config_path") + signatures = tuple( + (path.stat().st_mtime_ns, path.stat().st_size, path.stat().st_ino) + for path in (*paths, Path(source.metadata_config_path)) + ) + return _prepared_table_contents(paths, source.metadata_config_path, signatures) + + +@lru_cache(maxsize=1) +def _prepared_table_contents( + paths: tuple[Path, ...], metadata_path: str, signatures: tuple +) -> pl.DataFrame: + """Recover unique positions from existing windows, validating overlaps. + + Split target bounds select decisions. A preceding action may be recovered + from stored context, but observations outside the split are not targets. + """ + import torch + + from sequifier.helpers import ( + PANDAS_TO_TORCH_TYPES, + numpy_storage_to_pytorch, + read_data, + ) + from sequifier.io.pt_payload import load_pt_payload + + metadata = load_dataset_metadata(metadata_path) + if metadata.depth_layouts: + raise ValueError( + "policy table import currently supports scalar prepared columns" + ) + rows = {} + for path in paths: + if path.suffix.lower() == ".pt": + batch = load_pt_payload(path, n_classes=metadata.n_classes) + sequences = batch.sequences + ids, starts, pads = ( + batch.sequence_ids, + batch.start_item_positions, + batch.left_pad_lengths, + ) + lows, highs = ( + batch.split_start_item_positions, + batch.split_end_item_positions, + ) + else: + data = read_data(str(path), path.suffix.lower().lstrip(".")) + available = sorted(set(data["inputCol"].to_list())) + sequences, pads = numpy_storage_to_pytorch( + data, + { + column: PANDAS_TO_TORCH_TYPES[dtype] + for column, dtype in metadata.column_data_types.items() + }, + available, + metadata.window_length, + ) + positions = ( + data.group_by(["sequenceId", "subsequenceId"]) + .agg( + pl.col("startItemPosition").first(), + pl.col("splitStartItemPosition").first(), + pl.col("splitEndItemPosition").first(), + ) + .sort(["sequenceId", "subsequenceId"]) + ) + ids, starts, lows, highs = ( + torch.tensor(positions[name].to_numpy()) + for name in ( + "sequenceId", + "startItemPosition", + "splitStartItemPosition", + "splitEndItemPosition", + ) + ) + for row in range(len(ids)): + for offset in range( + int(pads[row]), next(iter(sequences.values())).shape[1] + ): + position = int(starts[row]) + offset + key = (int(ids[row]), position) + values = { + column: tensor[row, offset].item() + for column, tensor in sequences.items() + } + record = { + "sequenceId": key[0], + "itemPosition": position, + **values, + "__decision": int(lows[row]) <= position < int(highs[row]), + } + if key in rows and rows[key] != record: + raise ValueError( + "prepared history windows disagree on overlapping positions or split bounds" + ) + rows[key] = record + if not rows: + raise ValueError("prepared source contains no valid positions") + return pl.DataFrame(list(rows.values())).sort(["sequenceId", "itemPosition"]) + + +@dataclass(frozen=True) +class TableEpisodeReference: + paths: tuple[Path, ...] + source: SequenceSourceConfig + policy: PolicySchemaConfig + sequence_id: str | int + prepared: bool = False + + def load(self) -> SequenceEpisode: + columns = self.source.columns + data = ( + _prepared_table(self.paths, self.source).filter( + pl.col(columns.sequence_id) == self.sequence_id + ) + if self.prepared + else _scan(self.paths) + .filter(pl.col(columns.sequence_id) == self.sequence_id) + .collect() + ) + data = ordered_sequence_rows(data, columns.item_position) + metadata = ( + load_dataset_metadata(self.source.metadata_config_path) + if self.source.metadata_config_path + else None + ) + model_encoded = self.prepared or self.source.encoding == "model" + + def decode(value, column): + value = domain_scalar(value, column) + if model_encoded and metadata is not None: + if column in metadata.id_maps: + inverse = { + encoded: label + for label, encoded in metadata.id_maps[column].items() + } + if ( + isinstance(value, str) + or value != int(value) + or int(value) not in inverse + ): + raise ValueError(f"{column}: encoded ID has no domain label") + value = inverse[int(value)] + if column == (columns.action or self.policy.action_column): + if isinstance(value, str) and not isinstance( + self.policy.action_values[0], str + ): + try: + value = numeric_scalar(float(value), column) + except ValueError: + pass + matches = [ + action + for action in self.policy.action_values + if action == value or str(action) == str(value) + ] + if len(matches) == 1: + value = matches[0] + elif ( + metadata.normalize_real_columns + and column in metadata.selected_columns_statistics + ): + stats = metadata.selected_columns_statistics[column] + value = ( + numeric_scalar(value, column) * (stats["std"] + 1e-9) + + stats["mean"] + ) + return value + + preceding = None + if self.prepared: + decisions = data.filter(pl.col("__decision")) + if decisions.is_empty(): + raise ValueError("prepared episode contains no split-owned decisions") + first = decisions[columns.item_position][0] + previous = data.filter(pl.col(columns.item_position) == first - 1) + if len(previous): + preceding = decode( + previous[columns.action or self.policy.action_column][0], + columns.action or self.policy.action_column, + ) + data = decisions + rows = list(data.iter_rows(named=True)) + if not rows: + raise ValueError("sequence source changed while loading") + required = {columns.item_position, columns.action or self.policy.action_column} + observation_columns = { + name: columns.observations.get(name, name) + for name in self.policy.observation_columns + } + static_columns = { + name: columns.static_inputs.get(name, name) + for name in self.policy.static_columns + } + action_column = columns.action or self.policy.action_column + if action_column in observation_columns.values() or ( + self.source.static_data_path is None + and action_column in static_columns.values() + ): + raise ValueError( + "current action labels cannot also be policy inputs; map distinct source columns" + ) + if columns.previous_action == action_column: + raise ValueError( + "previous and current actions require distinct source columns" + ) + if set(columns.observations) - set(observation_columns) or set( + columns.static_inputs + ) - set(static_columns): + raise ValueError("table mappings must name declared policy features") + required.update(observation_columns.values()) + if self.source.static_data_path is None: + required.update(static_columns.values()) + for name in ( + columns.previous_action, + columns.bootstrap_allowed, + columns.group, + columns.episode_id, + ): + if name is not None: + required.add(name) + required.update(columns.successors.values()) + if required - set(data.columns): + raise ValueError( + f"policy table is missing columns: {sorted(required - set(data.columns))}" + ) + positions = tuple(row[columns.item_position] for row in rows) + if any( + isinstance(value, bool) or not isinstance(value, int) for value in positions + ): + raise ValueError("itemPosition must contain integers") + actions = tuple( + decode( + row[columns.action or self.policy.action_column], + columns.action or self.policy.action_column, + ) + for row in rows + ) + if columns.previous_action is not None: + values = [ + None + if row[columns.previous_action] is None + else decode( + row[columns.previous_action], + columns.action or self.policy.action_column, + ) + for row in rows + ] + if values[1:] != list(actions[:-1]): + raise ValueError( + "previous-action column must lag executed actions by exactly one decision" + ) + preceding = values[0] + starts = positions[0] == 0 and preceding is None + if self.source.static_data_path is not None: + static_data = ( + _scan((Path(self.source.static_data_path),)) + .filter(pl.col(columns.sequence_id) == self.sequence_id) + .collect() + ) + if len(static_data) != 1: + raise ValueError("static data requires exactly one row per sequenceId") + static_rows = list(static_data.iter_rows(named=True)) + else: + static_rows = rows + static = { + name: decode(static_rows[0][column], column) + for name, column in static_columns.items() + } + if any( + { + name: decode(row[column], column) + for name, column in static_columns.items() + } + != static + for row in static_rows + ): + raise ValueError("static inputs must remain constant within an episode") + + def flags(column): + if column is None or column not in data.columns: + return None + values = tuple(row[column] for row in rows) + if all(value is None for value in values): + return None + if any(not isinstance(value, bool) for value in values): + raise ValueError( + f"{column}: boundary/bootstrap columns must contain booleans" + ) + return values + + terminated, truncated = flags(columns.terminated), flags(columns.truncated) + declared = self.source.episode_end + if declared != "unknown": + current = terminated if declared == "terminated" else truncated + if current is not None and not current[-1]: + raise ValueError( + "episode_end conflicts with the recorded final boundary" + ) + current = (False,) * (len(rows) - 1) + (True,) + if declared == "terminated": + terminated = current + else: + truncated = current + successors = None + if columns.successors: + if set(columns.successors) != set(observation_columns): + raise ValueError( + "successor mapping must cover every observation column" + ) + successors = [] + for row in rows: + values = [row[column] for column in columns.successors.values()] + if all(value is None for value in values): + successors.append(None) + elif any(value is None for value in values): + raise ValueError( + "a missing successor must mask all observation fields" + ) + else: + successors.append( + { + name: decode(row[column], observation_columns[name]) + for name, column in columns.successors.items() + } + ) + group_values = [row[columns.group] for row in rows] if columns.group else [] + groups = ( + {domain_scalar(value, "collection group") for value in group_values} + if any(value is not None for value in group_values) + else set() + ) + if len(groups) > 1: + raise ValueError("collection group must be constant within a sequence") + identities = ( + {domain_scalar(row[columns.episode_id], "episode ID") for row in rows} + if columns.episode_id + else {self.sequence_id} + ) + if len(identities) != 1: + raise ValueError("episode ID must remain constant within a sequence") + rewards = None + if columns.reward and columns.reward in data.columns: + values = [row[columns.reward] for row in rows] + if any(value is not None for value in values): + rewards = tuple( + numeric_scalar(decode(value, columns.reward), "reward") + for value in values + ) + return SequenceEpisode( + episode_id=self.source.episode_namespace + str(next(iter(identities))), + item_positions=positions, + observations={ + name: tuple(decode(row[column], column) for row in rows) + for name, column in observation_columns.items() + }, + actions=actions, + static_inputs=static, + starts_episode=starts, + previous_action=preceding, + group_id=str(next(iter(groups))) if groups else None, + transitions=TransitionColumns( + rewards=rewards, + terminated=terminated, + truncated=truncated, + successors=tuple(successors) if successors is not None else None, + bootstrap_allowed=flags(columns.bootstrap_allowed), + ), + ) + + +def source_episodes(source: SequenceSourceConfig, policy: PolicySchemaConfig): + if source.split_index is not None: + from sequifier.config.policy_train_config import resolve_sequence_source + + source = resolve_sequence_source(source, Path.cwd()) + paths = tuple(source_paths(source)) + if source.manifest is not None: + manifest = json.loads(Path(source.manifest).read_text(encoding="utf-8")) + if manifest.get("format") == "sequifier.sequence-data": + recorded_columns = PolicyTableColumns.model_validate(manifest["columns"]) + if ( + source.columns != PolicyTableColumns() + and source.columns != recorded_columns + ): + raise ValueError( + "source mappings disagree with the committed sequence manifest" + ) + if source.encoding != "raw" or source.episode_end != "unknown": + raise ValueError( + "committed sequence data already declares its encoding and boundaries" + ) + source = source.model_copy( + update={"columns": recorded_columns, "format": "table"} + ) + kind = source.format + if kind == "auto": + suffixes = {path.suffix.lower() for path in paths} + if suffixes <= {".json"}: + kind = "trajectory" + elif suffixes <= {".csv", ".parquet"}: + kind = "table" + elif suffixes == {".pt"}: + kind = "prepared" + else: + raise ValueError( + "select a source format explicitly for mixed or unsupported files" + ) + if kind == "trajectory": + return [ + TrajectoryReference( + path, policy, source.action_semantics, source.episode_namespace + ) + for path in paths + ] + prepared = kind == "prepared" + data = _prepared_table(paths, source).lazy() if prepared else _scan(paths) + columns = source.columns + required = {columns.sequence_id, columns.item_position} + if required - set(data.collect_schema().names()): + raise ValueError( + "policy tables require sequenceId and itemPosition, or explicit column mappings" + ) + ids = ( + data.select(columns.sequence_id) + .unique(maintain_order=True) + .collect()[columns.sequence_id] + .to_list() + ) + if any( + isinstance(value, bool) or not isinstance(value, (int, str)) for value in ids + ): + raise ValueError("sequence IDs must be non-null integers or strings") + if source.static_data_path: + static_ids = ( + _scan((Path(source.static_data_path),)) + .select(columns.sequence_id) + .collect()[columns.sequence_id] + .to_list() + ) + if len(static_ids) != len(set(static_ids)) or not set(ids) <= set(static_ids): + raise ValueError( + "static data must cover every selected sequence ID without duplicates" + ) + return [ + TableEpisodeReference(paths, source, policy, identity, prepared) + for identity in ids + ] + + +def import_sequences(config) -> Path: + """Atomically commit raw Parquet shards and a checksummed source manifest. + + This is a lossless import, not a fit: split assignment, vocabularies, window + padding and return labels remain the responsibility of the training view. + """ + import pyarrow as pa + import pyarrow.parquet as pq + + source = config.resolved_source() + destination = (Path(config.project_root) / config.output_path).resolve() + if destination.exists(): + raise ValueError( + "sequence import output_path must name a new dataset directory" + ) + destination.parent.mkdir(parents=True, exist_ok=True) + fields = config.policy.observation_columns + config.policy.static_columns + reserved = { + "sequenceId", + "itemPosition", + "episodeId", + "__executed_action", + "__reward", + "__terminated", + "__truncated", + "__previous_action", + "__bootstrap", + "__group", + } + reserved.update("__next_" + column for column in config.policy.observation_columns) + if set(fields) & reserved: + raise ValueError( + "policy feature names collide with sequence import metadata columns" + ) + references = source_episodes(source, config.policy) + if not references: + raise ValueError("sequence import requires at least one episode") + stage = Path(tempfile.mkdtemp(prefix=".sequifier-import-", dir=destination.parent)) + shards, records, schema, seen = [], [], None, set() + columns = PolicyTableColumns( + action="__executed_action", + reward="__reward", + terminated="__terminated", + truncated="__truncated", + previous_action="__previous_action", + bootstrap_allowed="__bootstrap", + group=None, + episode_id="episodeId", + successors={ + column: "__next_" + column for column in config.policy.observation_columns + }, + ) + + def flush(): + nonlocal records, schema + if not records: + return + if schema is None: + inferred = pa.Table.from_pylist(records).schema + real = ( + set(fields) + - set(config.policy.categorical_columns) + - config.policy.categorical_n_classes.keys() + ) + numeric = real | {"__reward"} + if not any(isinstance(value, str) for value in config.policy.action_values): + numeric |= {"__executed_action", "__previous_action"} + numeric |= {"__next_" + column for column in real} + schema = pa.schema( + [ + pa.field( + field.name, + pa.float64() + if field.name in numeric + else pa.bool_() + if field.name in {"__terminated", "__truncated", "__bootstrap"} + else pa.string() + if field.name == "__group" + else inferred.field("__executed_action").type + if field.name == "__previous_action" + else inferred.field(field.name[len("__next_") :]).type + if field.name.startswith("__next_") + else field.type, + ) + for field in inferred + ] + ) + name = f"part-{len(shards):05d}.parquet" + path = stage / name + pq.write_table(pa.Table.from_pylist(records, schema=schema), path) + shards.append( + {"path": name, "sha256": _checksum(path), "decisions": len(records)} + ) + records = [] + + try: + for index, reference in enumerate(references): + episode = reference.load() + if episode.episode_id in seen: + raise ValueError("duplicate episode IDs in sequence import") + seen.add(episode.episode_id) + for position, item in enumerate(episode.item_positions): + transition = episode.transitions + successor = ( + transition.successors[position] + if transition is not None and transition.successors is not None + else None + ) + records.append( + { + "sequenceId": index, + "itemPosition": item, + "episodeId": episode.episode_id, + **episode.observation(position), + **episode.static_inputs, + "__executed_action": episode.actions[position], + "__previous_action": episode.previous_action + if position == 0 + else episode.actions[position - 1], + "__reward": transition.rewards[position] + if transition is not None and transition.rewards is not None + else None, + "__terminated": transition.terminated[position] + if transition is not None and transition.terminated is not None + else None, + "__truncated": transition.truncated[position] + if transition is not None and transition.truncated is not None + else None, + "__bootstrap": transition.bootstrap_allowed[position] + if transition is not None + and transition.bootstrap_allowed is not None + else None, + "__group": episode.group_id, + **{ + "__next_" + column: successor[column] + if successor is not None + else None + for column in config.policy.observation_columns + }, + } + ) + if (index + 1) % config.episodes_per_shard == 0: + flush() + flush() + # Group is optional globally; all-null group columns do not invent a group. + columns = columns.model_copy(update={"group": "__group"}) + manifest = { + "format": "sequifier.sequence-data", + "version": 1, + "episodes": len(seen), + "decisions": sum(shard["decisions"] for shard in shards), + "columns": columns.model_dump(), + "policy_schema": config.policy.model_dump(mode="json"), + "shards": shards, + "provenance": { + "source": source.model_dump(mode="json"), + "files": [ + {"path": str(path), "sha256": _checksum(path)} + for path in source_paths(source) + ], + }, + } + (stage / "manifest.json").write_text( + json.dumps(manifest, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + os.rename(stage, destination) + return destination / "manifest.json" + finally: + if stage.exists(): + shutil.rmtree(stage) diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index b73c9c97..3ccb440f 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -1,4 +1,4 @@ -"""Inspectable version-1 numeric forecast and policy episode records.""" +"""Inspectable version-1 forecast and policy collection records.""" import json import math @@ -15,6 +15,7 @@ ) from sequifier.config.rollout_config import InputBinding +from sequifier.io.values import DomainValue class Record(BaseModel): @@ -22,9 +23,9 @@ class Record(BaseModel): class SeedHistory(Record): - observations: dict[str, list[FiniteFloat]] + observations: dict[str, list[DomainValue]] attention_valid_mask: list[bool] - static_inputs: dict[str, FiniteFloat] + static_inputs: dict[str, DomainValue] @model_validator(mode="after") def validate_window(self): @@ -49,11 +50,11 @@ def validate_window(self): class Transition(Record): episode_id: str step_id: int = Field(ge=0) - observation: dict[str, FiniteFloat] - predictions: dict[str, FiniteFloat] - external_inputs: dict[str, FiniteFloat] + observation: dict[str, DomainValue] + predictions: dict[str, DomainValue] + external_inputs: dict[str, DomainValue] reward: FiniteFloat - successor_observation: dict[str, FiniteFloat] + successor_observation: dict[str, DomainValue] terminated: bool truncated: bool bootstrap_allowed: bool From 8e7cd9b70b2db74c81b9b5fbdfc7c4ee0fc33f12 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 15:04:08 +0100 Subject: [PATCH 07/71] reduce --- documentation/configs/train.md | 50 +-- .../configs/train-return-conditioned.yaml | 93 +++-- documentation/policy-learning.md | 167 ++++---- release-notes/unreleased-policy-training.md | 11 +- src/sequifier/artifacts/run_checkpoint.py | 8 +- src/sequifier/config/policy_config.py | 21 +- src/sequifier/config/policy_train_config.py | 367 +++++++----------- src/sequifier/config/training_plan.py | 20 +- src/sequifier/consolidated-docs.md | 217 ++++++----- src/sequifier/io/batch.py | 97 +---- src/sequifier/io/policy_dataset.py | 134 ++----- src/sequifier/io/sequence.py | 28 +- src/sequifier/io/tensors.py | 65 ++++ src/sequifier/make.py | 86 +++- src/sequifier/rl/history.py | 77 ---- src/sequifier/rl/inference.py | 12 +- src/sequifier/rl/learner.py | 249 +++++------- src/sequifier/rl/policy.py | 83 +--- src/sequifier/rl/runtime.py | 8 +- src/sequifier/rl/training.py | 240 +++++------- src/sequifier/rl_api.py | 29 +- src/sequifier/trajectory/dataset.py | 12 - src/sequifier/trajectory/importers.py | 328 +++++++--------- src/sequifier/trajectory/returns.py | 16 - .../test_normalization_parameter_groups.py | 61 --- tests/unit/test_static_preprocessing.py | 46 --- 26 files changed, 995 insertions(+), 1530 deletions(-) create mode 100644 src/sequifier/io/tensors.py delete mode 100644 src/sequifier/rl/history.py delete mode 100644 src/sequifier/trajectory/dataset.py delete mode 100644 tests/unit/test_normalization_parameter_groups.py delete mode 100644 tests/unit/test_static_preprocessing.py diff --git a/documentation/configs/train.md b/documentation/configs/train.md index d2275d7a..98f7e9f9 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -768,35 +768,23 @@ healthy active update. Momentum and weight decay cannot update parameters whose gradients are absent. -## Trajectory-backed policy training - -`sequifier train` also accepts one `return_conditioned` phase with named -trajectory sources and a categorical policy registry. This workflow supports -return conditioning or BC, with no supervised dataset placeholders. See the -[policy training guide](../policy-learning.md) and -[complete example](../example_inputs/configs/train-return-conditioned.yaml). -The phase owns `epochs`; the run owns `seed`; `optimization` configures updates. -Existing phases default to `algorithm: supervised`. Mixed algorithm plans and -IQL/PPO remain planned. - -Policy models use the shared configurable network and portable artifacts. They -support CPU/CUDA/MPS, backbone transfer with `initialize_from`, bounded file-backed -episode loading, and epoch-boundary resume through `optimization.resume`. - -## Policy sequence sources and shared authoring - -Policy phases accept ordinary CSV/Parquet demonstrations with `sequenceId` and -`itemPosition`. Use `data_sources` and phase `sources: [{source: demonstrations}]`; -the previous `trajectory_sources` spellings remain accepted. BC does not require -rewards or successor states. Return-conditioned training requires rewards and a -known return endpoint. JSON rollout logs are adapted into the same sequence data. - -Policy training can use the ordinary `model.backbone`, `model.interfaces` and -`dataset_training..parts` authoring patterns. Put the model-independent -observation/action schema in an interface's `policy` field and a sequence source -in each dataset part. The algorithm supplies action NLL. `global_training` can -supply the policy optimization settings and `context_length`, with -`training_objective: policy`; the compact `policies`/`optimization` surface remains -supported. One policy phase is currently supported. See the complete -[table training example](../example_inputs/configs/train-policy-from-table.yaml) +## Policy sequence training + +`sequifier train` accepts one `return_conditioned` phase. Put the observation/action +schema in `model.interfaces..policy`, a sequence source in each +`dataset_training..parts.`, and optimizer/context settings in +`global_training` with `training_objective: policy`. Reference sources with +`{source: dataset.part}`. The algorithm supplies action NLL. + +`return_conditioned: false` selects BC, which needs observations and action labels. +Return-conditioned training additionally requires rewards and a known return +endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories all +use the same episode representation. Models support CPU/CUDA/MPS, backbone +transfer via `initialize_from`, and epoch-boundary resume via +`global_training.resume`. Mixed phases, IQL/PPO and image/depth policies remain +planned. The former compact registries and configuration aliases have been +consolidated into the shared model/dataset authoring surface. + +See the [complete table example](../example_inputs/configs/train-policy-from-table.yaml), +[return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml) and [policy learning guide](../policy-learning.md). diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index cb595fc1..32b0f5c5 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -2,48 +2,77 @@ project_root: . model_name: policy device: cpu seed: 1010 - -data_sources: - demonstrations: - # One row per current observation/executed action/resulting reward. - paths: [data/demonstrations.parquet] - -# Set observation columns and action values to match your table. -policies: - actor: - observation_columns: [OBSERVATION_COLUMN] - action_column: action - action_values: [0.0, 1.0] - context_length: 4 - return_conditioned: true - returns: {gamma: 1.0, scale: 1.0} - backbone: +model: + backbone: + architecture: dim_model: 32 num_layers: 2 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} + attention: + n_heads: 4 + feed_forward: + dim: 128 + activation: swiglu positional_encoding_scope: global dropout: 0.0 - -training_plan: - phases: - - name: offline_policy - algorithm: return_conditioned - epochs: 80 - sources: [{source: demonstrations}] - policy: actor - -optimization: + interfaces: + actor: + input_columns: &id001 + - OBSERVATION_COLUMN + target_columns: + - action + policy: + observation_columns: *id001 + action_column: action + action_values: + - 0.0 + - 1.0 + return_conditioned: true + returns: + gamma: 1.0 + scale: 1.0 + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: + OBSERVATION_COLUMN: 1 + __policy_previous_action: 6 + __policy_return: 1 + decoder: + type: linear + prediction_length: 1 + support: 1 +dataset_training: + demonstrations: + model_interface: actor + parts: + recorded: + paths: + - data/demonstrations.parquet +global_training: + training_objective: policy + context_length: 4 batch_size: 64 num_workers: 0 episode_cache_size: 8 - resume: {policy: never} + resume: + policy: never learning_rate: 0.005 gradient_clip: 1.0 validation_fraction: 0.2 - optimizer: {name: Adam} - scheduler: {name: ConstantLR, factor: 1.0, total_iters: 1} - + optimizer: + name: Adam + scheduler: + name: ConstantLR + factor: 1.0 + total_iters: 1 +training_plan: + phases: + - name: offline_policy + algorithm: return_conditioned + epochs: 80 + sources: + - source: demonstrations.recorded + policy: actor model_path: models/policy/policy.pt report_path: outputs/policy/training-report.json diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index aa4b11cc..3f029057 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -17,22 +17,58 @@ unique and contiguous within each sequence, including across input shards. ```yaml project_root: . -model_name: policy -data_sources: +model_name: table-policy +device: cpu +seed: 1010 + +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128} + positional_encoding_scope: global + interfaces: + actor: + input_columns: [position, velocity] + target_columns: [action] + policy: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: + position: 1 + velocity: 1 + __policy_previous_action: 6 + decoder: {type: linear, prediction_length: 1, support: 1} + +dataset_training: demonstrations: - paths: [data/demonstrations.parquet] -policies: - actor: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - context_length: 16 - return_conditioned: false + model_interface: actor + parts: + recorded: + paths: [data/demonstrations.parquet] + +global_training: + training_objective: policy + context_length: 16 + batch_size: 64 + learning_rate: 0.005 + validation_fraction: 0.2 + training_plan: - algorithm: return_conditioned - epochs: 80 - sources: [{source: demonstrations}] - policy: actor + phases: + - name: behavior_cloning + algorithm: return_conditioned + policy: actor + epochs: 80 + sources: [{source: demonstrations.recorded}] ``` Run `sequifier train --config-path configs/train-policy.yaml`. BC @@ -49,19 +85,22 @@ Defaults use the policy feature names and the columns `sequenceId`, explicitly: ```yaml -data_sources: +dataset_training: demonstrations: - paths: [data/recorded.csv] - columns: - sequence_id: episode - item_position: step - observations: {position: measured_position, velocity: measured_velocity} - action: command - reward: score - terminated: task_finished - truncated: time_limit - previous_action: preceding_command - group: collection_session + model_interface: actor + parts: + recorded: + paths: [data/recorded.csv] + columns: + sequence_id: episode + item_position: step + observations: {position: measured_position, velocity: measured_velocity} + action: command + reward: score + terminated: task_finished + truncated: time_limit + previous_action: preceding_command + group: collection_session ``` Boundary columns, when present, contain booleans. No row may follow a boundary @@ -89,7 +128,7 @@ metadata rather than model features. Without `validation_sources`, training splits whole episodes deterministically, keeping repeated demonstrations and declared collection groups together. At least two independent groups are required. Alternatively declare a held-out source -and set `validation_sources: [held_out]` on the phase. Duplicate paths and episode +and set `validation_sources: [held_out.recorded]` on the phase. Duplicate paths and episode IDs, overlapping collection groups and repeated demonstrations across partitions are rejected. Normalization and categorical vocabulary fitting use training data. @@ -119,20 +158,19 @@ A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy adapted into the same sequence representation, with bindings retained in their recording format. Forecast recordings require `action_semantics: executed_predictions`; this does not convert forecast checkpoint -alignment into policy alignment. The old `trajectory_sources` registry and phase -field remain aliases for `data_sources` and `sources`. +alignment into policy alignment. Run paths resolve relative to `project_root`; manifest entries resolve relative -to the manifest directory. `optimization.num_workers` controls loader workers; +to the manifest directory. `global_training.num_workers` controls loader workers; `episode_cache_size` bounds cached episodes per worker. Table-backed episodes use filtered lazy scans. Prepared window reconstruction is eager per source and JSON records still require one episode to fit in memory. The best model is selected by held-out action NLL. `model_path` defaults to `models//.pt` and contains an ordinary `sequifier_model`. -`actor_path` remains an alias. The report records partitions, exclusions, the fitted +The report records partitions, exclusions, the fitted policy codec, decision counts and per-epoch NLL. Epochs belong to the phase, seed -and device to the run, and optimizer/scheduler settings to `optimization`. +and device to the run, and optimizer/scheduler settings to `global_training`. ## Optional committed import @@ -156,19 +194,19 @@ episodes_per_shard: 64 The output directory must be new. Import writes raw Parquet shards and a checksummed `manifest.json`, then atomically publishes the directory. It does not fit statistics, pad contexts or invent missing transitions. Train with -`data_sources.demonstrations.manifest: data/imported-demonstrations/manifest.json`; +`dataset_training.demonstrations.parts.recorded.manifest: data/imported-demonstrations/manifest.json`; the recorded column mappings are loaded automatically. The manifest retains source provenance, episode identities, optional successors and collection groups. ## Shared model and dataset authoring -The compact `policies` surface is optional. Policy training also accepts the -ordinary `model.backbone` and `model.interfaces` schemas, with a `policy` data -schema on each interface. Ingestion and decoder components keep their existing -syntax; the policy adapter adds the previous-action/return features. Use top-level -`context_length`, or `global_training.context_length`, to select a context smaller -than the backbone maximum. `global_training` accepts the policy optimization -settings as an alternative to `optimization` and uses `training_objective: policy`. +Policy training uses the ordinary `model.backbone` and `model.interfaces` +schemas, with a `policy` data schema on each interface. Ingestion and decoder +components keep their existing syntax; the policy adapter adds previous-action +and return features. `global_training.context_length` selects a context no larger +than the backbone maximum. `global_training` supplies optimizer settings and +`training_objective: policy`. The former compact policy registries and aliases +have been consolidated into this shared authoring surface. Likewise, `dataset_training.` accepts `model_interface` and `parts`, with each part containing a sequence source specification. Reference it with the ordinary @@ -184,33 +222,25 @@ for a complete example using the shared model, dataset and phase authoring surfa ## Configure the shared model -The policy's `backbone` accepts the architecture fields from ordinary training's -`model.backbone.architecture`. Optional `ingestion` and `decoder` use the same -component schemas and builders as forecasting. For example: +Configure the backbone at `model.backbone.architecture` and ingestion/decoder +components at `model.interfaces.`. For a larger model, for example: ```yaml -policies: - actor: - observation_columns: [position, velocity] - action_column: action - action_values: [-1.0, 1.0] - context_length: 16 - return_conditioned: true - backbone: +model: + backbone: + architecture: dim_model: 64 num_layers: 3 max_context_length: 16 attention: {n_heads: 4} feed_forward: {dim: 128, activation: swiglu} positional_encoding_scope: global - decoder: {type: mlp, hidden_dims: [64], prediction_length: 1, support: 1} ``` -Without an authored ingestion, the standard embedding ingestion projects real -features and embeds categorical features before an adapter maps to the backbone -input width. Real features receive one embedding channel and categorical features -share the remaining width. The default decoder is linear. A policy decoder must -have `support: 1` and `prediction_length: 1`. +Use an embedding ingestion to project real features and embed categorical +features before adapting to the backbone width. Supply ingestion and decoder +components explicitly as in the complete examples. A policy decoder must have +`support: 1` and `prediction_length: 1`. The internal input names `__policy_previous_action` and `__policy_return` are reserved. The latter exists only for return-conditioned policies. Static columns @@ -229,9 +259,8 @@ An observation may have the same external name as the action; in that case the model uses `__policy_action` as its separate target name. Default decoders infer the target automatically. -The default backbone has width 32, two layers, one attention head, and -feed-forward width 128. Context length must fit `max_context_length`. -Positional scope defaults to `global`; the standard ingestion also supports +Context length must fit `max_context_length`. +The standard ingestion also supports feature-scoped learned positions when configured. The standard attention, normalization, position encodings, dropout, and layer sharing options remain available. @@ -252,7 +281,7 @@ The portable model at `model_path` contains the best validation weights; the run checkpoint contains the latest training weights. ```yaml -optimization: +global_training: resume: policy: if_exists ``` @@ -357,7 +386,7 @@ execution requires `rollout.max_episode_steps: H`. No unknown tail is assigned zero. BC uses **all observed decisions**, including truncated and cutoff episodes, because its loss does not consume return labels. -Offline and online paths share `build_policy_inputs`, producing left-padded, +Offline and online paths share `PolicyWindowView.inputs`, producing left-padded, causal histories. The dataset slices to the context window before building inputs, retaining the actual preceding action at a cropped boundary. Each training example labels only its last position, so every decision contributes @@ -375,16 +404,24 @@ the start token, because earlier forecast seed actions are unknown. ```python from sequifier.config.train_config import load_train_config from sequifier.train import run_training -from sequifier.rl_api import load_policy, build_policy_inputs +from sequifier.rl_api import load_policy, PolicyWindowView config = load_train_config("configs/train-policy.yaml", {}, False) run_training(config) policy = load_policy(config.model_path, device="cpu") -# inputs = build_policy_inputs(policy.config, observations, previous_action_ids, -# desired_returns, static_inputs).to(policy.device) +# inputs = PolicyWindowView(policy.config).inputs( +# observations, previous_action_ids, desired_returns, static_inputs +# ).to(policy.device) # distribution = policy.distribution(inputs) ``` +Python callers use `SequifierInputs` for named model tensors and +`PolicyWindowView.inputs` for raw histories. `PolicySequenceDataset` accepts +`SequenceSourceConfig` entries or in-memory `SequenceEpisode` records; JSON logs +are adapted with `source_episodes`. The low-level `train_policy` takes training +and validation datasets plus a `ResolvedPolicyTrainConfig`, sharing the same +options and checkpoint lifecycle as `run_policy_training`. + `CategoricalPolicy` is an alignment adapter over the ordinary network, available as `policy.network`. `policy.execution_config` is the canonical model config, compatible with the shared artifact and export services. Distribution methods diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index 0721cd33..ef396a5e 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -23,9 +23,6 @@ Grouped episode splitting and training-only normalization remain in place. phases, critics, IQL/PPO, distributed policy learning, multimodal trajectories, and CLI ONNX policy execution remain future work. -No tests or training runs were executed, as requested. Validation used static -lint, formatting, and type checks. - Policy sequence representation follow-up: - Train BC directly from CSV/Parquet observation/action tables. Rewards, @@ -37,7 +34,9 @@ Policy sequence representation follow-up: - Categorical maps and sample statistics use shared preprocessing functions and fit only training episodes. New action vocabularies use global/decoder mappings; existing policy artifacts retain their legacy layout. -- Shared model/dataset/phase authoring is available alongside compact policy - configuration. Preprocess can optionally commit raw Parquet sequence shards +- Policy configuration uses shared model/dataset/phase authoring. Redundant + compact registries and dense input APIs have been removed. Preprocess can + optionally commit raw Parquet sequence shards with a checksummed manifest. -- No tests or training runs were written or executed for this change, as requested. +- Repository tests were moved to `../sequifier-test/tests/sequifier-refactor-unit`; + no tests were added. Verification uses lint, compilation and workflow smoke runs. diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 53d1c959..857362cc 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -83,12 +83,16 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig from sequifier.config.train_config import ResolvedSequifierConfig + training_config = dict(payload["training_config"]) config_type = ( ResolvedPolicyTrainConfig - if "trajectory_sources" in payload["training_config"] + if "trajectory_sources" in training_config else ResolvedSequifierConfig ) + if config_type is ResolvedPolicyTrainConfig and "actor_path" in training_config: + training_config["model_path"] = training_config.pop("actor_path") + if ( payload.get("initialization_policy_version") != 1 or payload.get("initialization_seed_derivation_version") != 1 @@ -106,7 +110,7 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": integration_state=IntegrationState( values=dict(payload["integration_state"]["values"]) ), - training_config=config_type.model_validate(payload["training_config"]), + training_config=config_type.model_validate(training_config), ) checkpoint.validate() return checkpoint diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 13851f96..3d761abd 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -14,6 +14,8 @@ ) from sequifier.io.values import DomainValue +NonemptyStr = Annotated[str, Field(min_length=1)] + class PolicyConfigBase(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) @@ -106,14 +108,14 @@ def validate_names(self): reserved = {"__policy_previous_action", "__policy_return", "__policy_action"} if reserved.intersection(names + [self.action_column]): raise ValueError("policy feature names use reserved internal columns") - if set(self.observation_n_classes) - set(self.observation_columns) or any( - count < 2 for count in self.observation_n_classes.values() + for counts, columns in ( + (self.observation_n_classes, self.observation_columns), + (self.static_n_classes, self.static_columns), ): - raise ValueError( - "observation_n_classes requires observation columns with at least two classes" - ) - if set(self.static_n_classes) - set(self.static_columns): - raise ValueError("static_n_classes must name static columns") + if counts.keys() - set(columns): + raise ValueError( + "class counts must name declared observation/static columns" + ) counts = {**self.observation_n_classes, **self.static_n_classes} if len(set(self.categorical_columns)) != len(self.categorical_columns) or set( self.categorical_columns @@ -174,8 +176,3 @@ class PolicyOptimizationConfig(PolicyConfigBase): scheduler_step_on: Literal["epoch"] = "epoch" accumulation_steps: Literal[1] = 1 layer_type_dtypes: None = None - - -class PolicyTrainingConfig(PolicyOptimizationConfig): - epochs: int = Field(default=80, gt=0) - seed: int = Field(default=1010, ge=0) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index b1b99f79..6a9e8dfe 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -3,10 +3,11 @@ from pathlib import Path from typing import Literal -from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator +from pydantic import Field, field_validator, model_validator from sequifier.config.policy_config import ( CategoricalPolicyConfig, + NonemptyStr, PolicyConfigBase, PolicyOptimizationConfig, ) @@ -19,38 +20,24 @@ class PolicyTableColumns(PolicyConfigBase): """Explicit row timing: current observation/action, then resulting reward.""" - sequence_id: str = "sequenceId" - item_position: str = "itemPosition" - observations: dict[str, str] = Field(default_factory=dict) - static_inputs: dict[str, str] = Field(default_factory=dict) - action: str | None = None - reward: str | None = "reward" - terminated: str | None = "terminated" - truncated: str | None = "truncated" - previous_action: str | None = None - successors: dict[str, str] = Field(default_factory=dict) - bootstrap_allowed: str | None = None - group: str | None = None - episode_id: str | None = None - - @model_validator(mode="after") - def validate_columns(self): - names = [ - value for value in self.model_dump().values() if isinstance(value, str) - ] - names += ( - list(self.observations.values()) - + list(self.static_inputs.values()) - + list(self.successors.values()) - ) - if any(not value for value in names): - raise ValueError("table column names must be nonempty") - return self + sequence_id: NonemptyStr = "sequenceId" + item_position: NonemptyStr = "itemPosition" + observations: dict[str, NonemptyStr] = Field(default_factory=dict) + static_inputs: dict[str, NonemptyStr] = Field(default_factory=dict) + action: NonemptyStr | None = None + reward: NonemptyStr | None = "reward" + terminated: NonemptyStr | None = "terminated" + truncated: NonemptyStr | None = "truncated" + previous_action: NonemptyStr | None = None + successors: dict[str, NonemptyStr] = Field(default_factory=dict) + bootstrap_allowed: NonemptyStr | None = None + group: NonemptyStr | None = None + episode_id: NonemptyStr | None = None class SequenceSourceConfig(PolicyConfigBase): - paths: list[str] = Field(default_factory=list) - manifest: str | None = None + paths: list[NonemptyStr] = Field(default_factory=list) + manifest: NonemptyStr | None = None format: Literal["auto", "trajectory", "table", "prepared"] = "auto" columns: PolicyTableColumns = Field(default_factory=PolicyTableColumns) # A file boundary alone says nothing about task completion. @@ -69,24 +56,17 @@ def validate_location(self): ) if sum((bool(self.paths), self.manifest is not None, metadata_split)) != 1: raise ValueError("trajectory source requires either paths or one manifest") - if any(not path for path in self.paths) or self.manifest == "": - raise ValueError("trajectory paths must be nonempty") - if self.split_index is not None and self.metadata_config_path is None: - raise ValueError("split_index requires metadata_config_path") - if self.encoding == "model" and self.metadata_config_path is None: - raise ValueError("model-encoded sources require metadata_config_path") - if self.format == "prepared" and self.metadata_config_path is None: - raise ValueError("prepared sources require metadata_config_path") + if ( + self.split_index is not None + or self.encoding == "model" + or self.format == "prepared" + ) and self.metadata_config_path is None: + raise ValueError( + "prepared/model-encoded sources and split_index require metadata_config_path" + ) return self -# Existing Python callers and authored trajectory_sources remain supported. -TrajectorySourceConfig = SequenceSourceConfig - - -PolicyTrainingPlanConfig = PolicyTrainingPlanModel - - def resolve_sequence_source( source: SequenceSourceConfig, root: Path ) -> SequenceSourceConfig: @@ -95,9 +75,8 @@ def resolve_sequence_source( for key in ("manifest", "metadata_config_path", "static_data_path"): if values[key] is not None: values[key] = str((root / values[key]).resolve()) - if ( - values["metadata_config_path"] is not None - and values["static_data_path"] is None + if source.metadata_config_path is not None and ( + source.static_data_path is None or source.split_index is not None ): from sequifier.config.metadata import load_dataset_metadata @@ -107,34 +86,28 @@ def resolve_sequence_source( or source.split_index is not None or any(Path(path).suffix.lower() == ".pt" for path in source.paths) ) - if prepared and metadata.static_features_path: + if ( + prepared + and source.static_data_path is None + and metadata.static_features_path + ): values["static_data_path"] = str( (root / metadata.static_features_path).resolve() ) - if source.split_index is not None: - from sequifier.config.metadata import load_dataset_metadata - - metadata_path = values["metadata_config_path"] - if not isinstance(metadata_path, str): - raise ValueError("prepared split sources require metadata_config_path") - metadata = load_dataset_metadata(metadata_path) - if source.split_index >= len(metadata.split_paths): - raise ValueError( - "source split_index is outside the prepared metadata splits" - ) - path = (root / metadata.split_paths[source.split_index]).resolve() - values["paths"] = ( - sorted( - str(item) - for item in path.glob( - "*.pt" if path.is_dir() and any(path.glob("*.pt")) else "*.parquet" + if source.split_index is not None: + if source.split_index >= len(metadata.split_paths): + raise ValueError( + "source split_index is outside the prepared metadata splits" ) + path = (root / metadata.split_paths[source.split_index]).resolve() + pattern = "*.pt" if any(path.glob("*.pt")) else "*.parquet" + values.update( + paths=sorted(str(item) for item in path.glob(pattern)) + if path.is_dir() + else [str(path)], + format="prepared", + split_index=None, ) - if path.is_dir() - else [str(path)] - ) - values["format"] = "prepared" - values["split_index"] = None return SequenceSourceConfig.model_validate(values) @@ -144,12 +117,9 @@ class _PolicyTrainConfigBase(PolicyConfigBase): device: str = "cpu" initialize_from: str | None = None seed: int = Field(default=1010, ge=0) - trajectory_sources: dict[str, SequenceSourceConfig] = Field( - min_length=1, - validation_alias=AliasChoices("data_sources", "trajectory_sources"), - ) + trajectory_sources: dict[str, SequenceSourceConfig] = Field(min_length=1) policies: dict[str, CategoricalPolicyConfig] = Field(min_length=1) - training_plan: PolicyTrainingPlanConfig + training_plan: PolicyTrainingPlanModel # Epochs and seed belong to the phase and run respectively. optimization: PolicyOptimizationConfig = Field( default_factory=PolicyOptimizationConfig @@ -167,135 +137,6 @@ def validate_model_name(cls, value): raise ValueError("model_name must be a single path component") return value - @model_validator(mode="before") - @classmethod - def normalize_plan(cls, values): - if not isinstance(values, dict): - return values - values = dict(values) - source_interfaces = {} - if "global_training" in values: - if "optimization" in values: - raise ValueError("configure global_training or optimization, not both") - training = dict(values.pop("global_training")) - context = training.pop("context_length", None) - if context is not None: - values["context_length"] = context - if training.pop("training_objective", "policy") != "policy": - raise ValueError("policy phases require training_objective: policy") - values["optimization"] = training - if "dataset_training" in values: - if "data_sources" in values or "trajectory_sources" in values: - raise ValueError("configure dataset_training or data_sources, not both") - sources = {} - for name, dataset in values.pop("dataset_training").items(): - if set(dataset) - {"model_interface", "parts"}: - raise ValueError( - "policy datasets configure model_interface and source parts; action NLL is supplied by the algorithm" - ) - if not dataset.get("parts"): - raise ValueError("policy datasets require nonempty source parts") - for part, source in dataset["parts"].items(): - sources[f"{name}.{part}"] = source - source_interfaces[f"{name}.{part}"] = dataset.get("model_interface") - values["data_sources"] = sources - # Canonical model authoring is shared with supervised learning. The - # compact policies registry remains a compatibility/convenience surface. - if "model" in values: - if "policies" in values: - raise ValueError("configure model or policies, not both") - from sequifier.config.train_config import ModelSpecModel - - model = ModelSpecModel.model_validate(values.pop("model")) - policies = {} - for name, interface in model.interfaces.items(): - if interface.policy is None: - raise ValueError( - "policy training model interfaces require a policy schema" - ) - schema = interface.policy.model_dump() - expected = schema["observation_columns"] + schema["static_columns"] - if interface.input_columns and interface.input_columns != expected: - raise ValueError( - "policy interface inputs must name its observation and static columns in order" - ) - target = schema["action_column"] - if interface.target_columns != [target]: - raise ValueError( - "policy interface targets must name its action column" - ) - if ( - interface.categorical_hashing - or interface.static_conditioning - or interface.preconditioning_image_encoder - or interface.feature_layout - ): - raise ValueError( - "this policy learner currently supports scalar policy inputs" - ) - policies[name] = { - **schema, - "interface_name": name, - "backbone": model.backbone.architecture.model_dump(), - "backbone_component": model.backbone.model_dump(), - "context_length": values.get( - "context_length", model.backbone.architecture.max_context_length - ), - "ingestion": interface.ingestion.model_dump(), - "decoder": interface.decoder.model_dump(), - } - values.pop("context_length", None) - values["policies"] = policies - elif "context_length" in values: - context = values.pop("context_length") - values["policies"] = { - name: {**policy, "context_length": context} - for name, policy in values.get("policies", {}).items() - } - plan = values.get("training_plan") - if plan is not None and isinstance(plan, BaseModel): - plan = plan.model_dump() - values["training_plan"] = plan - if isinstance(plan, dict) and "phases" not in plan: - values["training_plan"] = {"phases": [plan]} - plan = values.get("training_plan", {}) - if not isinstance(plan, dict): - plan = {} - normalized_phases = [] - for phase in plan.get("phases", []): - if isinstance(phase, BaseModel): - phase = phase.model_dump() - if not isinstance(phase, dict): - normalized_phases.append(phase) - continue - phase = dict(phase) - if "policy" not in phase and len(values.get("policies", {})) == 1: - phase["policy"] = next(iter(values["policies"])) - for reference in phase.get("sources", phase.get("trajectory_sources", [])): - if isinstance(reference, BaseModel): - reference = reference.model_dump() - name = ( - reference - if isinstance(reference, str) - else reference.get("source") - if isinstance(reference, dict) - else None - ) - if source_interfaces.get(name) is not None and source_interfaces[ - name - ] != phase.get("policy"): - raise ValueError( - "policy dataset model_interface disagrees with its phase policy" - ) - # Work on copied phases, never mutate the caller's authored mapping. - normalized_phases.append(phase) - if plan.get("phases"): - values["training_plan"] = {**plan, "phases": normalized_phases} - optimization = values.get("optimization", {}) - if isinstance(optimization, dict) and {"epochs", "seed"} & optimization.keys(): - raise ValueError("set epochs on the phase and seed on the run") - return values - @model_validator(mode="after") def validate_references(self): if len(self.training_plan.phases) != 1 or not isinstance( @@ -315,36 +156,98 @@ def validate_references(self): raise ValueError("training and validation sources must be disjoint") return self - @field_validator("actor_path", "report_path", check_fields=False) - @classmethod - def validate_output_path(cls, value): - if value == "": - raise ValueError("output paths must be nonempty") - return value - class PolicyTrainConfig(_PolicyTrainConfigBase): - """Authored paths may be omitted to select the standard output locations.""" - - actor_path: str | None = Field( - default=None, validation_alias=AliasChoices("model_path", "actor_path") - ) - report_path: str | None = None + """Canonical model/dataset authoring, resolved before field validation.""" - @property - def model_path(self) -> str | None: - return self.actor_path + model_path: NonemptyStr | None = None + report_path: NonemptyStr | None = None + @model_validator(mode="before") + @classmethod + def resolve_authoring(cls, values): + from sequifier.config.train_config import ModelSpecModel -class ResolvedPolicyTrainConfig(_PolicyTrainConfigBase): - """Run paths are absolute and the authored phase has been validated.""" + values = dict(values) + if { + "policies", + "trajectory_sources", + "data_sources", + "optimization", + "actor_path", + } & values.keys(): + raise ValueError( + "use model, dataset_training, global_training and model_path" + ) + model = ModelSpecModel.model_validate(values.pop("model", None)) + optimization = dict(values.pop("global_training", {})) + context = optimization.pop( + "context_length", model.backbone.architecture.max_context_length + ) + if optimization.pop("training_objective", "policy") != "policy": + raise ValueError("policy phases require training_objective: policy") + values["optimization"] = optimization + policies = {} + for name, interface in model.interfaces.items(): + schema = interface.policy + if schema is None: + raise ValueError("policy model interfaces require a policy schema") + if ( + interface.input_columns + and interface.input_columns + != schema.observation_columns + schema.static_columns + ): + raise ValueError( + "policy inputs must list observations and static columns in order" + ) + if interface.target_columns != [schema.action_column]: + raise ValueError("policy targets must name its action column") + if any( + ( + interface.categorical_hashing, + interface.static_conditioning, + interface.preconditioning_image_encoder, + interface.feature_layout, + ) + ): + raise ValueError("policy learning supports scalar inputs") + policies[name] = CategoricalPolicyConfig( + **schema.model_dump(), + interface_name=name, + context_length=context, + backbone=model.backbone.architecture, + backbone_component=model.backbone, + ingestion=interface.ingestion, + decoder=interface.decoder, + ) + values["policies"] = policies + phase = PolicyTrainingPlanModel.model_validate( + values.get("training_plan") + ).phases[0] + sources = {} + for name, dataset in values.pop("dataset_training", {}).items(): + if set(dataset) - {"model_interface", "parts"} or not dataset.get("parts"): + raise ValueError( + "policy datasets require model_interface and nonempty source parts" + ) + for part, source in dataset["parts"].items(): + reference = f"{name}.{part}" + if ( + reference + in phase.trajectory_sources + (phase.validation_sources or []) + and dataset.get("model_interface") != phase.policy + ): + raise ValueError( + "dataset model_interface disagrees with phase policy" + ) + sources[reference] = source + values["trajectory_sources"] = sources + return values - actor_path: str = Field(validation_alias=AliasChoices("model_path", "actor_path")) - report_path: str - @property - def model_path(self) -> str: - return self.actor_path +class ResolvedPolicyTrainConfig(_PolicyTrainConfigBase): + model_path: NonemptyStr + report_path: NonemptyStr def resolve_policy_train_config(config: PolicyTrainConfig) -> ResolvedPolicyTrainConfig: @@ -361,12 +264,12 @@ def absolute(path): } if config.initialize_from is not None: values["initialize_from"] = absolute(config.initialize_from) - values["actor_path"] = absolute( - config.actor_path or f"models/{config.model_name}/{config.model_name}.pt" + values["model_path"] = absolute( + config.model_path or f"models/{config.model_name}/{config.model_name}.pt" ) values["report_path"] = absolute( config.report_path or f"outputs/training/{config.model_name}/report.json" ) - if values["actor_path"] == values["report_path"]: - raise ValueError("actor_path and report_path must differ") + if values["model_path"] == values["report_path"]: + raise ValueError("model_path and report_path must differ") return ResolvedPolicyTrainConfig.model_validate(values) diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py index c92c46aa..1674984f 100644 --- a/src/sequifier/config/training_plan.py +++ b/src/sequifier/config/training_plan.py @@ -2,14 +2,7 @@ from typing import Annotated, Generic, Literal, Optional, TypeVar -from pydantic import ( - AliasChoices, - BaseModel, - ConfigDict, - Field, - field_validator, - model_validator, -) +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from sequifier.config.identifiers import identifier as _identifier from sequifier.typechecking import beartype @@ -77,16 +70,7 @@ class ReturnConditionedPhaseConfig(BaseModel): algorithm: Literal["return_conditioned"] name: str = Field(default="policy", min_length=1) epochs: int = Field(gt=0) - sources: list[TrainingSourceSpecModel] = Field( - min_length=1, validation_alias=AliasChoices("sources", "trajectory_sources") - ) - - @field_validator("sources", mode="before") - @classmethod - def normalize_sources(cls, values): - return [ - {"source": value} if isinstance(value, str) else value for value in values - ] + sources: list[TrainingSourceSpecModel] = Field(min_length=1) @model_validator(mode="after") def validate_sources(self): diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 909fc9a2..c5b5a4a3 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -1719,37 +1719,25 @@ healthy active update. Momentum and weight decay cannot update parameters whose gradients are absent. -## Trajectory-backed policy training - -`sequifier train` also accepts one `return_conditioned` phase with named -trajectory sources and a categorical policy registry. This workflow supports -return conditioning or BC, with no supervised dataset placeholders. See the -[policy training guide](../policy-learning.md) and -[complete example](../example_inputs/configs/train-return-conditioned.yaml). -The phase owns `epochs`; the run owns `seed`; `optimization` configures updates. -Existing phases default to `algorithm: supervised`. Mixed algorithm plans and -IQL/PPO remain planned. - -Policy models use the shared configurable network and portable artifacts. They -support CPU/CUDA/MPS, backbone transfer with `initialize_from`, bounded file-backed -episode loading, and epoch-boundary resume through `optimization.resume`. - -## Policy sequence sources and shared authoring - -Policy phases accept ordinary CSV/Parquet demonstrations with `sequenceId` and -`itemPosition`. Use `data_sources` and phase `sources: [{source: demonstrations}]`; -the previous `trajectory_sources` spellings remain accepted. BC does not require -rewards or successor states. Return-conditioned training requires rewards and a -known return endpoint. JSON rollout logs are adapted into the same sequence data. - -Policy training can use the ordinary `model.backbone`, `model.interfaces` and -`dataset_training..parts` authoring patterns. Put the model-independent -observation/action schema in an interface's `policy` field and a sequence source -in each dataset part. The algorithm supplies action NLL. `global_training` can -supply the policy optimization settings and `context_length`, with -`training_objective: policy`; the compact `policies`/`optimization` surface remains -supported. One policy phase is currently supported. See the complete -[table training example](../example_inputs/configs/train-policy-from-table.yaml) +## Policy sequence training + +`sequifier train` accepts one `return_conditioned` phase. Put the observation/action +schema in `model.interfaces..policy`, a sequence source in each +`dataset_training..parts.`, and optimizer/context settings in +`global_training` with `training_objective: policy`. Reference sources with +`{source: dataset.part}`. The algorithm supplies action NLL. + +`return_conditioned: false` selects BC, which needs observations and action labels. +Return-conditioned training additionally requires rewards and a known return +endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories all +use the same episode representation. Models support CPU/CUDA/MPS, backbone +transfer via `initialize_from`, and epoch-boundary resume via +`global_training.resume`. Mixed phases, IQL/PPO and image/depth policies remain +planned. The former compact registries and configuration aliases have been +consolidated into the shared model/dataset authoring surface. + +See the [complete table example](../example_inputs/configs/train-policy-from-table.yaml), +[return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml) and [policy learning guide](../policy-learning.md). @@ -2368,22 +2356,58 @@ unique and contiguous within each sequence, including across input shards. ```yaml project_root: . -model_name: policy -data_sources: +model_name: table-policy +device: cpu +seed: 1010 + +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128} + positional_encoding_scope: global + interfaces: + actor: + input_columns: [position, velocity] + target_columns: [action] + policy: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: + position: 1 + velocity: 1 + __policy_previous_action: 6 + decoder: {type: linear, prediction_length: 1, support: 1} + +dataset_training: demonstrations: - paths: [data/demonstrations.parquet] -policies: - actor: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - context_length: 16 - return_conditioned: false + model_interface: actor + parts: + recorded: + paths: [data/demonstrations.parquet] + +global_training: + training_objective: policy + context_length: 16 + batch_size: 64 + learning_rate: 0.005 + validation_fraction: 0.2 + training_plan: - algorithm: return_conditioned - epochs: 80 - sources: [{source: demonstrations}] - policy: actor + phases: + - name: behavior_cloning + algorithm: return_conditioned + policy: actor + epochs: 80 + sources: [{source: demonstrations.recorded}] ``` Run `sequifier train --config-path configs/train-policy.yaml`. BC @@ -2400,19 +2424,22 @@ Defaults use the policy feature names and the columns `sequenceId`, explicitly: ```yaml -data_sources: +dataset_training: demonstrations: - paths: [data/recorded.csv] - columns: - sequence_id: episode - item_position: step - observations: {position: measured_position, velocity: measured_velocity} - action: command - reward: score - terminated: task_finished - truncated: time_limit - previous_action: preceding_command - group: collection_session + model_interface: actor + parts: + recorded: + paths: [data/recorded.csv] + columns: + sequence_id: episode + item_position: step + observations: {position: measured_position, velocity: measured_velocity} + action: command + reward: score + terminated: task_finished + truncated: time_limit + previous_action: preceding_command + group: collection_session ``` Boundary columns, when present, contain booleans. No row may follow a boundary @@ -2440,7 +2467,7 @@ metadata rather than model features. Without `validation_sources`, training splits whole episodes deterministically, keeping repeated demonstrations and declared collection groups together. At least two independent groups are required. Alternatively declare a held-out source -and set `validation_sources: [held_out]` on the phase. Duplicate paths and episode +and set `validation_sources: [held_out.recorded]` on the phase. Duplicate paths and episode IDs, overlapping collection groups and repeated demonstrations across partitions are rejected. Normalization and categorical vocabulary fitting use training data. @@ -2470,20 +2497,19 @@ A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy adapted into the same sequence representation, with bindings retained in their recording format. Forecast recordings require `action_semantics: executed_predictions`; this does not convert forecast checkpoint -alignment into policy alignment. The old `trajectory_sources` registry and phase -field remain aliases for `data_sources` and `sources`. +alignment into policy alignment. Run paths resolve relative to `project_root`; manifest entries resolve relative -to the manifest directory. `optimization.num_workers` controls loader workers; +to the manifest directory. `global_training.num_workers` controls loader workers; `episode_cache_size` bounds cached episodes per worker. Table-backed episodes use filtered lazy scans. Prepared window reconstruction is eager per source and JSON records still require one episode to fit in memory. The best model is selected by held-out action NLL. `model_path` defaults to `models//.pt` and contains an ordinary `sequifier_model`. -`actor_path` remains an alias. The report records partitions, exclusions, the fitted +The report records partitions, exclusions, the fitted policy codec, decision counts and per-epoch NLL. Epochs belong to the phase, seed -and device to the run, and optimizer/scheduler settings to `optimization`. +and device to the run, and optimizer/scheduler settings to `global_training`. ## Optional committed import @@ -2507,19 +2533,19 @@ episodes_per_shard: 64 The output directory must be new. Import writes raw Parquet shards and a checksummed `manifest.json`, then atomically publishes the directory. It does not fit statistics, pad contexts or invent missing transitions. Train with -`data_sources.demonstrations.manifest: data/imported-demonstrations/manifest.json`; +`dataset_training.demonstrations.parts.recorded.manifest: data/imported-demonstrations/manifest.json`; the recorded column mappings are loaded automatically. The manifest retains source provenance, episode identities, optional successors and collection groups. ## Shared model and dataset authoring -The compact `policies` surface is optional. Policy training also accepts the -ordinary `model.backbone` and `model.interfaces` schemas, with a `policy` data -schema on each interface. Ingestion and decoder components keep their existing -syntax; the policy adapter adds the previous-action/return features. Use top-level -`context_length`, or `global_training.context_length`, to select a context smaller -than the backbone maximum. `global_training` accepts the policy optimization -settings as an alternative to `optimization` and uses `training_objective: policy`. +Policy training uses the ordinary `model.backbone` and `model.interfaces` +schemas, with a `policy` data schema on each interface. Ingestion and decoder +components keep their existing syntax; the policy adapter adds previous-action +and return features. `global_training.context_length` selects a context no larger +than the backbone maximum. `global_training` supplies optimizer settings and +`training_objective: policy`. The former compact policy registries and aliases +have been consolidated into this shared authoring surface. Likewise, `dataset_training.` accepts `model_interface` and `parts`, with each part containing a sequence source specification. Reference it with the ordinary @@ -2535,33 +2561,25 @@ for a complete example using the shared model, dataset and phase authoring surfa ## Configure the shared model -The policy's `backbone` accepts the architecture fields from ordinary training's -`model.backbone.architecture`. Optional `ingestion` and `decoder` use the same -component schemas and builders as forecasting. For example: +Configure the backbone at `model.backbone.architecture` and ingestion/decoder +components at `model.interfaces.`. For a larger model, for example: ```yaml -policies: - actor: - observation_columns: [position, velocity] - action_column: action - action_values: [-1.0, 1.0] - context_length: 16 - return_conditioned: true - backbone: +model: + backbone: + architecture: dim_model: 64 num_layers: 3 max_context_length: 16 attention: {n_heads: 4} feed_forward: {dim: 128, activation: swiglu} positional_encoding_scope: global - decoder: {type: mlp, hidden_dims: [64], prediction_length: 1, support: 1} ``` -Without an authored ingestion, the standard embedding ingestion projects real -features and embeds categorical features before an adapter maps to the backbone -input width. Real features receive one embedding channel and categorical features -share the remaining width. The default decoder is linear. A policy decoder must -have `support: 1` and `prediction_length: 1`. +Use an embedding ingestion to project real features and embed categorical +features before adapting to the backbone width. Supply ingestion and decoder +components explicitly as in the complete examples. A policy decoder must have +`support: 1` and `prediction_length: 1`. The internal input names `__policy_previous_action` and `__policy_return` are reserved. The latter exists only for return-conditioned policies. Static columns @@ -2580,9 +2598,8 @@ An observation may have the same external name as the action; in that case the model uses `__policy_action` as its separate target name. Default decoders infer the target automatically. -The default backbone has width 32, two layers, one attention head, and -feed-forward width 128. Context length must fit `max_context_length`. -Positional scope defaults to `global`; the standard ingestion also supports +Context length must fit `max_context_length`. +The standard ingestion also supports feature-scoped learned positions when configured. The standard attention, normalization, position encodings, dropout, and layer sharing options remain available. @@ -2603,7 +2620,7 @@ The portable model at `model_path` contains the best validation weights; the run checkpoint contains the latest training weights. ```yaml -optimization: +global_training: resume: policy: if_exists ``` @@ -2708,7 +2725,7 @@ execution requires `rollout.max_episode_steps: H`. No unknown tail is assigned zero. BC uses **all observed decisions**, including truncated and cutoff episodes, because its loss does not consume return labels. -Offline and online paths share `build_policy_inputs`, producing left-padded, +Offline and online paths share `PolicyWindowView.inputs`, producing left-padded, causal histories. The dataset slices to the context window before building inputs, retaining the actual preceding action at a cropped boundary. Each training example labels only its last position, so every decision contributes @@ -2726,16 +2743,24 @@ the start token, because earlier forecast seed actions are unknown. ```python from sequifier.config.train_config import load_train_config from sequifier.train import run_training -from sequifier.rl_api import load_policy, build_policy_inputs +from sequifier.rl_api import load_policy, PolicyWindowView config = load_train_config("configs/train-policy.yaml", {}, False) run_training(config) policy = load_policy(config.model_path, device="cpu") -# inputs = build_policy_inputs(policy.config, observations, previous_action_ids, -# desired_returns, static_inputs).to(policy.device) +# inputs = PolicyWindowView(policy.config).inputs( +# observations, previous_action_ids, desired_returns, static_inputs +# ).to(policy.device) # distribution = policy.distribution(inputs) ``` +Python callers use `SequifierInputs` for named model tensors and +`PolicyWindowView.inputs` for raw histories. `PolicySequenceDataset` accepts +`SequenceSourceConfig` entries or in-memory `SequenceEpisode` records; JSON logs +are adapted with `source_episodes`. The low-level `train_policy` takes training +and validation datasets plus a `ResolvedPolicyTrainConfig`, sharing the same +options and checkpoint lifecycle as `run_policy_training`. + `CategoricalPolicy` is an alignment adapter over the ordinary network, available as `policy.network`. `policy.execution_config` is the canonical model config, compatible with the shared artifact and export services. Distribution methods diff --git a/src/sequifier/io/batch.py b/src/sequifier/io/batch.py index 1af074e8..b65bd179 100644 --- a/src/sequifier/io/batch.py +++ b/src/sequifier/io/batch.py @@ -1,16 +1,16 @@ -from collections.abc import Sequence from dataclasses import dataclass from typing import Optional import torch +from sequifier.io.tensors import TensorRecord, collate_tensor_records from sequifier.typechecking import beartype REQUIRED_METADATA_KEYS = frozenset({"attention_valid_mask", "target_valid_mask"}) @dataclass(frozen=True) -class SequifierInputs: +class SequifierInputs(TensorRecord): """Named model tensors, independent of labels or learning algorithm.""" inputs: dict[str, torch.Tensor] @@ -20,43 +20,12 @@ class SequifierInputs: def attention_valid_mask(self) -> torch.Tensor: return self.metadata["attention_valid_mask"] - def to(self, device: str | torch.device) -> "SequifierInputs": - return SequifierInputs( - { - key: value.to(device, non_blocking=True) - for key, value in self.inputs.items() - }, - { - key: value.to(device, non_blocking=True) - for key, value in self.metadata.items() - }, - ) - -def collate_model_inputs(examples: Sequence[SequifierInputs]) -> SequifierInputs: - if not examples: - raise ValueError("cannot collate empty model inputs") - first = examples[0] - if any( - value.inputs.keys() != first.inputs.keys() - or value.metadata.keys() != first.metadata.keys() - for value in examples - ): - raise ValueError("model inputs must share feature and metadata schemas") - return SequifierInputs( - { - key: torch.cat([value.inputs[key] for value in examples]) - for key in first.inputs - }, - { - key: torch.cat([value.metadata[key] for value in examples]) - for key in first.metadata - }, - ) +collate_model_inputs = collate_tensor_records @dataclass(frozen=True) -class SequifierBatch: +class SequifierBatch(TensorRecord): inputs: dict[str, torch.Tensor] targets: dict[str, torch.Tensor] metadata: dict[str, torch.Tensor] @@ -67,21 +36,6 @@ class SequifierBatch: def model_inputs(self) -> SequifierInputs: return SequifierInputs(self.inputs, self.metadata) - def to(self, device: str | torch.device) -> "SequifierBatch": - model_inputs = self.model_inputs.to(device) - return SequifierBatch( - model_inputs.inputs, - { - key: value.to(device, non_blocking=True) - for key, value in self.targets.items() - }, - model_inputs.metadata, - self.sequence_ids.to(device) if self.sequence_ids is not None else None, - self.subsequence_ids.to(device) - if self.subsequence_ids is not None - else None, - ) - @beartype def __post_init__(self) -> None: metadata_keys = self.metadata.keys() if self.metadata is not None else set() @@ -92,46 +46,5 @@ def __post_init__(self) -> None: f"SequifierBatch metadata is missing required keys: {missing}" ) - @beartype - def pin_memory(self) -> "SequifierBatch": - return SequifierBatch( - inputs={k: v.pin_memory() for k, v in self.inputs.items()}, - targets={k: v.pin_memory() for k, v in self.targets.items()}, - metadata={k: v.pin_memory() for k, v in self.metadata.items()}, - sequence_ids=( - self.sequence_ids.pin_memory() - if self.sequence_ids is not None - else None - ), - subsequence_ids=( - self.subsequence_ids.pin_memory() - if self.subsequence_ids is not None - else None - ), - ) - - -def collate_sequence_batches(examples: Sequence[SequifierBatch]) -> SequifierBatch: - model_inputs = collate_model_inputs([value.model_inputs for value in examples]) - first = examples[0] - if any(value.targets.keys() != first.targets.keys() for value in examples): - raise ValueError("sequence batches must share target schemas") - - def identities(name): - values = [getattr(value, name) for value in examples] - if all(value is None for value in values): - return None - if any(value is None for value in values): - raise ValueError(f"sequence batches must consistently supply {name}") - return torch.cat(values) - return SequifierBatch( - model_inputs.inputs, - { - key: torch.cat([value.targets[key] for value in examples]) - for key in first.targets - }, - model_inputs.metadata, - identities("sequence_ids"), - identities("subsequence_ids"), - ) +collate_sequence_batches = collate_tensor_records diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index 54522712..8cc7965d 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -5,10 +5,8 @@ import random from bisect import bisect_right from collections import OrderedDict -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from decimal import Decimal -from pathlib import Path -from typing import Literal, overload import polars as pl import torch @@ -16,7 +14,7 @@ from sequifier.config.policy_config import CategoricalPolicyConfig from sequifier.config.policy_train_config import SequenceSourceConfig -from sequifier.io.batch import SequifierBatch, collate_sequence_batches +from sequifier.io.batch import SequifierBatch from sequifier.io.policy_window import ( PolicyWindowView, action_global_ids, @@ -31,57 +29,33 @@ ) from sequifier.io.sequence import SequenceEpisode from sequifier.io.values import numeric_scalar -from sequifier.trajectory.importers import ( - TrajectoryReference, - sequence_from_trajectory, - source_episodes, -) +from sequifier.trajectory.importers import source_episodes from sequifier.trajectory.returns import return_to_go -from sequifier.trajectory.schema import Trajectory - -# Compatibility names now refer to the standard supervised batch contract. -PolicyExample = SequifierBatch -collate_policy_examples = collate_sequence_batches -def _load(source, config, action_semantics) -> SequenceEpisode: +def _load(source) -> SequenceEpisode: if isinstance(source, SequenceEpisode): return SequenceEpisode.model_validate(source.model_dump()) - if isinstance(source, Trajectory): - return sequence_from_trajectory(source, config, action_semantics) - if isinstance(source, (str, Path)): - return TrajectoryReference(Path(source), config, action_semantics).load() return source.load() -class EpisodeCollection(Sequence[SequenceEpisode]): +class EpisodeCollection: """Keep source-backed episodes on disk, with a bounded per-worker cache.""" - def __init__(self, sources, cache_size, config, action_semantics): + def __init__(self, sources, cache_size): self.sources = list(sources) self.cache_size = cache_size - self.config = config - self.action_semantics = action_semantics self.cache: OrderedDict[int, SequenceEpisode] = OrderedDict() def __len__(self): return len(self.sources) - @overload - def __getitem__(self, index: int) -> SequenceEpisode: ... + def __iter__(self) -> Iterator[SequenceEpisode]: + return (self[index] for index in range(len(self))) - @overload - def __getitem__(self, index: slice) -> Sequence[SequenceEpisode]: ... - - def __getitem__( - self, index: int | slice - ) -> SequenceEpisode | Sequence[SequenceEpisode]: - if isinstance(index, slice): - return [self[i] for i in range(*index.indices(len(self)))] + def __getitem__(self, index: int) -> SequenceEpisode: if index not in self.cache: - self.cache[index] = _load( - self.sources[index], self.config, self.action_semantics - ) + self.cache[index] = _load(self.sources[index]) if len(self.cache) > self.cache_size: self.cache.popitem(last=False) self.cache.move_to_end(index) @@ -100,25 +74,17 @@ def _canonical(value): def episode_content_key(episode, *, length=None, returns=None) -> str: """Hash learning content, independently of log format, IDs and provenance.""" - if isinstance(episode, Trajectory): - values = { - "observations": [step.observation for step in episode.transitions], - "actions": [step.predictions for step in episode.transitions], - "static_inputs": episode.seed.static_inputs, - } - else: - length = len(episode.actions) if length is None else length - values = { - "observations": { - column: entries[:length] - for column, entries in episode.observations.items() - }, - "actions": episode.actions[:length], - "static_inputs": episode.static_inputs, - "previous_action": episode.previous_action, - "starts_episode": episode.starts_episode, - "returns": returns, - } + length = len(episode.actions) if length is None else length + values = { + "observations": { + column: entries[:length] for column, entries in episode.observations.items() + }, + "actions": episode.actions[:length], + "static_inputs": episode.static_inputs, + "previous_action": episode.previous_action, + "starts_episode": episode.starts_episode, + "returns": returns, + } return hashlib.sha256( json.dumps(_canonical(values), sort_keys=True, allow_nan=False).encode() ).hexdigest() @@ -138,17 +104,12 @@ def __init__( sources: Sequence, config: CategoricalPolicyConfig, *, - action_semantics: Literal["policy", "executed_predictions"] = "policy", episode_cache_size: int = 8, ): - if action_semantics not in {"policy", "executed_predictions"}: - raise ValueError("action_semantics must be policy or executed_predictions") if episode_cache_size < 1: raise ValueError("episode_cache_size must be positive") self.config = config - self.action_semantics = action_semantics self.episode_cache_size = episode_cache_size - self.excluded_episode_ids = [] self.exclusions = {} self.all_episode_ids = [] self.episode_ids = [] @@ -160,25 +121,20 @@ def __init__( self._labels = OrderedDict() eligible = [] seen = set() - tables = [ - str(source) + expanded = [ + episode for source in sources - if isinstance(source, (str, Path)) - and Path(source).suffix.lower() in {".csv", ".parquet"} + for episode in ( + source_episodes(source, config) + if isinstance(source, SequenceSourceConfig) + else [source] + ) ] - expanded = [] - for source in sources: - if isinstance(source, SequenceSourceConfig): - expanded.extend(source_episodes(source, config)) - elif not (isinstance(source, (str, Path)) and str(source) in tables): - expanded.append(source) - if tables: - expanded.extend(source_episodes(SequenceSourceConfig(paths=tables), config)) categorical = ( set(config.categorical_columns) | config.categorical_n_classes.keys() ) for source in expanded: - episode = _load(source, config, action_semantics) + episode = _load(source) if episode.episode_id in seen: raise ValueError(f"duplicate episode ID {episode.episode_id!r}") seen.add(episode.episode_id) @@ -207,7 +163,6 @@ def __init__( else len(episode.actions) ) if length is None: - self.excluded_episode_ids.append(episode.episode_id) self.exclusions[episode.episode_id] = ( "missing_rewards" if episode.transitions is None @@ -231,9 +186,7 @@ def __init__( self.cumulative_lengths.append( length + (self.cumulative_lengths[-1] if self.cumulative_lengths else 0) ) - self.episodes = EpisodeCollection( - eligible, episode_cache_size, config, action_semantics - ) + self.episodes = EpisodeCollection(eligible, episode_cache_size) def __len__(self): return self.cumulative_lengths[-1] if self.cumulative_lengths else 0 @@ -279,7 +232,6 @@ def fit_categorical_config(self) -> CategoricalPolicyConfig: def use_config(self, config: CategoricalPolicyConfig) -> None: self.config = config - self.episodes.config = config # Validate held-out values against the fixed training codec before updates. for episode, length in zip(self.episodes, self.lengths): for column, values in episode.observations.items(): @@ -372,28 +324,6 @@ def normalization_statistics(self): for column, (_, mean, std) in moments.items() } - def normalization_rows(self): - observations, static = [], [] - for episode, length in zip(self.episodes, self.lengths): - observations += [ - [ - encode_feature( - episode.observations[column][position], column, self.config - ) - for column in self.config.observation_columns - ] - for position in range(length) - ] - static.append( - [ - encode_feature(episode.static_inputs[column], column, self.config) - for column in self.config.static_columns - ] - ) - return torch.tensor(observations, dtype=torch.float32), torch.tensor( - static, dtype=torch.float32 - ) - class EpisodeWindowSampler(Sampler): """Shuffle episodes and their decisions while keeping disk access local.""" @@ -456,11 +386,7 @@ def root(index): PolicySequenceDataset( [source for group in partition for source in group], dataset.config, - action_semantics=dataset.action_semantics, episode_cache_size=dataset.episode_cache_size, ) for partition in (partitions[count:], partitions[:count]) ) - - -TrajectoryDataset = PolicySequenceDataset diff --git a/src/sequifier/io/sequence.py b/src/sequifier/io/sequence.py index 4f3ccd2f..2b9a6f74 100644 --- a/src/sequifier/io/sequence.py +++ b/src/sequifier/io/sequence.py @@ -45,6 +45,14 @@ class TransitionColumns(BaseModel): successors: tuple[dict[str, DomainValue] | None, ...] | None = None bootstrap_allowed: tuple[StrictBool, ...] | None = None + def at(self, index: int) -> dict: + """Read a transition without inventing absent capabilities.""" + return { + name: values[index] if values is not None else None + for name in type(self).model_fields + for values in [getattr(self, name)] + } + class SequenceEpisode(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) @@ -92,21 +100,13 @@ def validate_sequence(self): if values is not None and len(values) != length: raise ValueError(f"transition {field} must match sequence length") for index in range(length): - terminated = ( - transitions.terminated is not None and transitions.terminated[index] - ) - truncated = ( - transitions.truncated is not None and transitions.truncated[index] - ) + step = transitions.at(index) + terminated, truncated = step["terminated"], step["truncated"] if (terminated or truncated) and index != length - 1: raise ValueError( "a sequence cannot contain a decision after an episode boundary" ) - successor = ( - transitions.successors[index] - if transitions.successors is not None - else None - ) + successor = step["successors"] if successor is not None: if successor.keys() != self.observations.keys(): raise ValueError( @@ -114,11 +114,7 @@ def validate_sequence(self): ) if index + 1 < length and successor != self.observation(index + 1): raise ValueError("explicit successor breaks sequence continuity") - bootstrap = ( - transitions.bootstrap_allowed[index] - if transitions.bootstrap_allowed is not None - else False - ) + bootstrap = step["bootstrap_allowed"] if terminated and bootstrap: raise ValueError("task termination must disable bootstrap") if bootstrap and index + 1 == length and successor is None: diff --git a/src/sequifier/io/tensors.py b/src/sequifier/io/tensors.py new file mode 100644 index 00000000..640e2dac --- /dev/null +++ b/src/sequifier/io/tensors.py @@ -0,0 +1,65 @@ +"""Device transfer and collation for records of named or optional tensors.""" + +from dataclasses import dataclass, fields +from typing import TypeVar, cast + +import torch + +Record = TypeVar("Record", bound="TensorRecord") + + +def _map_tensors(value, operation): + if value is None: + return None + if isinstance(value, dict): + return {key: _map_tensors(item, operation) for key, item in value.items()} + if isinstance(value, TensorRecord): + return type(value)( + **{ + field.name: _map_tensors(getattr(value, field.name), operation) + for field in fields(value) + } + ) + return operation(value) + + +@dataclass(frozen=True) +class TensorRecord: + """Mixin for dataclasses containing tensors, dictionaries and optional fields.""" + + def to(self: Record, device: str | torch.device) -> Record: + return cast( + Record, + _map_tensors(self, lambda value: value.to(device, non_blocking=True)), + ) + + def pin_memory(self: Record) -> Record: + return cast(Record, _map_tensors(self, lambda value: value.pin_memory())) + + +def collate_tensor_records(values): + """Concatenate corresponding tensors, rejecting inconsistent record schemas.""" + if not values: + raise ValueError("cannot collate empty tensor records") + first = values[0] + if any(type(value) is not type(first) for value in values): + raise ValueError("tensor records must share types and optional fields") + if first is None: + return None + if isinstance(first, dict): + if any(value.keys() != first.keys() for value in values): + raise ValueError("tensor records must share feature and metadata schemas") + return { + key: collate_tensor_records([value[key] for value in values]) + for key in first + } + if isinstance(first, TensorRecord): + return type(first)( + **{ + field.name: collate_tensor_records( + [getattr(value, field.name) for value in values] + ) + for field in fields(first) + } + ) + return torch.cat(values) diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 5bfdefc2..22011c5a 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -95,28 +95,78 @@ model_name: policy device: cpu seed: 1010 -data_sources: +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 4 + attention: + n_heads: 4 + feed_forward: + dim: 128 + activation: swiglu + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: &id001 + - OBSERVATION_COLUMN + target_columns: + - action + policy: + observation_columns: *id001 + action_column: action + action_values: + - 0.0 + - 1.0 + return_conditioned: true + returns: + gamma: 1.0 + scale: 1.0 + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: + OBSERVATION_COLUMN: 1 + __policy_previous_action: 6 + __policy_return: 1 + decoder: + type: linear + prediction_length: 1 + support: 1 +dataset_training: demonstrations: - paths: [PLEASE_FILL.csv] -policies: - actor: - observation_columns: [EXAMPLE_OBSERVATION] - action_column: action - action_values: [0.0, 1.0] - context_length: 4 - return_conditioned: true - returns: {gamma: 1.0, scale: 1.0} -training_plan: - algorithm: return_conditioned - epochs: 80 - sources: [{source: demonstrations}] - policy: actor -optimization: + model_interface: actor + parts: + recorded: + paths: + - PLEASE_FILL.csv +global_training: + training_objective: policy + context_length: 4 batch_size: 64 + num_workers: 0 + episode_cache_size: 8 + resume: + policy: never learning_rate: 0.005 + gradient_clip: 1.0 validation_fraction: 0.2 - episode_cache_size: 8 - resume: {policy: never} + optimizer: + name: Adam + scheduler: + name: ConstantLR + factor: 1.0 + total_iters: 1 +training_plan: + phases: + - name: offline_policy + algorithm: return_conditioned + epochs: 80 + sources: + - source: demonstrations.recorded + policy: actor """ policy_infer_config_string = """project_root: . diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py deleted file mode 100644 index 6933dcde..00000000 --- a/src/sequifier/rl/history.py +++ /dev/null @@ -1,77 +0,0 @@ -"""One causal policy window builder shared by offline training and execution.""" - -from collections.abc import Mapping, Sequence -from dataclasses import dataclass - -import torch -from torch import Tensor - -from sequifier.config.policy_config import CategoricalPolicyConfig -from sequifier.io.batch import SequifierInputs, collate_model_inputs -from sequifier.io.policy_window import PolicyWindowView, action_index # noqa: F401 - - -@dataclass(frozen=True) -class PolicyInputs: - observations: Tensor - previous_actions: Tensor - attention_valid_mask: Tensor - static_inputs: Tensor - desired_returns: Tensor | None - - def to(self, device: str | torch.device) -> "PolicyInputs": - return PolicyInputs( - self.observations.to(device), - self.previous_actions.to(device), - self.attention_valid_mask.to(device), - self.static_inputs.to(device), - self.desired_returns.to(device) - if self.desired_returns is not None - else None, - ) - - -def collate_policy_inputs(inputs: Sequence[PolicyInputs | SequifierInputs]): - if not inputs: - raise ValueError("cannot collate empty policy inputs") - named = [value for value in inputs if isinstance(value, SequifierInputs)] - if named: - if len(named) != len(inputs): - raise ValueError("cannot mix named and legacy dense policy inputs") - return collate_model_inputs(named) - legacy = [value for value in inputs if isinstance(value, PolicyInputs)] - conditioned = legacy[0].desired_returns is not None - if any((value.desired_returns is not None) != conditioned for value in legacy): - raise ValueError("policy batches cannot mix conditioning modes") - return PolicyInputs( - observations=torch.cat([value.observations for value in legacy]), - previous_actions=torch.cat([value.previous_actions for value in legacy]), - attention_valid_mask=torch.cat( - [value.attention_valid_mask for value in legacy] - ), - static_inputs=torch.cat([value.static_inputs for value in legacy]), - desired_returns=( - torch.cat( - [ - value.desired_returns - for value in legacy - if value.desired_returns is not None - ] - ) - if conditioned - else None - ), - ) - - -def build_policy_inputs( - config: CategoricalPolicyConfig, - observations: Sequence[Mapping], - previous_actions: Sequence[int], - desired_returns: Sequence[float] | None, - static_inputs: Mapping, -) -> SequifierInputs: - """Compatibility entry point for the shared policy window view.""" - return PolicyWindowView(config).inputs( - observations, previous_actions, desired_returns, static_inputs - ) diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index fec31d64..6c990c01 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -9,15 +9,12 @@ from sequifier.config.policy_infer_config import PolicyInferenceConfig from sequifier.helpers import configure_determinism, normalize_path, write_data +from sequifier.io.batch import collate_model_inputs +from sequifier.io.policy_window import PolicyWindowView, action_index from sequifier.io.sequence import ordered_sequence_rows from sequifier.io.values import domain_fields, domain_scalar from sequifier.rl.checkpoint import load_policy from sequifier.rl.distributions import CategoricalActionDistribution -from sequifier.rl.history import ( - action_index, - build_policy_inputs, - collate_policy_inputs, -) from sequifier.rl.runtime import collect_policy_rollout from sequifier.rollout.factories import load_adapter from sequifier.rollout.history import numeric_scalar @@ -123,7 +120,7 @@ def flush(): if not pending_inputs: return with torch.no_grad(): - inputs = collate_policy_inputs(pending_inputs).to(policy.device) + inputs = collate_model_inputs(pending_inputs).to(policy.device) decision = CategoricalActionDistribution(policy(inputs)[:, -1, :]) actions = ( ( @@ -216,8 +213,7 @@ def flush(): previous_actions.append(previous_action) budgets.append(budget(row)) pending_inputs.append( - build_policy_inputs( - spec, + PolicyWindowView(spec).inputs( list(observations), list(previous_actions), list(budgets) if spec.return_conditioned else None, diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 2468a904..d21c09c7 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -1,177 +1,106 @@ """Algorithm boundary used by trajectory-backed train runs and Python callers.""" import json -from dataclasses import dataclass, field from pathlib import Path -from typing import Protocol from sequifier.artifacts.run_checkpoint import checkpoint_path -from sequifier.config.policy_config import PolicyTrainingConfig -from sequifier.config.policy_train_config import ( - ResolvedPolicyTrainConfig, - TrajectorySourceConfig, -) -from sequifier.io.policy_dataset import TrajectoryDataset, split_by_episode +from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig +from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.rl.checkpoint import save_policy from sequifier.rl.training import PolicyTrainingResult, train_policy -from sequifier.trajectory.importers import source_episodes, source_paths +from sequifier.trajectory.importers import source_paths -class AlgorithmRuntime(Protocol): - def prepare(self) -> None: ... - def update(self) -> PolicyTrainingResult: ... - def evaluate(self) -> dict: ... - def checkpoint(self) -> None: ... - - -def trajectory_paths(source: TrajectorySourceConfig) -> list[Path]: - """Compatibility alias for source path/manifest resolution.""" - return source_paths(source) - - -@dataclass -class ReturnConditionedRuntime: - config: ResolvedPolicyTrainConfig - training: TrajectoryDataset = field(init=False) - validation: TrajectoryDataset = field(init=False) - excluded_episode_ids: list[str] = field(init=False) - exclusions: dict[str, str] = field(init=False) - sources: dict[str, list[str]] = field(init=False) - result: PolicyTrainingResult = field(init=False) - - def prepare(self) -> None: - phase = self.config.training_plan.phases[0] - spec = self.config.policies[phase.policy] - all_paths: dict[str, list[Path]] = {} - for name in phase.trajectory_sources + (phase.validation_sources or []): - all_paths[name] = trajectory_paths(self.config.trajectory_sources[name]) - flattened = [path for paths in all_paths.values() for path in paths] - protected_paths = set(flattened) - for name in all_paths: - manifest = self.config.trajectory_sources[name].manifest - if manifest is not None: - protected_paths.add(Path(manifest).resolve()) - source = self.config.trajectory_sources[name] - for path in (source.metadata_config_path, source.static_data_path): - if path is not None: - protected_paths.add(Path(path).resolve()) - output_paths = [ - Path(self.config.actor_path), - Path(self.config.report_path), - checkpoint_path(self.config), - ] - if len(output_paths) != len(set(output_paths)): - raise ValueError("model, report, and run checkpoint paths must differ") - if set(output_paths) & protected_paths: - raise ValueError("training outputs must not overwrite trajectory inputs") - if len(flattened) != len(set(flattened)): - raise ValueError( - "trajectory files cannot appear in multiple selected sources" - ) - - def dataset(names: list[str]) -> TrajectoryDataset: - return TrajectoryDataset( - [ - episode - for name in names - for episode in source_episodes( - self.config.trajectory_sources[name], spec - ) - ], - spec, - episode_cache_size=self.config.optimization.episode_cache_size, - ) - - corpus = dataset(phase.trajectory_sources) - self.excluded_episode_ids = list(corpus.excluded_episode_ids) - self.exclusions = dict(corpus.exclusions) - if phase.validation_sources is None: - self.training, self.validation = split_by_episode( - corpus, - self.config.optimization.validation_fraction, - self.config.seed, - group_identical=True, - ) - else: - self.training = corpus - self.validation = dataset(phase.validation_sources) - self.excluded_episode_ids += self.validation.excluded_episode_ids - self.exclusions.update(self.validation.exclusions) - if set(corpus.all_episode_ids) & set(self.validation.all_episode_ids): - raise ValueError( - "duplicate episode IDs across training/validation sources" - ) - if not len(self.training) or not len(self.validation): - raise ValueError( - f"policy training needs nonempty eligible partitions; exclusions: {corpus.exclusions}" - ) - training_groups = set(self.training.group_ids) - {None} - validation_groups = set(self.validation.group_ids) - {None} - if training_groups & validation_groups: - raise ValueError( - "collection groups cannot cross training/validation partitions" - ) - if set(self.training.demonstration_keys) & set( - self.validation.demonstration_keys - ): - raise ValueError( - "identical trajectories cannot cross training/validation partitions" - ) - self.sources = { - name: [str(path) for path in paths] for name, paths in all_paths.items() - } - - def update(self) -> PolicyTrainingResult: - phase = self.config.training_plan.phases[0] - optimization = PolicyTrainingConfig( - **self.config.optimization.model_dump(), - epochs=phase.epochs, - seed=self.config.seed, +def run_policy_training(config: ResolvedPolicyTrainConfig) -> PolicyTrainingResult: + """Prepare episode partitions, train, and publish the selected policy and report.""" + phase = config.training_plan.phases[0] + spec = config.policies[phase.policy] + all_paths: dict[str, list[Path]] = {} + for name in phase.trajectory_sources + (phase.validation_sources or []): + all_paths[name] = source_paths(config.trajectory_sources[name]) + flattened = [path for paths in all_paths.values() for path in paths] + protected_paths = set(flattened) + protected_paths.update( + Path(path).resolve() + for name in all_paths + for source in [config.trajectory_sources[name]] + for path in ( + source.manifest, + source.metadata_config_path, + source.static_data_path, ) - self.result = train_policy( - self.training, self.validation, optimization, run_config=self.config + if path is not None + ) + output_paths = [ + Path(config.model_path), + Path(config.report_path), + checkpoint_path(config), + ] + if len(output_paths) != len(set(output_paths)): + raise ValueError("model, report, and run checkpoint paths must differ") + if set(output_paths) & protected_paths: + raise ValueError("training outputs must not overwrite trajectory inputs") + if len(flattened) != len(set(flattened)): + raise ValueError("trajectory files cannot appear in multiple selected sources") + + def dataset(names: list[str]) -> PolicySequenceDataset: + return PolicySequenceDataset( + [config.trajectory_sources[name] for name in names], + spec, + episode_cache_size=config.optimization.episode_cache_size, ) - return self.result - def evaluate(self) -> dict: - result = self.result - return { - "format": "sequifier.policy-training-report", - "version": 1, - "algorithm": "return_conditioned", - "config": self.config.model_dump(mode="json"), - "sources": self.sources, - "policy": result.policy.config.model_dump(mode="json"), - "training_episode_ids": list(result.training_episode_ids), - "validation_episode_ids": list(result.validation_episode_ids), - "excluded_episode_ids": self.excluded_episode_ids, - "exclusions": self.exclusions, - "training_decisions": len(self.training), - "validation_decisions": len(self.validation), - "best_epoch": result.best_epoch, - "optimizer_steps": result.optimizer_steps, - "epochs": list(result.metrics), - "monitor": { - "metric": "action_nll", - "direction": "min", - "value": result.metrics[result.best_epoch]["validation_nll"], - }, - "artifact_kind": "sequifier_model", - "actor_path": self.config.actor_path, - } - - def checkpoint(self) -> None: - save_policy( - self.result.policy, self.config.actor_path, metadata=self.evaluate() + corpus = dataset(phase.trajectory_sources) + if phase.validation_sources is None: + training, validation = split_by_episode( + corpus, + config.optimization.validation_fraction, + config.seed, + group_identical=True, ) - - -def run_policy_training(config: ResolvedPolicyTrainConfig) -> PolicyTrainingResult: - runtime: AlgorithmRuntime = ReturnConditionedRuntime(config) - runtime.prepare() - result = runtime.update() - report = runtime.evaluate() - runtime.checkpoint() + else: + training = corpus + validation = dataset(phase.validation_sources) + if set(corpus.all_episode_ids) & set(validation.all_episode_ids): + raise ValueError("duplicate episode IDs across training/validation sources") + if not len(training) or not len(validation): + raise ValueError( + f"policy training needs nonempty eligible partitions; exclusions: {corpus.exclusions}" + ) + for attribute in ("group_ids", "demonstration_keys"): + if (set(getattr(training, attribute)) & set(getattr(validation, attribute))) - { + None + }: + raise ValueError(f"{attribute} cannot cross training/validation partitions") + exclusions = {**corpus.exclusions, **validation.exclusions} + sources = {name: [str(path) for path in paths] for name, paths in all_paths.items()} + + result = train_policy(training, validation, config) + report = { + "format": "sequifier.policy-training-report", + "version": 1, + "algorithm": "return_conditioned", + "config": config.model_dump(mode="json"), + "sources": sources, + "policy": result.policy.config.model_dump(mode="json"), + "training_episode_ids": list(result.training_episode_ids), + "validation_episode_ids": list(result.validation_episode_ids), + "excluded_episode_ids": list(exclusions), + "exclusions": exclusions, + "training_decisions": len(training), + "validation_decisions": len(validation), + "best_epoch": result.best_epoch, + "optimizer_steps": result.optimizer_steps, + "epochs": list(result.metrics), + "monitor": { + "metric": "action_nll", + "direction": "min", + "value": result.metrics[result.best_epoch]["validation_nll"], + }, + "artifact_kind": "sequifier_model", + "model_path": config.model_path, + } + save_policy(result.policy, config.model_path, metadata=report) destination = Path(config.report_path) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text( diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 083a5e02..74d96c3e 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -19,7 +19,6 @@ from sequifier.model.factory import build_transformer_network from sequifier.model.ingestions import get_feature_embedding_dims from sequifier.rl.distributions import CategoricalActionDistribution -from sequifier.rl.history import PolicyInputs def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu"): @@ -179,81 +178,10 @@ def set_normalization_statistics( fitted[column] = {"mean": mean, "std": std} self.interface.selected_columns_statistics = fitted - def set_normalization(self, observations: Tensor, static_inputs: Tensor) -> None: - """Convenience API for in-memory callers; file training fits incrementally.""" - statistics = {} - for values, columns in ( - (observations, self.config.observation_columns), - (static_inputs, self.config.static_columns), - ): - if ( - values.ndim != 2 - or values.shape[1] != len(columns) - or not len(values) - or not torch.isfinite(values).all() - ): - raise ValueError( - "normalization requires finite rows matching policy columns" - ) - for index, column in enumerate(columns): - if column not in self.config.categorical_n_classes: - import polars as pl - - from sequifier.io.preprocessing import _finite_mean_and_std - - fitted = ( - values[:, index].detach().to(device="cpu", dtype=torch.float64) - ) - mean, std = _finite_mean_and_std( - pl.Series(column, fitted.numpy()), column - ) - statistics[column] = {"mean": mean, "std": std} - self.set_normalization_statistics(statistics) - - def named_inputs(self, inputs: PolicyInputs | SequifierInputs) -> SequifierInputs: - if isinstance(inputs, SequifierInputs): - return inputs - # Older Python callers can still supply the dense convenience object. - features = { - column: inputs.observations[:, :, index] - for index, column in enumerate(self.config.observation_columns) - } - features.update( - { - column: inputs.static_inputs[:, index, None].expand_as( - inputs.previous_actions - ) - for index, column in enumerate(self.config.static_columns) - } - ) - for column in self.config.categorical_n_classes: - values = features[column] - if (values != values.round()).any(): - raise ValueError( - f"{column}: categorical observations must be integer IDs" - ) - features[column] = values.long() - ids = action_global_ids(self.config) - actions = inputs.previous_actions - if ( - actions.dtype != torch.int64 - or (actions < 0).any() - or (actions > len(ids)).any() - ): - raise ValueError("invalid previous action IDs") - lookup = torch.tensor(ids + [ids[-1] + 1], device=actions.device) - features[PREVIOUS_ACTION] = lookup[actions] - if inputs.desired_returns is not None: - features[DESIRED_RETURN] = inputs.desired_returns - return SequifierInputs( - features, {"attention_valid_mask": inputs.attention_valid_mask} - ) - def model_inputs( - self, inputs: PolicyInputs | SequifierInputs + self, inputs: SequifierInputs ) -> tuple[dict[str, Tensor], dict[str, Tensor]]: """Encode raw named histories without mutating stored data or masks.""" - inputs = self.named_inputs(inputs) self.validate_inputs(inputs) features = dict(inputs.inputs) mask = inputs.attention_valid_mask @@ -267,7 +195,7 @@ def model_inputs( ) return features, inputs.metadata - def forward(self, inputs: PolicyInputs | SequifierInputs) -> Tensor: + def forward(self, inputs: SequifierInputs) -> Tensor: features, metadata = self.model_inputs(inputs) logits = self.network( features, metadata, interface_name=self.interface_name @@ -276,13 +204,10 @@ def forward(self, inputs: PolicyInputs | SequifierInputs) -> Tensor: ~inputs.attention_valid_mask[:, -logits.shape[1] :, None], 0.0 ) - def distribution( - self, inputs: PolicyInputs | SequifierInputs - ) -> CategoricalActionDistribution: + def distribution(self, inputs: SequifierInputs) -> CategoricalActionDistribution: return CategoricalActionDistribution(self(inputs)) - def validate_inputs(self, inputs: PolicyInputs | SequifierInputs) -> None: - inputs = self.named_inputs(inputs) + def validate_inputs(self, inputs: SequifierInputs) -> None: expected = set(self.interface.input_columns) if set(inputs.inputs) != expected: raise ValueError( diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index cb635d2e..8e62d428 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -5,9 +5,9 @@ import torch from sequifier.config.rollout_config import RolloutConfig +from sequifier.io.policy_window import PolicyWindowView from sequifier.io.values import DomainValue, domain_fields from sequifier.rl.distributions import CategoricalActionDistribution -from sequifier.rl.history import build_policy_inputs from sequifier.rl.policy import CategoricalPolicy from sequifier.rollout.contracts import EnvironmentAdapter, ResetRequest from sequifier.rollout.coordinator import collect_rollout @@ -74,8 +74,7 @@ def __call__(self, history: SeedHistory) -> dict[str, DomainValue]: observation = { column: values[-1] for column, values in history.observations.items() } - inputs = build_policy_inputs( - spec, + inputs = PolicyWindowView(spec).inputs( list(self.observations) + [observation], list(self.previous_actions) + [self.previous_action], list(self.budgets) + [self.budget] if spec.return_conditioned else None, @@ -158,8 +157,7 @@ def validate_history(history): } domain_fields(observation, set(spec.observation_columns), "policy observation") # Validate the final successor too, even when no subsequent action is needed. - inputs = build_policy_inputs( - spec, + inputs = PolicyWindowView(spec).inputs( [observation], [len(spec.action_values)], [numeric_scalar(desired_return, "desired_return")] diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 1b2fe874..308633b2 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -19,15 +19,11 @@ RunCheckpointStore, select_run_checkpoint, ) -from sequifier.config.policy_config import PolicyTrainingConfig from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig from sequifier.integration.callbacks import IntegrationManager from sequifier.integration.contexts import StepIdentity -from sequifier.io.policy_dataset import ( - EpisodeWindowSampler, - TrajectoryDataset, - collate_policy_examples, -) +from sequifier.io.batch import collate_sequence_batches +from sequifier.io.policy_dataset import EpisodeWindowSampler, PolicySequenceDataset from sequifier.model.factory import InterfaceRuntimeMetadata from sequifier.objectives import PolicyObjective from sequifier.rl.policy import CategoricalPolicy @@ -48,33 +44,23 @@ class PolicyTrainingResult: def train_policy( - training: TrajectoryDataset, - validation: TrajectoryDataset, - config: PolicyTrainingConfig, + training: PolicySequenceDataset, + validation: PolicySequenceDataset, + run_config: ResolvedPolicyTrainConfig, *, - run_config: ResolvedPolicyTrainConfig | None = None, integrations: IntegrationManager | None = None, ) -> PolicyTrainingResult: if not len(training) or not len(validation): raise ValueError("policy training and validation partitions must be nonempty") - if ( - training.config != validation.config - or training.action_semantics != validation.action_semantics - ): + if training.config != validation.config: raise ValueError( "training and validation must share the policy/return definition" ) - if run_config is not None: - phase = run_config.training_plan.phases[0] - expected = PolicyTrainingConfig( - **run_config.optimization.model_dump(), - epochs=phase.epochs, - seed=run_config.seed, - ) - if config != expected or training.config != run_config.policies[phase.policy]: - raise ValueError( - "policy training arguments must match the resolved run configuration" - ) + phase = run_config.training_plan.phases[0] + config = run_config.optimization + epochs, seed = phase.epochs, run_config.seed + if training.config != run_config.policies[phase.policy]: + raise ValueError("dataset policy must match the resolved run configuration") train_ids = tuple(training.episode_ids) validation_ids = tuple(validation.episode_ids) if set(train_ids) & set(validation_ids): @@ -84,12 +70,10 @@ def train_policy( fitted = training.fit_categorical_config() training.use_config(fitted) validation.use_config(fitted) - if run_config is not None: - policies = dict(run_config.policies) - policies[run_config.training_plan.phases[0].policy] = fitted - run_config = run_config.model_copy(update={"policies": policies}) + policies = {**run_config.policies, phase.policy: fitted} + run_config = run_config.model_copy(update={"policies": policies}) # Seed model initialization and data order independently. - device = run_config.device if run_config is not None else "cpu" + device = run_config.device cuda_index = torch.device(device).index cuda_devices = ( [cuda_index if cuda_index is not None else torch.cuda.current_device()] @@ -97,17 +81,11 @@ def train_policy( else [] ) with torch.random.fork_rng(devices=cuda_devices): - torch.manual_seed(config.seed) + torch.manual_seed(seed) policy = CategoricalPolicy(training.config, device=device) policy.set_normalization_statistics(training.normalization_statistics()) - resume_path = ( - select_run_checkpoint(run_config) if run_config is not None else None - ) - if ( - run_config is not None - and run_config.initialize_from is not None - and resume_path is None - ): + resume_path = select_run_checkpoint(run_config) + if run_config.initialize_from is not None and resume_path is None: source, source_config, _ = load_execution_model( run_config.initialize_from, device=device ) @@ -152,22 +130,22 @@ def likelihood(batch): ) optimization = OptimizationRuntime.create( - config, device, policy.network.parameters(), phase_epochs=config.epochs + config, device, policy.network.parameters(), phase_epochs=epochs ) integrations = integrations or IntegrationManager() - generator = torch.Generator().manual_seed(config.seed) + generator = torch.Generator().manual_seed(seed) train_loader = DataLoader( training, batch_size=config.batch_size, sampler=EpisodeWindowSampler(training, generator), num_workers=config.num_workers, generator=generator, - collate_fn=collate_policy_examples, + collate_fn=collate_sequence_batches, ) validation_loader = DataLoader( validation, batch_size=config.batch_size, - collate_fn=collate_policy_examples, + collate_fn=collate_sequence_batches, num_workers=config.num_workers, ) metrics = [] @@ -177,13 +155,8 @@ def likelihood(batch): global_batch = 0 first_epoch = 0 random_state = RandomStateManager(torch.device(device)) - store = ( - RunCheckpointStore(run_config, run_config.model_name) - if run_config is not None - else None - ) + store = RunCheckpointStore(run_config, run_config.model_name) if resume_path is not None: - assert store is not None and run_config is not None checkpoint = store.load(resume_path) previous = checkpoint.training_config if not isinstance(previous, ResolvedPolicyTrainConfig): @@ -226,110 +199,97 @@ def likelihood(batch): integrations.load_state_dict(checkpoint.integration_state.values) del checkpoint, state optimization.optimizer.zero_grad(set_to_none=True) - for epoch in range(first_epoch, config.epochs): - policy.train() - total_loss = 0.0 - decisions = 0 - for batch_index, batch in enumerate(train_loader): - likelihood_result = likelihood(batch) - loss = likelihood_result.backward_loss - if not torch.isfinite(loss): - raise ValueError("policy action likelihood became non-finite") - count = int(likelihood_result.accounting_count) - total_loss += sum( - float(value) for value in likelihood_result.accounting_sums.values() - ) - decisions += count - identity = StepIdentity( - epoch, - batch_index, - global_batch, - optimization.optimizer_step, - 0, - 1, - 0, - 1, - ) - optimization.accumulate(loss, identity, integrations, policy.network) - result = optimization.complete_step( - policy.network, identity, integrations, UpdatePolicy() - ) - if result.stop_requested or not result.applied: - raise RuntimeError( - "local policy optimizer did not apply its update" + + def run_epoch(loader, epoch, *, updating): + nonlocal global_batch + policy.train(updating) + total, decisions = 0.0, 0 + with torch.set_grad_enabled(updating): + for batch_index, batch in enumerate(loader): + result = likelihood(batch) + if not torch.isfinite(result.backward_loss): + raise ValueError("policy action likelihood became non-finite") + total += sum( + float(value) for value in result.accounting_sums.values() ) - global_batch += 1 - policy.eval() - validation_loss = 0.0 - validation_count = 0 - with torch.no_grad(): - for batch in validation_loader: - likelihood_result = likelihood(batch) - if not torch.isfinite(likelihood_result.backward_loss): - raise ValueError( - "validation action likelihood became non-finite" + decisions += int(result.accounting_count) + if updating: + identity = StepIdentity( + epoch, + batch_index, + global_batch, + optimization.optimizer_step, + 0, + 1, + 0, + 1, ) - validation_loss += sum( - float(value) - for value in likelihood_result.accounting_sums.values() - ) - validation_count += int(likelihood_result.accounting_count) - validation_loss /= validation_count + optimization.accumulate( + result.backward_loss, identity, integrations, policy.network + ) + step = optimization.complete_step( + policy.network, identity, integrations, UpdatePolicy() + ) + if step.stop_requested or not step.applied: + raise RuntimeError( + "local policy optimizer did not apply its update" + ) + global_batch += 1 + return total / decisions + + def weights(): + return { + name: value.detach().cpu().clone() + for name, value in policy.network.state_dict().items() + } + + for epoch in range(first_epoch, epochs): + training_loss = run_epoch(train_loader, epoch, updating=True) + validation_loss = run_epoch(validation_loader, epoch, updating=False) metrics.append( { "epoch": float(epoch), - "training_nll": total_loss / decisions, + "training_nll": training_loss, "validation_nll": validation_loss, } ) logger.info( - f"Policy epoch {epoch + 1}/{config.epochs}: " - f"action_nll={total_loss / decisions:.6f}, " + f"Policy epoch {epoch + 1}/{epochs}: " + f"action_nll={training_loss:.6f}, " f"validation_action_nll={validation_loss:.6f}" ) if validation_loss < best_loss: best_loss = validation_loss best_epoch = epoch - best_weights = { - name: value.detach().cpu().clone() - for name, value in policy.network.state_dict().items() - } + best_weights = weights() optimization.step_scheduler(validation_loss) - if store is not None and run_config is not None: - checkpoint = RunCheckpoint( - format_version=RUN_CHECKPOINT_FORMAT_VERSION, - model=build_model_artifact( - policy.network, - policy.execution_config, - state_dict={ - name: value.detach().cpu().clone() - for name, value in policy.network.state_dict().items() - }, - ), - optimization=optimization.state_dict(), - run_state={ - "algorithm": "return_conditioned", - "next_epoch": epoch + 1, - "metrics": metrics, - "best_loss": best_loss, - "best_epoch": best_epoch, - "best_weights": best_weights, - "global_batch": global_batch, - "data_generator": generator.get_state(), - }, - random_state=DistributedRandomState( - states=(random_state.capture_local().state_dict(),) - ), - loader_state=LoaderState( - parts=_loader_contract(training, validation) - ), - integration_state=IntegrationState( - values=integrations.state_dict() - ), - training_config=run_config, - ) - store.save(checkpoint, "latest") - del checkpoint + checkpoint = RunCheckpoint( + format_version=RUN_CHECKPOINT_FORMAT_VERSION, + model=build_model_artifact( + policy.network, + policy.execution_config, + state_dict=weights(), + ), + optimization=optimization.state_dict(), + run_state={ + "algorithm": "return_conditioned", + "next_epoch": epoch + 1, + "metrics": metrics, + "best_loss": best_loss, + "best_epoch": best_epoch, + "best_weights": best_weights, + "global_batch": global_batch, + "data_generator": generator.get_state(), + }, + random_state=DistributedRandomState( + states=(random_state.capture_local().state_dict(),) + ), + loader_state=LoaderState(parts=_loader_contract(training, validation)), + integration_state=IntegrationState(values=integrations.state_dict()), + training_config=run_config, + ) + store.save(checkpoint, "latest") + del checkpoint if best_weights is None: raise ValueError( "policy training did not produce finite validation likelihood" @@ -347,7 +307,7 @@ def likelihood(batch): def _loader_contract( - training: TrajectoryDataset, validation: TrajectoryDataset + training: PolicySequenceDataset, validation: PolicySequenceDataset ) -> dict: return { "policy": { @@ -364,7 +324,7 @@ def _resume_contract(config: ResolvedPolicyTrainConfig) -> dict: values = config.model_dump(mode="json") for key in ( "device", - "actor_path", + "model_path", "report_path", "initialize_from", "project_root", diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index 0b3dd8c4..d1f29d6b 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -4,7 +4,6 @@ CategoricalPolicyConfig, PolicyOptimizationConfig, PolicySchemaConfig, - PolicyTrainingConfig, ReturnDefinition, ) from sequifier.config.policy_infer_config import PolicyInferenceConfig @@ -14,33 +13,22 @@ ResolvedPolicyTrainConfig, ReturnConditionedPhaseConfig, SequenceSourceConfig, - TrajectorySourceConfig, resolve_policy_train_config, ) from sequifier.config.sequence_import_config import SequenceImportConfig -from sequifier.io.batch import SequifierBatch, SequifierInputs -from sequifier.io.policy_dataset import PolicySequenceDataset -from sequifier.io.policy_window import PolicyWindowView +from sequifier.io.batch import SequifierBatch, SequifierInputs, collate_sequence_batches +from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode +from sequifier.io.policy_window import PolicyWindowView, action_index from sequifier.io.sequence import SequenceEpisode, TransitionColumns from sequifier.rl.checkpoint import load_policy, save_policy from sequifier.rl.distributions import ( CategoricalActionDistribution, masked_action_likelihood, ) -from sequifier.rl.history import PolicyInputs, action_index, build_policy_inputs -from sequifier.rl.learner import ( - AlgorithmRuntime, - ReturnConditionedRuntime, - run_policy_training, -) +from sequifier.rl.learner import run_policy_training from sequifier.rl.policy import CategoricalPolicy from sequifier.rl.runtime import PolicyExecutor, collect_policy_rollout from sequifier.rl.training import PolicyTrainingResult, train_policy -from sequifier.trajectory.dataset import ( - TrajectoryDataset, - collate_policy_examples, - split_by_episode, -) from sequifier.trajectory.importers import import_sequences, source_episodes from sequifier.trajectory.returns import remaining_return, return_to_go @@ -60,25 +48,17 @@ "PolicyInferenceConfig", "PolicyTrainConfig", "PolicyOptimizationConfig", - "TrajectorySourceConfig", "ReturnConditionedPhaseConfig", "ResolvedPolicyTrainConfig", "resolve_policy_train_config", - "AlgorithmRuntime", - "ReturnConditionedRuntime", "run_policy_training", "CategoricalPolicyConfig", - "PolicyTrainingConfig", "ReturnDefinition", "CategoricalActionDistribution", "CategoricalPolicy", - "PolicyInputs", "PolicyExecutor", "PolicyTrainingResult", - "TrajectoryDataset", "action_index", - "build_policy_inputs", - "collate_policy_examples", "collect_policy_rollout", "load_policy", "masked_action_likelihood", @@ -86,5 +66,6 @@ "return_to_go", "save_policy", "split_by_episode", + "collate_sequence_batches", "train_policy", ] diff --git a/src/sequifier/trajectory/dataset.py b/src/sequifier/trajectory/dataset.py deleted file mode 100644 index 1be229b7..00000000 --- a/src/sequifier/trajectory/dataset.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Compatibility exports for the shared policy sequence dataset.""" - -from sequifier.io.policy_dataset import EpisodeCollection as EpisodeCollection -from sequifier.io.policy_dataset import EpisodeWindowSampler as EpisodeWindowSampler -from sequifier.io.policy_dataset import PolicyExample as PolicyExample -from sequifier.io.policy_dataset import PolicySequenceDataset as PolicySequenceDataset -from sequifier.io.policy_dataset import TrajectoryDataset as TrajectoryDataset -from sequifier.io.policy_dataset import ( - collate_policy_examples as collate_policy_examples, -) -from sequifier.io.policy_dataset import episode_content_key as episode_content_key -from sequifier.io.policy_dataset import split_by_episode as split_by_episode diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 3b808fc0..37af5541 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -83,27 +83,27 @@ def sequence_from_trajectory( ) if episode.prediction_columns != [policy.action_column]: raise ValueError("recorded actions do not match the policy action column") + steps = episode.transitions return SequenceEpisode( episode_id=episode.episode_id, - item_positions=tuple(step.step_id for step in episode.transitions), + item_positions=tuple(step.step_id for step in steps), observations={ - column: tuple(step.observation[column] for step in episode.transitions) + column: tuple(step.observation[column] for step in steps) for column in episode.seed.observations }, - actions=tuple( - step.predictions[policy.action_column] for step in episode.transitions - ), + actions=tuple(step.predictions[policy.action_column] for step in steps), static_inputs=episode.seed.static_inputs, transitions=TransitionColumns( - rewards=tuple(step.reward for step in episode.transitions), - terminated=tuple(step.terminated for step in episode.transitions), - truncated=tuple(step.truncated for step in episode.transitions), - successors=tuple( - step.successor_observation for step in episode.transitions - ), - bootstrap_allowed=tuple( - step.bootstrap_allowed for step in episode.transitions - ), + **{ + field: tuple(getattr(step, attribute) for step in steps) + for field, attribute in { + "rewards": "reward", + "terminated": "terminated", + "truncated": "truncated", + "successors": "successor_observation", + "bootstrap_allowed": "bootstrap_allowed", + }.items() + } ), ) @@ -325,188 +325,162 @@ def decode(value, column): rows = list(data.iter_rows(named=True)) if not rows: raise ValueError("sequence source changed while loading") - required = {columns.item_position, columns.action or self.policy.action_column} - observation_columns = { - name: columns.observations.get(name, name) - for name in self.policy.observation_columns - } - static_columns = { - name: columns.static_inputs.get(name, name) - for name in self.policy.static_columns - } action_column = columns.action or self.policy.action_column - if action_column in observation_columns.values() or ( - self.source.static_data_path is None - and action_column in static_columns.values() - ): - raise ValueError( - "current action labels cannot also be policy inputs; map distinct source columns" + observations, static_inputs = ( + {name: mapping.get(name, name) for name in names} + for names, mapping in ( + (self.policy.observation_columns, columns.observations), + (self.policy.static_columns, columns.static_inputs), ) - if columns.previous_action == action_column: + ) + inputs = set(observations.values()) + if self.source.static_data_path is None: + inputs.update(static_inputs.values()) + if action_column in inputs or columns.previous_action == action_column: raise ValueError( - "previous and current actions require distinct source columns" + "current action labels require distinct input/previous-action columns" ) - if set(columns.observations) - set(observation_columns) or set( - columns.static_inputs - ) - set(static_columns): - raise ValueError("table mappings must name declared policy features") - required.update(observation_columns.values()) - if self.source.static_data_path is None: - required.update(static_columns.values()) - for name in ( - columns.previous_action, - columns.bootstrap_allowed, - columns.group, - columns.episode_id, + if ( + set(columns.observations) - observations.keys() + or set(columns.static_inputs) - static_inputs.keys() ): - if name is not None: - required.add(name) - required.update(columns.successors.values()) + raise ValueError("table mappings must name declared policy features") + required = ( + inputs + | {columns.item_position, action_column} + | set(columns.successors.values()) + ) + required.update( + filter( + None, + ( + columns.previous_action, + columns.bootstrap_allowed, + columns.group, + columns.episode_id, + ), + ) + ) if required - set(data.columns): raise ValueError( f"policy table is missing columns: {sorted(required - set(data.columns))}" ) - positions = tuple(row[columns.item_position] for row in rows) - if any( - isinstance(value, bool) or not isinstance(value, int) for value in positions - ): - raise ValueError("itemPosition must contain integers") - actions = tuple( - decode( - row[columns.action or self.policy.action_column], - columns.action or self.policy.action_column, - ) - for row in rows - ) + + def values(column, decoder=decode): + return tuple(decoder(row[column], column) for row in rows) + + def optional_values(column, decoder=decode): + if ( + column is None + or column not in data.columns + or data[column].null_count() == len(rows) + ): + return None + return values(column, decoder) + + def constant(entries, label): + unique = set(entries) + if len(unique) != 1: + raise ValueError(f"{label} must remain constant within an episode") + return next(iter(unique)) + + positions = values(columns.item_position, lambda value, _: value) + actions = values(action_column) if columns.previous_action is not None: - values = [ - None - if row[columns.previous_action] is None - else decode( - row[columns.previous_action], - columns.action or self.policy.action_column, - ) - for row in rows - ] - if values[1:] != list(actions[:-1]): + preceding_actions = values( + columns.previous_action, + lambda value, _: None + if value is None + else decode(value, action_column), + ) + if preceding_actions[1:] != actions[:-1]: raise ValueError( "previous-action column must lag executed actions by exactly one decision" ) - preceding = values[0] - starts = positions[0] == 0 and preceding is None + preceding = preceding_actions[0] if self.source.static_data_path is not None: - static_data = ( + static_rows = ( _scan((Path(self.source.static_data_path),)) .filter(pl.col(columns.sequence_id) == self.sequence_id) .collect() ) - if len(static_data) != 1: + if len(static_rows) != 1: raise ValueError("static data requires exactly one row per sequenceId") - static_rows = list(static_data.iter_rows(named=True)) + static_rows = list(static_rows.iter_rows(named=True)) else: static_rows = rows static = { - name: decode(static_rows[0][column], column) - for name, column in static_columns.items() + name: constant( + (decode(row[column], column) for row in static_rows), + f"static input {name}", + ) + for name, column in static_inputs.items() } - if any( - { - name: decode(row[column], column) - for name, column in static_columns.items() - } - != static - for row in static_rows - ): - raise ValueError("static inputs must remain constant within an episode") - - def flags(column): - if column is None or column not in data.columns: - return None - values = tuple(row[column] for row in rows) - if all(value is None for value in values): - return None - if any(not isinstance(value, bool) for value in values): - raise ValueError( - f"{column}: boundary/bootstrap columns must contain booleans" - ) - return values - - terminated, truncated = flags(columns.terminated), flags(columns.truncated) + # Strict boolean and length checks belong to TransitionColumns/SequenceEpisode. + transitions = { + field: optional_values(getattr(columns, field), lambda value, _: value) + for field in ("terminated", "truncated", "bootstrap_allowed") + } + transitions = TransitionColumns(**transitions).model_dump(exclude_none=True) declared = self.source.episode_end if declared != "unknown": - current = terminated if declared == "terminated" else truncated + current = transitions.get(declared) if current is not None and not current[-1]: raise ValueError( "episode_end conflicts with the recorded final boundary" ) - current = (False,) * (len(rows) - 1) + (True,) - if declared == "terminated": - terminated = current - else: - truncated = current + transitions[declared] = (False,) * (len(rows) - 1) + (True,) successors = None if columns.successors: - if set(columns.successors) != set(observation_columns): + if columns.successors.keys() != observations.keys(): raise ValueError( "successor mapping must cover every observation column" ) successors = [] for row in rows: - values = [row[column] for column in columns.successors.values()] - if all(value is None for value in values): - successors.append(None) - elif any(value is None for value in values): + present = [ + row[column] is not None for column in columns.successors.values() + ] + if any(present) and not all(present): raise ValueError( "a missing successor must mask all observation fields" ) - else: - successors.append( - { - name: decode(row[column], observation_columns[name]) - for name, column in columns.successors.items() - } - ) - group_values = [row[columns.group] for row in rows] if columns.group else [] - groups = ( - {domain_scalar(value, "collection group") for value in group_values} - if any(value is not None for value in group_values) - else set() - ) - if len(groups) > 1: - raise ValueError("collection group must be constant within a sequence") - identities = ( - {domain_scalar(row[columns.episode_id], "episode ID") for row in rows} + successors.append( + { + name: decode(row[column], observations[name]) + for name, column in columns.successors.items() + } + if all(present) + else None + ) + groups = optional_values(columns.group, domain_scalar) + identity = ( + constant(values(columns.episode_id, domain_scalar), "episode ID") if columns.episode_id - else {self.sequence_id} + else self.sequence_id ) - if len(identities) != 1: - raise ValueError("episode ID must remain constant within a sequence") - rewards = None - if columns.reward and columns.reward in data.columns: - values = [row[columns.reward] for row in rows] - if any(value is not None for value in values): - rewards = tuple( - numeric_scalar(decode(value, columns.reward), "reward") - for value in values - ) return SequenceEpisode( - episode_id=self.source.episode_namespace + str(next(iter(identities))), + episode_id=self.source.episode_namespace + str(identity), item_positions=positions, observations={ - name: tuple(decode(row[column], column) for row in rows) - for name, column in observation_columns.items() + name: values(column) for name, column in observations.items() }, actions=actions, static_inputs=static, - starts_episode=starts, + starts_episode=positions[0] == 0 and preceding is None, previous_action=preceding, - group_id=str(next(iter(groups))) if groups else None, + group_id=str(constant(groups, "collection group")) + if groups is not None + else None, transitions=TransitionColumns( - rewards=rewards, - terminated=terminated, - truncated=truncated, + **transitions, + rewards=optional_values( + columns.reward, + lambda value, column: numeric_scalar( + decode(value, column), "reward" + ), + ), successors=tuple(successors) if successors is not None else None, - bootstrap_allowed=flags(columns.bootstrap_allowed), ), ) @@ -658,22 +632,26 @@ def flush(): if not any(isinstance(value, str) for value in config.policy.action_values): numeric |= {"__executed_action", "__previous_action"} numeric |= {"__next_" + column for column in real} + types = {name: pa.float64() for name in numeric} + types.update( + { + name: pa.bool_() + for name in ("__terminated", "__truncated", "__bootstrap") + } + ) + types["__group"] = pa.string() + types.setdefault( + "__previous_action", inferred.field("__executed_action").type + ) + types.update( + { + "__next_" + column: types.get(column, inferred.field(column).type) + for column in config.policy.observation_columns + } + ) schema = pa.schema( [ - pa.field( - field.name, - pa.float64() - if field.name in numeric - else pa.bool_() - if field.name in {"__terminated", "__truncated", "__bootstrap"} - else pa.string() - if field.name == "__group" - else inferred.field("__executed_action").type - if field.name == "__previous_action" - else inferred.field(field.name[len("__next_") :]).type - if field.name.startswith("__next_") - else field.type, - ) + pa.field(field.name, types.get(field.name, field.type)) for field in inferred ] ) @@ -692,12 +670,10 @@ def flush(): raise ValueError("duplicate episode IDs in sequence import") seen.add(episode.episode_id) for position, item in enumerate(episode.item_positions): - transition = episode.transitions - successor = ( - transition.successors[position] - if transition is not None and transition.successors is not None - else None + transition = ( + episode.transitions.at(position) if episode.transitions else {} ) + successor = transition.get("successors") records.append( { "sequenceId": index, @@ -709,19 +685,15 @@ def flush(): "__previous_action": episode.previous_action if position == 0 else episode.actions[position - 1], - "__reward": transition.rewards[position] - if transition is not None and transition.rewards is not None - else None, - "__terminated": transition.terminated[position] - if transition is not None and transition.terminated is not None - else None, - "__truncated": transition.truncated[position] - if transition is not None and transition.truncated is not None - else None, - "__bootstrap": transition.bootstrap_allowed[position] - if transition is not None - and transition.bootstrap_allowed is not None - else None, + **{ + key: transition.get(field) + for key, field in { + "__reward": "rewards", + "__terminated": "terminated", + "__truncated": "truncated", + "__bootstrap": "bootstrap_allowed", + }.items() + }, "__group": episode.group_id, **{ "__next_" + column: successor[column] diff --git a/src/sequifier/trajectory/returns.py b/src/sequifier/trajectory/returns.py index b07843b9..a9795120 100644 --- a/src/sequifier/trajectory/returns.py +++ b/src/sequifier/trajectory/returns.py @@ -2,9 +2,7 @@ from collections.abc import Sequence -from sequifier.config.policy_config import ReturnDefinition from sequifier.rollout.history import numeric_scalar -from sequifier.trajectory.schema import Trajectory def return_to_go(rewards: Sequence[float], gamma: float = 1.0) -> list[float]: @@ -34,17 +32,3 @@ def remaining_return(desired_return: float, reward: float, gamma: float = 1.0) - / gamma, "successor desired_return", ) - - -def labeled_episode_length( - trajectory: Trajectory, definition: ReturnDefinition -) -> int | None: - """Exclude unknown tails; an explicit horizon labels only a fully observed prefix.""" - length = len(trajectory.transitions) - horizon = definition.finite_horizon - if horizon is not None and length >= horizon: - return horizon - if trajectory.transitions[-1].terminated: - return length - # A time limit is not a known terminal return without a finite-horizon objective. - return None diff --git a/tests/unit/test_normalization_parameter_groups.py b/tests/unit/test_normalization_parameter_groups.py deleted file mode 100644 index 36b59f6c..00000000 --- a/tests/unit/test_normalization_parameter_groups.py +++ /dev/null @@ -1,61 +0,0 @@ -import pytest -from torch import nn - -from sequifier.config.freezing_config import LayerFreezingConfigFields -from sequifier.model.parameter_catalog import ( - ParameterCatalog, - semantic_optimizer_groups, -) -from sequifier.training.runtime import _policy_parameter_ids - - -@pytest.mark.parametrize( - "normalization", - [ - lambda: nn.GroupNorm(2, 4), - lambda: nn.BatchNorm1d(4), - lambda: nn.BatchNorm2d(4), - lambda: nn.BatchNorm3d(4), - ], -) -def test_normalization_classification_freezing_and_optimizer_routing(normalization): - model = nn.Module() - model.ingestion = nn.Sequential(normalization(), nn.Linear(4, 4)) - model.backbone = nn.LayerNorm(4) - model.backbone.weight._sequifier_depth_parameter = True - - catalog = ParameterCatalog(model) - ingestion_normalization = catalog.select( - component="ingestion", semantic_group="normalization" - ) - backbone_normalization = catalog.select( - component="backbone", semantic_group="normalization" - ) - assert {item.parameter_kind for item in ingestion_normalization} == { - "weight", - "bias", - } - assert {item.parameter_kind for item in backbone_normalization} == { - "weight", - "bias", - } - - normalization_ids = { - id(catalog.parameter(item.parameter_id)) for item in ingestion_normalization - } - frozen = _policy_parameter_ids( - model.ingestion, - LayerFreezingConfigFields(freeze=["normalization"]), - "ingestion", - ) - assert frozen == normalization_ids - - optimizer_groups = { - group["group_id"]: {id(parameter) for parameter in group["params"]} - for group in semantic_optimizer_groups(catalog) - } - assert optimizer_groups["ingestion.normalization"] == normalization_ids - assert optimizer_groups["backbone.normalization"] == { - id(catalog.parameter(item.parameter_id)) for item in backbone_normalization - } - assert "ingestion.free_parameter" not in optimizer_groups diff --git a/tests/unit/test_static_preprocessing.py b/tests/unit/test_static_preprocessing.py deleted file mode 100644 index ac2a5b12..00000000 --- a/tests/unit/test_static_preprocessing.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Regression coverage for reused static preprocessing metadata.""" - -import json - -import polars as pl - -from sequifier.preprocess import Preprocessor - - -def test_reused_integer_static_categories_keep_their_ids(tmp_path): - metadata_path = tmp_path / "configs" / "metadata_configs" / "events.json" - metadata_path.parent.mkdir(parents=True) - (tmp_path / "data").mkdir() - existing = { - "static_columns": ["category"], - "n_classes": {"category": 5}, - "column_data_types": {"category": "Int64"}, - "id_maps": {"category": {10: 3, 20: 4}}, - "selected_columns_statistics": {}, - } - metadata_path.write_text(json.dumps(existing)) - - preprocessor = Preprocessor.__new__(Preprocessor) - preprocessor._static_source = pl.DataFrame( - {"sequenceId": [1, 2], "category": [10, 20]} - ) - preprocessor._preexisting_static_metadata = json.loads(metadata_path.read_text()) - preprocessor.static_categorical_columns = ["category"] - preprocessor.static_real_columns = [] - preprocessor.normalize_real_columns = True - preprocessor.project_root = str(tmp_path) - preprocessor.data_name_root = "events" - preprocessor.merge_output = True - preprocessor.output_n_classes = {} - preprocessor.output_column_data_types = {} - preprocessor.output_id_maps = {} - preprocessor.output_selected_columns_statistics = {} - - preprocessor._finalize_static_source() - - processed = pl.read_parquet(tmp_path / "data" / "events-static.parquet") - assert processed["category"].to_list() == [3, 4] - assert json.loads(metadata_path.read_text())["id_maps"]["category"] == { - "10": 3, - "20": 4, - } From a52d731d25f772b4dcefc3b9341190fcd9cfde0c Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 15:52:55 +0100 Subject: [PATCH 08/71] consolidation --- documentation/configs/train.md | 2 +- documentation/policy-learning.md | 13 +- release-notes/unreleased-policy-training.md | 12 + src/sequifier/artifacts/run_checkpoint.py | 18 +- src/sequifier/config/infer_config.py | 4 +- src/sequifier/config/policy_config.py | 89 +-- src/sequifier/config/policy_infer_config.py | 14 +- src/sequifier/config/policy_train_config.py | 178 +----- src/sequifier/config/train_config.py | 177 +++++- src/sequifier/consolidated-docs.md | 15 +- src/sequifier/io/policy_dataset.py | 16 +- src/sequifier/io/policy_window.py | 40 ++ src/sequifier/io/pt_payload.py | 28 + .../io/sequifier_dataset_from_file.py | 45 +- src/sequifier/make.py | 57 +- src/sequifier/rl/inference.py | 123 ++-- src/sequifier/rl/learner.py | 42 +- src/sequifier/rl/policy.py | 80 +-- src/sequifier/rl/runtime.py | 48 +- src/sequifier/rl/training.py | 398 ++++--------- src/sequifier/rl_api.py | 10 +- src/sequifier/runtime/builder.py | 10 +- src/sequifier/train.py | 8 +- src/sequifier/training/checkpoint_service.py | 5 + src/sequifier/training/engine.py | 30 + src/sequifier/training/loss.py | 6 + src/sequifier/training/runtime.py | 11 + src/sequifier/training/state.py | 4 + src/sequifier/trajectory/importers.py | 559 ++++++++---------- 29 files changed, 891 insertions(+), 1151 deletions(-) diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 98f7e9f9..dda3ba6b 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -780,7 +780,7 @@ schema in `model.interfaces..policy`, a sequence source in each Return-conditioned training additionally requires rewards and a known return endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories all use the same episode representation. Models support CPU/CUDA/MPS, backbone -transfer via `initialize_from`, and epoch-boundary resume via +transfer via `initialize_from`, and shared-engine checkpoint resume via `global_training.resume`. Mixed phases, IQL/PPO and image/depth policies remain planned. The former compact registries and configuration aliases have been consolidated into the shared model/dataset authoring surface. diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 3f029057..71ce22cd 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -287,13 +287,19 @@ global_training: ``` `never` is the default. `required` also requires an explicit `checkpoint_path`, -using the shared resume configuration. Resume occurs at epoch boundaries; -an interrupted epoch is repeated. Trajectory content, partition order, policy, +using the shared resume configuration. Policy runs use the same `TrainingRun`, +`TrainingEngine`, evaluation and checkpoint services as supervised runs. Resume +restores the saved optimizer boundary and loader positions. Trajectory content, partition order, policy, and learning settings must match. Device, output paths, worker/cache settings, and total epoch budget may change. Restored scheduler state retains its existing schedule when extending the epoch budget. Numerical reproducibility across devices is governed by PyTorch. +Private policy run checkpoints written before the shared-engine consolidation +use a different run-state schema. Start a new run with `initialize_from` pointing +to the old model or checkpoint; its execution weights remain loadable. New runs +write ordinary Sequifier run state and loader state. + ## Predict actions with `sequifier infer` Inference reads policy alignment and preprocessing directly from a portable @@ -419,7 +425,8 @@ Python callers use `SequifierInputs` for named model tensors and `PolicyWindowView.inputs` for raw histories. `PolicySequenceDataset` accepts `SequenceSourceConfig` entries or in-memory `SequenceEpisode` records; JSON logs are adapted with `source_episodes`. The low-level `train_policy` takes training -and validation datasets plus a `ResolvedPolicyTrainConfig`, sharing the same +and validation datasets plus a `PolicyTrainConfig` from +`sequifier.config.train_config`, sharing the same options and checkpoint lifecycle as `run_policy_training`. `CategoricalPolicy` is an alignment adapter over the ordinary network, available diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index ef396a5e..e3edc1cc 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -40,3 +40,15 @@ Policy sequence representation follow-up: with a checksummed manifest. - Repository tests were moved to `../sequifier-test/tests/sequifier-refactor-unit`; no tests were added. Verification uses lint, compilation and workflow smoke runs. + +Architecture consolidation: + +- Policy runs use the ordinary TrainingRun, TrainingEngine, evaluation, run state, + checkpoint restore and loader-state services. The private policy loop is gone. +- Model components and global optimization settings use their shared schemas; + policy records retain observation/action alignment rather than copying models. +- CSV inference and training share table-to-episode validation and causal windows. + Tabular prepared windows share the stored-tensor reader with supervised data. +- Sequence import writes columnar episodes directly, retaining optional transitions. +- Earlier private policy run checkpoints require a new run initialized from their + model weights. Portable models and execution weights remain readable. diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 857362cc..554ff562 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -80,18 +80,12 @@ def state_dict(self) -> dict[str, Any]: def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": if payload.get("artifact_type") != "sequifier_run_checkpoint": raise ValueError("Unsupported checkpoint: expected new Sequifier format.") - from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig from sequifier.config.train_config import ResolvedSequifierConfig - training_config = dict(payload["training_config"]) - config_type = ( - ResolvedPolicyTrainConfig - if "trajectory_sources" in training_config - else ResolvedSequifierConfig - ) - - if config_type is ResolvedPolicyTrainConfig and "actor_path" in training_config: - training_config["model_path"] = training_config.pop("actor_path") + if "trajectory_sources" in payload["training_config"]: + raise ValueError( + "Private policy run state requires a new run using initialize_from" + ) if ( payload.get("initialization_policy_version") != 1 @@ -110,7 +104,9 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": integration_state=IntegrationState( values=dict(payload["integration_state"]["values"]) ), - training_config=config_type.model_validate(training_config), + training_config=ResolvedSequifierConfig.model_validate( + payload["training_config"] + ), ) checkpoint.validate() return checkpoint diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 4dc46a0d..83041e54 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -439,9 +439,9 @@ def load_inferer_config( from sequifier.config.train_config import load_train_config training = load_train_config(training_config_path, {}, False) - from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig + from sequifier.config.train_config import PolicyTrainConfig - if isinstance(training, ResolvedPolicyTrainConfig): + if isinstance(training, PolicyTrainConfig): raise ValueError( "policy inference requires model_path pointing to its portable artifact" ) diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 3d761abd..0fa14158 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -4,14 +4,6 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator -from sequifier.config.components import ( - BackboneArchitectureConfig, - BackboneComponentConfig, - ComponentSpec, - DecoderComponentConfig, - IngestionComponentConfig, - ResumeConfig, -) from sequifier.io.values import DomainValue NonemptyStr = Annotated[str, Field(min_length=1)] @@ -27,53 +19,6 @@ class ReturnDefinition(PolicyConfigBase): finite_horizon: int | None = Field(default=None, gt=0) -class PolicyModelConfig(PolicyConfigBase): - context_length: int = Field(default=4, gt=0) - interface_name: str = Field(default="policy", min_length=1) - backbone: BackboneArchitectureConfig - backbone_component: BackboneComponentConfig | None = None - ingestion: IngestionComponentConfig | None = None - decoder: DecoderComponentConfig | None = None - - @model_validator(mode="before") - @classmethod - def resolve_backbone(cls, values): - if not isinstance(values, dict): - return values - values = dict(values) - if "backbone" not in values: - width = 32 - values["backbone"] = { - "dim_model": width, - "num_layers": 2, - "max_context_length": values.get("context_length", 4), - "attention": {"n_heads": 1}, - "feed_forward": {"dim": width * 4}, - "positional_encoding_scope": "global", - } - elif isinstance(values["backbone"], dict): - values["backbone"] = dict(values["backbone"]) - values["backbone"].setdefault("positional_encoding_scope", "global") - return values - - @model_validator(mode="after") - def validate_backbone(self): - if self.context_length > self.backbone.max_context_length: - raise ValueError( - "policy context_length exceeds backbone max_context_length" - ) - if self.backbone_component is not None: - if self.backbone_component.architecture != self.backbone: - raise ValueError( - "backbone component architecture must match the policy backbone" - ) - if self.backbone_component.repository is not None: - raise ValueError( - "policy phases currently use initialize_from instead of backbone repositories" - ) - return self - - class PolicySchemaConfig(PolicyConfigBase): alignment: Literal["current_observation_previous_action"] = ( "current_observation_previous_action" @@ -147,32 +92,10 @@ def categorical_n_classes(self) -> dict[str, int]: return {**self.observation_n_classes, **self.static_n_classes} -class CategoricalPolicyConfig(PolicySchemaConfig, PolicyModelConfig): - @model_validator(mode="after") - def validate_decoder(self): - if self.decoder is not None and ( - self.decoder.support != 1 or self.decoder.prediction_length != 1 - ): - raise ValueError( - "policy decoders require support=1 and prediction_length=1" - ) - return self +class CategoricalPolicyConfig(PolicySchemaConfig): + """Resolved alignment metadata; architecture belongs to the ordinary model.""" - -class PolicyOptimizationConfig(PolicyConfigBase): - batch_size: int = Field(default=64, gt=0) - num_workers: int = Field(default=0, ge=0) - episode_cache_size: int = Field(default=8, gt=0) - resume: ResumeConfig | None = None - validation_fraction: float = Field(default=0.2, gt=0, lt=1) - learning_rate: float = Field(default=0.005, gt=0, allow_inf_nan=False) - gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) - optimizer: ComponentSpec = Field(default_factory=lambda: ComponentSpec(name="Adam")) - scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - scheduler_step_on: Literal["epoch"] = "epoch" - accumulation_steps: Literal[1] = 1 - layer_type_dtypes: None = None + # Older artifacts embedded model components here. The model bundle owns them. + model_config = ConfigDict(extra="ignore", frozen=True) + context_length: int = Field(gt=0) + interface_name: NonemptyStr diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 91a6205d..cf4f73cb 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -3,9 +3,9 @@ from pathlib import Path from typing import Literal -import torch from pydantic import Field, model_validator +from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.config.policy_config import PolicyConfigBase from sequifier.config.rollout_config import RolloutConfig @@ -60,16 +60,8 @@ def policy_inference_config(values: dict) -> PolicyInferenceConfig | None: for path in paths: if not isinstance(path, str) or not path.lower().endswith(".pt"): continue - payload = torch.load( - Path(values.get("project_root", ".")) / path, - map_location="cpu", - weights_only=False, - ) - if payload.get("artifact_type") == "sequifier_run_checkpoint": - payload = payload["model"] - for name, interface in ( - payload.get("model_config", {}).get("interfaces", {}).items() - ): + artifact = read_execution_artifact(Path(values.get("project_root", ".")) / path) + for name, interface in artifact.model_config.values["interfaces"].items(): if interface.get("policy") is not None and ( selected is None or selected == name ): diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 6a9e8dfe..8c573c2a 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -3,18 +3,9 @@ from pathlib import Path from typing import Literal -from pydantic import Field, field_validator, model_validator +from pydantic import Field, model_validator -from sequifier.config.policy_config import ( - CategoricalPolicyConfig, - NonemptyStr, - PolicyConfigBase, - PolicyOptimizationConfig, -) -from sequifier.config.training_plan import ( - PolicyTrainingPlanModel, - ReturnConditionedPhaseConfig, -) +from sequifier.config.policy_config import NonemptyStr, PolicyConfigBase class PolicyTableColumns(PolicyConfigBase): @@ -111,165 +102,6 @@ def resolve_sequence_source( return SequenceSourceConfig.model_validate(values) -class _PolicyTrainConfigBase(PolicyConfigBase): - project_root: str = "." - model_name: str = Field(min_length=1) - device: str = "cpu" - initialize_from: str | None = None - seed: int = Field(default=1010, ge=0) - trajectory_sources: dict[str, SequenceSourceConfig] = Field(min_length=1) - policies: dict[str, CategoricalPolicyConfig] = Field(min_length=1) - training_plan: PolicyTrainingPlanModel - # Epochs and seed belong to the phase and run respectively. - optimization: PolicyOptimizationConfig = Field( - default_factory=PolicyOptimizationConfig - ) - - @property - def global_training(self) -> PolicyOptimizationConfig: - """Expose the shared checkpoint-selection contract.""" - return self.optimization - - @field_validator("model_name") - @classmethod - def validate_model_name(cls, value): - if value in {".", ".."} or "/" in value or "\\" in value: - raise ValueError("model_name must be a single path component") - return value - - @model_validator(mode="after") - def validate_references(self): - if len(self.training_plan.phases) != 1 or not isinstance( - self.training_plan.phases[0], ReturnConditionedPhaseConfig - ): - raise ValueError("this learner supports one return_conditioned phase") - phase = self.training_plan.phases[0] - if phase.policy not in self.policies: - raise ValueError(f"unknown policy {phase.policy!r}") - for names in (phase.trajectory_sources, phase.validation_sources or []): - if len(names) != len(set(names)): - raise ValueError("trajectory source references must be unique") - unknown = set(names) - self.trajectory_sources.keys() - if unknown: - raise ValueError(f"unknown trajectory sources: {sorted(unknown)}") - if set(phase.trajectory_sources) & set(phase.validation_sources or []): - raise ValueError("training and validation sources must be disjoint") - return self - - -class PolicyTrainConfig(_PolicyTrainConfigBase): - """Canonical model/dataset authoring, resolved before field validation.""" - - model_path: NonemptyStr | None = None - report_path: NonemptyStr | None = None - - @model_validator(mode="before") - @classmethod - def resolve_authoring(cls, values): - from sequifier.config.train_config import ModelSpecModel - - values = dict(values) - if { - "policies", - "trajectory_sources", - "data_sources", - "optimization", - "actor_path", - } & values.keys(): - raise ValueError( - "use model, dataset_training, global_training and model_path" - ) - model = ModelSpecModel.model_validate(values.pop("model", None)) - optimization = dict(values.pop("global_training", {})) - context = optimization.pop( - "context_length", model.backbone.architecture.max_context_length - ) - if optimization.pop("training_objective", "policy") != "policy": - raise ValueError("policy phases require training_objective: policy") - values["optimization"] = optimization - policies = {} - for name, interface in model.interfaces.items(): - schema = interface.policy - if schema is None: - raise ValueError("policy model interfaces require a policy schema") - if ( - interface.input_columns - and interface.input_columns - != schema.observation_columns + schema.static_columns - ): - raise ValueError( - "policy inputs must list observations and static columns in order" - ) - if interface.target_columns != [schema.action_column]: - raise ValueError("policy targets must name its action column") - if any( - ( - interface.categorical_hashing, - interface.static_conditioning, - interface.preconditioning_image_encoder, - interface.feature_layout, - ) - ): - raise ValueError("policy learning supports scalar inputs") - policies[name] = CategoricalPolicyConfig( - **schema.model_dump(), - interface_name=name, - context_length=context, - backbone=model.backbone.architecture, - backbone_component=model.backbone, - ingestion=interface.ingestion, - decoder=interface.decoder, - ) - values["policies"] = policies - phase = PolicyTrainingPlanModel.model_validate( - values.get("training_plan") - ).phases[0] - sources = {} - for name, dataset in values.pop("dataset_training", {}).items(): - if set(dataset) - {"model_interface", "parts"} or not dataset.get("parts"): - raise ValueError( - "policy datasets require model_interface and nonempty source parts" - ) - for part, source in dataset["parts"].items(): - reference = f"{name}.{part}" - if ( - reference - in phase.trajectory_sources + (phase.validation_sources or []) - and dataset.get("model_interface") != phase.policy - ): - raise ValueError( - "dataset model_interface disagrees with phase policy" - ) - sources[reference] = source - values["trajectory_sources"] = sources - return values - - -class ResolvedPolicyTrainConfig(_PolicyTrainConfigBase): - model_path: NonemptyStr - report_path: NonemptyStr - - -def resolve_policy_train_config(config: PolicyTrainConfig) -> ResolvedPolicyTrainConfig: - root = Path(config.project_root).resolve() - - def absolute(path): - return str((root / path).resolve()) - - values = config.model_dump(mode="json") - values["project_root"] = str(root) - values["trajectory_sources"] = { - name: resolve_sequence_source(source, root).model_dump() - for name, source in config.trajectory_sources.items() - } - if config.initialize_from is not None: - values["initialize_from"] = absolute(config.initialize_from) - values["model_path"] = absolute( - config.model_path or f"models/{config.model_name}/{config.model_name}.pt" - ) - values["report_path"] = absolute( - config.report_path or f"outputs/training/{config.model_name}/report.json" - ) - if values["model_path"] == values["report_path"]: - raise ValueError("model_path and report_path must differ") - return ResolvedPolicyTrainConfig.model_validate(values) +class PolicyDatasetConfig(PolicyConfigBase): + model_interface: NonemptyStr + parts: dict[str, SequenceSourceConfig] = Field(min_length=1) diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 299281e4..2118db6f 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -59,12 +59,16 @@ OptimizerPlan, resolve_plan_scheduler_arguments, ) -from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig +from sequifier.config.policy_config import ( + CategoricalPolicyConfig, + NonemptyStr, + PolicySchemaConfig, +) from sequifier.config.policy_train_config import ( - PolicyTrainConfig, - ResolvedPolicyTrainConfig, - resolve_policy_train_config, + PolicyDatasetConfig, + resolve_sequence_source, ) +from sequifier.config.training_plan import PolicyTrainingPlanModel from sequifier.config.training_plan import ( SupervisedTrainingPlanModel as TrainingPlanModel, ) @@ -401,6 +405,8 @@ class GlobalTrainingSpecModel(BaseModel): max_ram_gb: int | float = 16 world_size: int = Field(default=1, gt=0) num_workers: int = Field(default=0, ge=0) + episode_cache_size: int = Field(default=8, gt=0) + validation_fraction: float = Field(default=0.2, gt=0, lt=1) backend: str = "nccl" layer_type_dtypes: Optional[dict[str, str]] = None layer_autocast: bool = False @@ -734,6 +740,159 @@ def validate_interface_names(cls, value): return value +class PolicyTrainConfig(BaseModel): + """One policy phase using the shared model and global training schemas.""" + + model_config = ConfigDict(extra="forbid") + project_root: str = "." + model_name: NonemptyStr = Field(pattern=r"^[^/\\]+$") + device: str = "cpu" + seed: int = Field(default=1010, ge=0) + initialize_from: str | None = None + model: ModelSpecModel + dataset_training: dict[str, PolicyDatasetConfig] = Field(min_length=1) + global_training: GlobalTrainingSpecModel + training_plan: PolicyTrainingPlanModel + model_path: NonemptyStr + report_path: NonemptyStr + + @model_validator(mode="before") + @classmethod + def defaults(cls, values): + values = dict(values) + model = ModelSpecModel.model_validate(values.get("model")) + values["model"] = model + training = values.get("global_training", {}) + if isinstance(training, GlobalTrainingSpecModel): + training = training.model_dump() + values["global_training"] = { + "training_objective": "policy", + "target_offset": 0, + "context_length": model.backbone.architecture.max_context_length, + "batch_size": 64, + "inference_batch_size": 1, + "learning_rate": 0.005, + "gradient_clip": 1.0, + "accumulation_steps": 1, + "torch_compile": "none", + "reset_optimization_on_phase": False, + "calculate_validation_loss_on_initialization": False, + "scheduler": {"name": "ConstantLR", "factor": 1.0, "total_iters": 1}, + **training, + } + root = Path(values.get("project_root", ".")).resolve() + values["project_root"] = str(root) + name = values.get("model_name", "") + for key, default in ( + ("model_path", f"models/{name}/{name}.pt"), + ("report_path", f"outputs/training/{name}/report.json"), + ): + path = values.get(key) + if path == "": + raise ValueError("output paths must be nonempty") + values[key] = str((root / (path or default)).resolve()) + if values.get("initialize_from") is not None: + values["initialize_from"] = str( + (root / values["initialize_from"]).resolve() + ) + return values + + @property + def sources(self): + return { + f"{name}.{part}": source + for name, dataset in self.dataset_training.items() + for part, source in dataset.parts.items() + } + + @property + def policy(self) -> CategoricalPolicyConfig: + name = self.training_plan.phases[0].policy + schema = self.model.interfaces[name].policy + if schema is None: + raise ValueError("selected interface requires a policy schema") + return CategoricalPolicyConfig( + **schema.model_dump(), + interface_name=name, + context_length=self.global_training.context_length, + ) + + @model_validator(mode="after") + def resolve(self): + if self.model_name in {".", ".."}: + raise ValueError("model_name must name a single directory") + if len(self.training_plan.phases) != 1: + raise ValueError("policy training requires one return_conditioned phase") + phase = self.training_plan.phases[0] + if phase.policy not in self.model.interfaces: + raise ValueError(f"unknown policy interface {phase.policy!r}") + policy = self.policy + interface = self.model.interfaces[phase.policy] + if ( + interface.input_columns + and interface.input_columns + != policy.observation_columns + policy.static_columns + ): + raise ValueError( + "policy inputs must list observations and static columns in order" + ) + if interface.target_columns != [policy.action_column]: + raise ValueError("policy targets must name its action column") + if interface.decoder.support != 1 or interface.decoder.prediction_length != 1: + raise ValueError( + "policy decoders require support=1 and prediction_length=1" + ) + if any( + ( + interface.categorical_hashing, + interface.static_conditioning, + interface.preconditioning_image_encoder, + interface.feature_layout, + ) + ): + raise ValueError("policy learning supports scalar inputs") + if ( + self.global_training.distributed + or self.global_training.training_objective != "policy" + or self.global_training.target_offset != 0 + or self.global_training.window_stride not in {None, 1} + ): + raise ValueError( + "policy training requires local policy alignment, target_offset=0 and stride=1" + ) + if ( + self.global_training.context_length + > self.model.backbone.architecture.max_context_length + ): + raise ValueError("policy context exceeds backbone max_context_length") + if self.model_path == self.report_path: + raise ValueError("model_path and report_path must differ") + if self.model.backbone.repository is not None: + raise ValueError("policy runs use initialize_from for backbone transfer") + names = phase.trajectory_sources + (phase.validation_sources or []) + if len(names) != len(set(names)) or set(names) - self.sources.keys(): + raise ValueError( + "phase sources must be unique, disjoint dataset.part references" + ) + for name in names: + dataset = self.dataset_training[name.partition(".")[0]] + if dataset.model_interface != phase.policy: + raise ValueError("dataset model_interface disagrees with phase policy") + root = Path(self.project_root) + self.dataset_training = { + name: dataset.model_copy( + update={ + "parts": { + part: resolve_sequence_source(source, root) + for part, source in dataset.parts.items() + } + } + ) + for name, dataset in self.dataset_training.items() + } + return self + + class DatasetPartSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -1160,7 +1319,7 @@ class ResolvedTrainingSource(BaseModel): class ResolvedTrainingPhase(BaseModel): model_config = ConfigDict(extra="forbid") - algorithm: Literal["supervised"] = "supervised" + algorithm: Literal["supervised", "return_conditioned"] = "supervised" name: str epochs: int @@ -1294,7 +1453,7 @@ def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: @dataclass(frozen=True) class LoadedTrainConfig: config: SequifierConfig | PolicyTrainConfig - resolved: ResolvedSequifierConfig | ResolvedPolicyTrainConfig + resolved: ResolvedSequifierConfig | PolicyTrainConfig metadata: dict[str, DatasetMetadata] @@ -2185,9 +2344,7 @@ def load_train_config_with_source( "trajectory training does not accept supervised metadata overrides" ) authored = PolicyTrainConfig.model_validate(raw) - return LoadedTrainConfig( - config=authored, resolved=resolve_policy_train_config(authored), metadata={} - ) + return LoadedTrainConfig(config=authored, resolved=authored, metadata={}) config = try_catch_excess_keys(config_path, SequifierConfig, raw) part_overrides: dict[str, dict[str, str]] = {} @@ -2251,7 +2408,7 @@ def load_train_config_with_source( @beartype def load_train_config( config_path: str, args_config: dict[str, Any], skip_metadata: bool -) -> ResolvedSequifierConfig | ResolvedPolicyTrainConfig: +) -> ResolvedSequifierConfig | PolicyTrainConfig: return load_train_config_with_source( config_path, args_config, skip_metadata ).resolved diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index c5b5a4a3..3e7488b3 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -1731,7 +1731,7 @@ schema in `model.interfaces..policy`, a sequence source in each Return-conditioned training additionally requires rewards and a known return endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories all use the same episode representation. Models support CPU/CUDA/MPS, backbone -transfer via `initialize_from`, and epoch-boundary resume via +transfer via `initialize_from`, and shared-engine checkpoint resume via `global_training.resume`. Mixed phases, IQL/PPO and image/depth policies remain planned. The former compact registries and configuration aliases have been consolidated into the shared model/dataset authoring surface. @@ -2626,13 +2626,19 @@ global_training: ``` `never` is the default. `required` also requires an explicit `checkpoint_path`, -using the shared resume configuration. Resume occurs at epoch boundaries; -an interrupted epoch is repeated. Trajectory content, partition order, policy, +using the shared resume configuration. Policy runs use the same `TrainingRun`, +`TrainingEngine`, evaluation and checkpoint services as supervised runs. Resume +restores the saved optimizer boundary and loader positions. Trajectory content, partition order, policy, and learning settings must match. Device, output paths, worker/cache settings, and total epoch budget may change. Restored scheduler state retains its existing schedule when extending the epoch budget. Numerical reproducibility across devices is governed by PyTorch. +Private policy run checkpoints written before the shared-engine consolidation +use a different run-state schema. Start a new run with `initialize_from` pointing +to the old model or checkpoint; its execution weights remain loadable. New runs +write ordinary Sequifier run state and loader state. + ## Predict actions with `sequifier infer` Inference reads policy alignment and preprocessing directly from a portable @@ -2758,7 +2764,8 @@ Python callers use `SequifierInputs` for named model tensors and `PolicyWindowView.inputs` for raw histories. `PolicySequenceDataset` accepts `SequenceSourceConfig` entries or in-memory `SequenceEpisode` records; JSON logs are adapted with `source_episodes`. The low-level `train_policy` takes training -and validation datasets plus a `ResolvedPolicyTrainConfig`, sharing the same +and validation datasets plus a `PolicyTrainConfig` from +`sequifier.config.train_config`, sharing the same options and checkpoint lifecycle as `run_policy_training`. `CategoricalPolicy` is an alignment adapter over the ordinary network, available diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index 8cc7965d..c4af613b 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -269,20 +269,8 @@ def __getitem__(self, index: int) -> SequifierBatch: ) episode = self.episodes[episode_index] action_ids, returns = self._episode_labels(episode_index) - start = max(0, step + 1 - self.config.context_length) - initial = ( - len(self.config.action_values) - if episode.starts_episode - else action_index(episode.previous_action, self.config) - ) - previous = ([initial] if start == 0 else []) + action_ids[ - max(0, start - 1) : step - ] - inputs = PolicyWindowView(self.config).inputs( - [episode.observation(position) for position in range(start, step + 1)], - previous, - returns[start : step + 1] if returns is not None else None, - episode.static_inputs, + inputs = PolicyWindowView(self.config).episode_inputs( + episode, step, returns, action_ids ) targets = torch.zeros((1, self.config.context_length), dtype=torch.int64) targets[0, -1] = action_global_ids(self.config)[action_ids[step]] diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py index 1ffc700e..976ab3bf 100644 --- a/src/sequifier/io/policy_window.py +++ b/src/sequifier/io/policy_window.py @@ -14,6 +14,21 @@ DESIRED_RETURN = "__policy_return" +def normalize_policy_features(features, metadata, interface): + """Apply persisted policy transforms identically in training and prediction.""" + features = dict(features) + mask = metadata["attention_valid_mask"] + for column, statistics in interface.selected_columns_statistics.items(): + features[column] = ( + (features[column] - statistics["mean"]) / (statistics["std"] + 1e-9) + ).masked_fill(~mask, 0.0) + if interface.policy.return_conditioned: + features[DESIRED_RETURN] = ( + features[DESIRED_RETURN] / interface.policy.returns.scale + ) + return features + + def action_index(value, config: CategoricalPolicyConfig) -> int: value = domain_scalar(value, "recorded action") try: @@ -77,6 +92,31 @@ def __init__(self, config: CategoricalPolicyConfig): StoredWindowLayout(config.context_length, 0, 2), self.view ) + def episode_inputs( + self, episode, step: int, returns, action_ids=None + ) -> SequifierInputs: + """Expose one causal decision from a validated episode in either lifecycle.""" + start = max(0, step + 1 - self.config.context_length) + initial = ( + len(self.config.action_values) + if episode.starts_episode + else action_index(episode.previous_action, self.config) + ) + previous = [initial] if start == 0 else [] + if action_ids is None: + previous += [ + action_index(value, self.config) + for value in episode.actions[max(0, start - 1) : step] + ] + else: + previous += action_ids[max(0, start - 1) : step] + return self.inputs( + [episode.observation(position) for position in range(start, step + 1)], + previous, + returns[start : step + 1] if returns is not None else None, + episode.static_inputs, + ) + def inputs( self, observations: Sequence[Mapping], diff --git a/src/sequifier/io/pt_payload.py b/src/sequifier/io/pt_payload.py index ae35139f..a4c7a91e 100644 --- a/src/sequifier/io/pt_payload.py +++ b/src/sequifier/io/pt_payload.py @@ -170,6 +170,34 @@ def validate(self, layouts=None, *, n_classes=None): return self +def tabular_tensor_batch( + data, column_data_types, columns, window_length +) -> StoredTensorBatch: + """Read the legacy tabular representation into the standard stored batch.""" + import polars as pl + + from sequifier.helpers import numpy_storage_to_pytorch + + bounds = ("startItemPosition", "splitStartItemPosition", "splitEndItemPosition") + if not set(bounds) <= set(data.columns): + raise ValueError( + "prepared windows require explicit start positions and split bounds" + ) + sequences, pads = numpy_storage_to_pytorch( + data, column_data_types, columns, window_length + ) + positions = ( + data.group_by(["sequenceId", "subsequenceId"]) + .agg(*(pl.col(name).first() for name in bounds)) + .sort(["sequenceId", "subsequenceId"]) + ) + ids, subsequences, starts, lows, highs = ( + torch.tensor(positions[name].to_numpy(), dtype=torch.int64) + for name in ("sequenceId", "subsequenceId", *bounds) + ) + return StoredTensorBatch(sequences, ids, subsequences, starts, pads, lows, highs) + + def load_pt_payload(path, *, layouts=None, n_classes=None) -> StoredTensorBatch: value = torch.load(path, map_location="cpu", weights_only=True) if isinstance(value, dict): diff --git a/src/sequifier/io/sequifier_dataset_from_file.py b/src/sequifier/io/sequifier_dataset_from_file.py index 9aba96fa..3c66a2fd 100644 --- a/src/sequifier/io/sequifier_dataset_from_file.py +++ b/src/sequifier/io/sequifier_dataset_from_file.py @@ -2,7 +2,6 @@ from collections.abc import Iterator from typing import Any -import polars as pl import torch import torch.distributed as dist from loguru import logger @@ -11,7 +10,6 @@ from sequifier.helpers import ( PANDAS_TO_TORCH_TYPES, configured_window_stride, - numpy_storage_to_pytorch, read_data, resolve_window_sampling_plan, ) @@ -117,38 +115,19 @@ def __init__(self, data_path: str, config: Any, shuffle: bool = True): self.split_end_item_positions = payload.split_end_item_positions else: assert data_df is not None - all_tensors, left_pad_lengths = numpy_storage_to_pytorch( - data=data_df, - column_data_types=column_data_types, - all_columns=all_columns, - window_length=config.storage_layout.window_length, + from sequifier.io.pt_payload import tabular_tensor_batch + + payload = tabular_tensor_batch( + data_df, + column_data_types, + all_columns, + config.storage_layout.window_length, ) - if { - "startItemPosition", - "splitStartItemPosition", - "splitEndItemPosition", - } <= set(data_df.columns): - positions = ( - data_df.group_by(["sequenceId", "subsequenceId"]) - .agg( - pl.col("startItemPosition").first(), - pl.col("splitStartItemPosition").first(), - pl.col("splitEndItemPosition").first(), - ) - .sort(["sequenceId", "subsequenceId"]) - ) - self.start_item_positions = torch.tensor( - positions["startItemPosition"].to_numpy(), dtype=torch.int64 - ) - self.sequence_ids = torch.tensor( - positions["sequenceId"].to_numpy(), dtype=torch.int64 - ) - self.split_start_item_positions = torch.tensor( - positions["splitStartItemPosition"].to_numpy(), dtype=torch.int64 - ) - self.split_end_item_positions = torch.tensor( - positions["splitEndItemPosition"].to_numpy(), dtype=torch.int64 - ) + all_tensors, left_pad_lengths = payload.sequences, payload.left_pad_lengths + self.start_item_positions = payload.start_item_positions + self.sequence_ids = payload.sequence_ids + self.split_start_item_positions = payload.split_start_item_positions + self.split_end_item_positions = payload.split_end_item_positions validate_split_bounds_available( self.start_item_positions, self.split_start_item_positions, diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 22011c5a..80c7d4cc 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -101,72 +101,39 @@ dim_model: 32 num_layers: 2 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 128 - activation: swiglu + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: &id001 - - OBSERVATION_COLUMN - target_columns: - - action + input_columns: [OBSERVATION_COLUMN] + target_columns: [action] policy: - observation_columns: *id001 - action_column: action - action_values: - - 0.0 - - 1.0 + observation_columns: [OBSERVATION_COLUMN] + action_values: [0.0, 1.0] return_conditioned: true - returns: - gamma: 1.0 - scale: 1.0 ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - OBSERVATION_COLUMN: 1 - __policy_previous_action: 6 - __policy_return: 1 - decoder: - type: linear - prediction_length: 1 - support: 1 + feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} + decoder: {type: linear, prediction_length: 1, support: 1} dataset_training: demonstrations: model_interface: actor parts: - recorded: - paths: - - PLEASE_FILL.csv + recorded: {paths: [PLEASE_FILL.csv]} global_training: training_objective: policy context_length: 4 batch_size: 64 - num_workers: 0 - episode_cache_size: 8 - resume: - policy: never learning_rate: 0.005 - gradient_clip: 1.0 validation_fraction: 0.2 - optimizer: - name: Adam - scheduler: - name: ConstantLR - factor: 1.0 - total_iters: 1 + episode_cache_size: 8 + resume: {policy: never} training_plan: phases: - - name: offline_policy - algorithm: return_conditioned - epochs: 80 - sources: - - source: demonstrations.recorded - policy: actor + - {algorithm: return_conditioned, policy: actor, epochs: 80, sources: [{source: demonstrations.recorded}]} """ policy_infer_config_string = """project_root: . diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 6c990c01..aa721937 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -1,6 +1,5 @@ """CLI policy prediction and environment execution using the shared model lifecycle.""" -from collections import deque from pathlib import Path import polars as pl @@ -8,16 +7,18 @@ from loguru import logger from sequifier.config.policy_infer_config import PolicyInferenceConfig +from sequifier.config.policy_train_config import PolicyTableColumns from sequifier.helpers import configure_determinism, normalize_path, write_data from sequifier.io.batch import collate_model_inputs -from sequifier.io.policy_window import PolicyWindowView, action_index +from sequifier.io.policy_window import PolicyWindowView from sequifier.io.sequence import ordered_sequence_rows -from sequifier.io.values import domain_fields, domain_scalar +from sequifier.io.values import domain_scalar from sequifier.rl.checkpoint import load_policy from sequifier.rl.distributions import CategoricalActionDistribution from sequifier.rl.runtime import collect_policy_rollout from sequifier.rollout.factories import load_adapter from sequifier.rollout.history import numeric_scalar +from sequifier.trajectory.importers import table_episode from sequifier.trajectory.schema import SeedHistory from sequifier.trajectory.store import save_trajectory @@ -152,84 +153,66 @@ def flush(): pending_inputs.clear() pending_rows.clear() - with torch.no_grad(): - for group in groups: - group = ordered_sequence_rows(group, config.item_position_column) - first_position = ( - group[config.item_position_column][0] - if config.item_position_column in group.columns - else 0 - ) - if first_position > 0 and config.previous_action_column is None: + view = PolicyWindowView(spec) + for group in groups: + if config.item_position_column not in group.columns: + group = group.with_row_index(config.item_position_column) + group = ordered_sequence_rows(group, config.item_position_column) + first_position = group[config.item_position_column][0] + previous_column = config.previous_action_column + if previous_column is not None: + previous = group[previous_column].to_list() + if any( + value is None for value in previous[0 if first_position > 0 else 1 :] + ): raise ValueError( - "fragment inference requires an explicit preceding action" + "null previous actions are allowed only at sequence start" ) - observations, previous_actions, budgets = ( - deque(maxlen=spec.context_length) for _ in range(3) - ) - previous_action = len(spec.action_values) - initial_static = None - if ( - len(group) > 1 - and config.previous_action_column is None - and spec.action_column not in group.columns - ): + actions = previous[1:] + else: + if first_position > 0: raise ValueError( - "sequential policy inference requires recorded actions or previous_action_column" + "fragment inference requires an explicit preceding action" ) - if ( - len(group) > 1 - and config.previous_action_column is None - and spec.action_column in spec.observation_columns + spec.static_columns + if len(group) > 1 and ( + spec.action_column not in group.columns + or spec.action_column in spec.observation_columns + spec.static_columns ): raise ValueError( - "previous_action_column is required when observations and actions share a name" - ) - for position, row in enumerate(group.iter_rows(named=True)): - static = domain_fields( - {column: row[column] for column in spec.static_columns}, - set(spec.static_columns), - "policy static inputs", + "sequential inference requires recorded actions or previous_action_column" ) - if initial_static is not None and static != initial_static: - raise ValueError( - "policy static inputs must remain constant within a sequence" - ) - initial_static = static - observations.append( - {column: row[column] for column in spec.observation_columns} - ) - if config.previous_action_column is not None: - value = row[config.previous_action_column] - if (position > 0 or first_position > 0) and value is None: - raise ValueError( - "null previous actions are allowed only at sequence start" - ) - previous_action = ( - len(spec.action_values) - if value is None - else action_index(value, spec) - ) - previous_actions.append(previous_action) - budgets.append(budget(row)) - pending_inputs.append( - PolicyWindowView(spec).inputs( - list(observations), - list(previous_actions), - list(budgets) if spec.return_conditioned else None, - static, - ) - ) - pending_rows.append(row["__policy_row"]) - if len(pending_inputs) >= config.inference_batch_size: - flush() - if config.previous_action_column is None and position + 1 < len(group): - previous_action = action_index(row[spec.action_column], spec) + actions = group[spec.action_column].to_list()[:-1] if len(group) > 1 else [] + label = "__inference_action" + while label in group.columns: + label += "_" + episode = table_episode( + group.with_columns(pl.Series(label, actions + [spec.action_values[0]])), + spec, + PolicyTableColumns( + action=label, + previous_action=previous_column, + item_position=config.item_position_column, + reward=None, + terminated=None, + truncated=None, + ), + sequence_id=0, + ) + returns = ( + [budget(row) for row in group.iter_rows(named=True)] + if spec.return_conditioned + else None + ) + for step, row_index in enumerate(group["__policy_row"]): + pending_inputs.append(view.episode_inputs(episode, step, returns)) + pending_rows.append(row_index) + if len(pending_inputs) >= config.inference_batch_size: + flush() flush() predictions = pl.DataFrame(outputs).sort("__policy_row").drop("__policy_row") identity = [ column - for column in (config.sequence_id_column, "itemPosition") + for column in (config.sequence_id_column, config.item_position_column) if column in data.columns and column not in predictions.columns ] predictions = ( diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index d21c09c7..a7ebe973 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -1,29 +1,31 @@ """Algorithm boundary used by trajectory-backed train runs and Python callers.""" import json +import re from pathlib import Path from sequifier.artifacts.run_checkpoint import checkpoint_path -from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig +from sequifier.config.train_config import PolicyTrainConfig from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode +from sequifier.logging_paths import dataset_artifact_prefix from sequifier.rl.checkpoint import save_policy from sequifier.rl.training import PolicyTrainingResult, train_policy from sequifier.trajectory.importers import source_paths -def run_policy_training(config: ResolvedPolicyTrainConfig) -> PolicyTrainingResult: +def run_policy_training(config: PolicyTrainConfig) -> PolicyTrainingResult: """Prepare episode partitions, train, and publish the selected policy and report.""" phase = config.training_plan.phases[0] - spec = config.policies[phase.policy] + spec = config.policy all_paths: dict[str, list[Path]] = {} for name in phase.trajectory_sources + (phase.validation_sources or []): - all_paths[name] = source_paths(config.trajectory_sources[name]) + all_paths[name] = source_paths(config.sources[name]) flattened = [path for paths in all_paths.values() for path in paths] protected_paths = set(flattened) protected_paths.update( Path(path).resolve() for name in all_paths - for source in [config.trajectory_sources[name]] + for source in [config.sources[name]] for path in ( source.manifest, source.metadata_config_path, @@ -36,8 +38,30 @@ def run_policy_training(config: ResolvedPolicyTrainConfig) -> PolicyTrainingResu Path(config.report_path), checkpoint_path(config), ] + checkpoint = output_paths[-1] + prefix = dataset_artifact_prefix(config.project_root, config.model_name) + output_paths += [checkpoint.parent / "manifest.json"] + [ + Path(f"{prefix}-{name}.csv") + for name in ( + "training", + "training-full", + "validation", + "validation-full", + "validation-class-shares", + ) + ] + for path in protected_paths: + snapshot = re.fullmatch( + re.escape(config.model_name) + r"-epoch-(\d+)(?:-batch-\d+)?\.pt", path.name + ) + if path.parent == checkpoint.parent and snapshot is not None: + epoch = int(snapshot[1]) + if 0 < epoch <= phase.epochs: + raise ValueError( + "training snapshots must not overwrite trajectory inputs" + ) if len(output_paths) != len(set(output_paths)): - raise ValueError("model, report, and run checkpoint paths must differ") + raise ValueError("training outputs must use distinct paths") if set(output_paths) & protected_paths: raise ValueError("training outputs must not overwrite trajectory inputs") if len(flattened) != len(set(flattened)): @@ -45,16 +69,16 @@ def run_policy_training(config: ResolvedPolicyTrainConfig) -> PolicyTrainingResu def dataset(names: list[str]) -> PolicySequenceDataset: return PolicySequenceDataset( - [config.trajectory_sources[name] for name in names], + [config.sources[name] for name in names], spec, - episode_cache_size=config.optimization.episode_cache_size, + episode_cache_size=config.global_training.episode_cache_size, ) corpus = dataset(phase.trajectory_sources) if phase.validation_sources is None: training, validation = split_by_episode( corpus, - config.optimization.validation_fraction, + config.global_training.validation_fraction, config.seed, group_identical=True, ) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 74d96c3e..0426921e 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -14,14 +14,15 @@ PREVIOUS_ACTION, action_global_ids, action_target, + normalize_policy_features, ) from sequifier.model.execution_schema import ExecutionSchema -from sequifier.model.factory import build_transformer_network -from sequifier.model.ingestions import get_feature_embedding_dims from sequifier.rl.distributions import CategoricalActionDistribution -def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu"): +def policy_execution_config( + config: CategoricalPolicyConfig, model, device: str = "cpu" +): """Resolve policy features through the same execution schema as forecasts. Actions are decoder IDs; the extra input-only ID is the episode-start token. @@ -43,27 +44,12 @@ def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu" PREVIOUS_ACTION: action_width + 1, target: action_width, } - categorical_inputs = [column for column in columns if column in categorical] - width = max(config.backbone.dim_model, len(real) + 2 * len(categorical_inputs)) - default_ingestion = { - "type": "embedding", - "output_dim": width, - "feature_embedding_dims": { - **{column: 1 for column in real}, - **get_feature_embedding_dims( - width - len(real), categorical_inputs, [], n_classes - ), - }, - } + interface = model.interfaces[config.interface_name] values = { "training_objective": "policy", "context_length": config.context_length, "target_offset": 0, - "backbone": ( - config.backbone_component.model_dump(mode="json") - if config.backbone_component is not None - else {"architecture": config.backbone.model_dump(mode="json")} - ), + "backbone": model.backbone.model_dump(mode="json"), "interfaces": { config.interface_name: { "policy": config.model_dump(mode="json"), @@ -92,20 +78,8 @@ def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu" for column in real if column != DESIRED_RETURN }, - "ingestion": ( - config.ingestion.model_dump(mode="json") - if config.ingestion - else default_ingestion - ), - "decoder": ( - config.decoder.model_dump(mode="json") - if config.decoder - else { - "type": "linear", - "prediction_length": 1, - "support": 1, - } - ), + "ingestion": interface.ingestion.model_dump(mode="json"), + "decoder": interface.decoder.model_dump(mode="json"), } }, } @@ -115,28 +89,14 @@ def policy_execution_config(config: CategoricalPolicyConfig, device: str = "cpu" class CategoricalPolicy(nn.Module): """A convenience view; the network owns all trainable weights and execution.""" - def __init__(self, config: CategoricalPolicyConfig, *, device: str = "cpu"): - super().__init__() - self.config = config - self.execution_config = policy_execution_config(config, device) - self.network = build_transformer_network( - self.execution_config, device=torch.device(device) - ).network - self.interface_name = config.interface_name - @classmethod def from_network(cls, network, execution_config, interface_name: str): - interface = next( - dataset.interface - for dataset in execution_config.dataset_training.values() - if dataset.model_interface == interface_name - ) + interface = execution_config.interface(interface_name) if interface.policy is None: raise ValueError( f"Model interface {interface_name!r} has no policy alignment" ) - policy = cls.__new__(cls) - nn.Module.__init__(policy) + policy = cls() policy.config = interface.policy policy.execution_config = execution_config policy.interface_name = interface_name @@ -145,11 +105,7 @@ def from_network(cls, network, execution_config, interface_name: str): @property def interface(self): - return next( - dataset.interface - for dataset in self.execution_config.dataset_training.values() - if dataset.model_interface == self.interface_name - ) + return self.execution_config.interface(self.interface_name) @property def device(self) -> torch.device: @@ -183,17 +139,9 @@ def model_inputs( ) -> tuple[dict[str, Tensor], dict[str, Tensor]]: """Encode raw named histories without mutating stored data or masks.""" self.validate_inputs(inputs) - features = dict(inputs.inputs) - mask = inputs.attention_valid_mask - for column, statistics in self.interface.selected_columns_statistics.items(): - features[column] = ( - (features[column] - statistics["mean"]) / (statistics["std"] + 1e-9) - ).masked_fill(~mask, 0.0) - if self.config.return_conditioned: - features[DESIRED_RETURN] = ( - features[DESIRED_RETURN] / self.config.returns.scale - ) - return features, inputs.metadata + return normalize_policy_features( + inputs.inputs, inputs.metadata, self.interface + ), inputs.metadata def forward(self, inputs: SequifierInputs) -> Tensor: features, metadata = self.model_inputs(inputs) diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index 8e62d428..5d5f508f 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -1,7 +1,5 @@ """Return-conditioned actors executed through the ordinary rollout coordinator.""" -from collections import deque - import torch from sequifier.config.rollout_config import RolloutConfig @@ -43,17 +41,14 @@ def reset(self, request: ResetRequest) -> None: raise ValueError( "policy reset requires one current observation; prior actions are unknown" ) - if spec.return_conditioned: - self.budget = numeric_scalar(self.initial_return, "initial desired_return") - else: - if self.initial_return is not None: - raise ValueError("behavior cloning has no desired-return input") - self.budget = None - self.observations = deque(maxlen=spec.context_length) - self.previous_actions = deque(maxlen=spec.context_length) - self.budgets = deque(maxlen=spec.context_length) + if not spec.return_conditioned and self.initial_return is not None: + raise ValueError("behavior cloning has no desired-return input") self.previous_action = len(spec.action_values) - self.return_trace = [self.budget] if spec.return_conditioned else [] + self.return_trace = ( + [numeric_scalar(self.initial_return, "initial desired_return")] + if spec.return_conditioned + else [] + ) self.previous_action_trace = [] self.log_probabilities = [] self.generator = torch.Generator( @@ -75,9 +70,21 @@ def __call__(self, history: SeedHistory) -> dict[str, DomainValue]: column: values[-1] for column, values in history.observations.items() } inputs = PolicyWindowView(spec).inputs( - list(self.observations) + [observation], - list(self.previous_actions) + [self.previous_action], - list(self.budgets) + [self.budget] if spec.return_conditioned else None, + [ + { + name: values[position] + for name, values in history.observations.items() + } + for position, valid in enumerate(history.attention_valid_mask) + if valid + ], + self.previous_action_trace[ + max(0, len(self.previous_action_trace) - spec.context_length + 1) : + ] + + [self.previous_action], + self.return_trace[-spec.context_length :] + if spec.return_conditioned + else None, history.static_inputs, ) with torch.no_grad(): @@ -108,17 +115,14 @@ def observe(self, transition: Transition) -> None: spec.action_column: spec.action_values[index] }: raise ValueError("executed transition does not match the policy decision") - self.observations.append(observation) - self.previous_actions.append(self.previous_action) self.previous_action_trace.append(self.previous_action) self.log_probabilities.append(log_probability) if spec.return_conditioned: - assert self.budget is not None - self.budgets.append(self.budget) - self.budget = remaining_return( - self.budget, transition.reward, spec.returns.gamma + self.return_trace.append( + remaining_return( + self.return_trace[-1], transition.reward, spec.returns.gamma + ) ) - self.return_trace.append(self.budget) self.previous_action = index self.pending = None self.ended = transition.terminated or transition.truncated diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 308633b2..af2ba556 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -1,36 +1,28 @@ -"""Supervised masked action likelihood using Sequifier's optimizer transaction.""" +"""Policy datasets adapted to Sequifier's ordinary TrainingRun and TrainingEngine.""" +import hashlib +import json from dataclasses import dataclass +from typing import Literal import torch -from loguru import logger from torch.utils.data import DataLoader -from sequifier.artifacts.model_artifact import ( - build_model_artifact, - load_execution_model, +from sequifier.artifacts.model_artifact import load_execution_model +from sequifier.config.train_config import ( + EvaluationMonitorSpecModel, + PolicyTrainConfig, + ResolvedTrainingPhase, + ResolvedTrainingSource, ) -from sequifier.artifacts.run_checkpoint import ( - RUN_CHECKPOINT_FORMAT_VERSION, - DistributedRandomState, - IntegrationState, - LoaderState, - RunCheckpoint, - RunCheckpointStore, - select_run_checkpoint, -) -from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig from sequifier.integration.callbacks import IntegrationManager -from sequifier.integration.contexts import StepIdentity from sequifier.io.batch import collate_sequence_batches from sequifier.io.policy_dataset import EpisodeWindowSampler, PolicySequenceDataset -from sequifier.model.factory import InterfaceRuntimeMetadata -from sequifier.objectives import PolicyObjective -from sequifier.rl.policy import CategoricalPolicy -from sequifier.runtime.random_state import RandomState, RandomStateManager -from sequifier.training.loss import LossService, PreparedBatch -from sequifier.training.optimization import OptimizationRuntime, UpdatePolicy -from sequifier.training.runtime import DatasetRuntime, _criterion_modules +from sequifier.rl.policy import CategoricalPolicy, policy_execution_config +from sequifier.runtime.builder import RunBuilder, TrainingRun +from sequifier.runtime.context import ExecutionEnvironment +from sequifier.training.engine import TrainingEngine +from sequifier.training.runtime import DatasetPartRuntime, PartLoaderFactory @dataclass(frozen=True) @@ -46,292 +38,130 @@ class PolicyTrainingResult: def train_policy( training: PolicySequenceDataset, validation: PolicySequenceDataset, - run_config: ResolvedPolicyTrainConfig, + run_config: PolicyTrainConfig, *, integrations: IntegrationManager | None = None, ) -> PolicyTrainingResult: + phase = run_config.training_plan.phases[0] if not len(training) or not len(validation): raise ValueError("policy training and validation partitions must be nonempty") - if training.config != validation.config: + if training.config != validation.config or training.config != run_config.policy: raise ValueError( - "training and validation must share the policy/return definition" + "dataset policies must match the resolved training configuration" ) - phase = run_config.training_plan.phases[0] - config = run_config.optimization - epochs, seed = phase.epochs, run_config.seed - if training.config != run_config.policies[phase.policy]: - raise ValueError("dataset policy must match the resolved run configuration") - train_ids = tuple(training.episode_ids) - validation_ids = tuple(validation.episode_ids) - if set(train_ids) & set(validation_ids): + if set(training.episode_ids) & set(validation.episode_ids): raise ValueError( "training and validation partitions must not share episode IDs" ) fitted = training.fit_categorical_config() - training.use_config(fitted) - validation.use_config(fitted) - policies = {**run_config.policies, phase.policy: fitted} - run_config = run_config.model_copy(update={"policies": policies}) - # Seed model initialization and data order independently. - device = run_config.device - cuda_index = torch.device(device).index - cuda_devices = ( - [cuda_index if cuda_index is not None else torch.cuda.current_device()] - if device.startswith("cuda") - else [] + for dataset in (training, validation): + dataset.use_config(fitted) + config = policy_execution_config(fitted, run_config.model, run_config.device) + config.project_root, config.model_name, config.seed = ( + run_config.project_root, + run_config.model_name, + run_config.seed, ) - with torch.random.fork_rng(devices=cuda_devices): - torch.manual_seed(seed) - policy = CategoricalPolicy(training.config, device=device) - policy.set_normalization_statistics(training.normalization_statistics()) - resume_path = select_run_checkpoint(run_config) - if run_config.initialize_from is not None and resume_path is None: - source, source_config, _ = load_execution_model( - run_config.initialize_from, device=device + config.global_training = run_config.global_training + name, part_name = fitted.interface_name, "episodes" + interface = config.interface(name) + interface.selected_columns_statistics = training.normalization_statistics() + source = ResolvedTrainingSource( + source=f"{name}.{part_name}", dataset=name, part=part_name + ) + config.training_plan = [ + ResolvedTrainingPhase( + name=phase.name, + algorithm="return_conditioned", + epochs=phase.epochs, + mode="sequential", + sources=[source], + ) + ] + config.evaluation_sources = [source] + config.evaluation_monitor = EvaluationMonitorSpecModel( + source=source.source, metric="loss", mode="min" + ) + + def configure(run: TrainingRun, resuming: bool) -> None: + factory = PartLoaderFactory(config, name, part_name) + contract = {} + partitions: dict[Literal["training", "validation"], PolicySequenceDataset] = { + "training": training, + "validation": validation, + } + for split, dataset in partitions.items(): + generator = torch.Generator().manual_seed(config.seed) + factory.generators[split] = generator + factory.loaders[split] = DataLoader( + dataset, + batch_size=config.global_training.batch_size, + sampler=EpisodeWindowSampler(dataset, generator) + if split == "training" + else None, + num_workers=config.global_training.num_workers, + generator=generator, + collate_fn=collate_sequence_batches, + ) + contract[split] = hashlib.sha256( + json.dumps([dataset.content_keys, dataset.episode_ids]).encode() + ).hexdigest() + run.datasets.resolve(name).parts[part_name] = DatasetPartRuntime( + part_name, factory, data_contract=contract + ) + if not resuming and run_config.initialize_from is not None: + source_network, source_config, _ = load_execution_model( + run_config.initialize_from, device=config.device ) if ( source_config.model.backbone.architecture - != policy.execution_config.model.backbone.architecture + != config.model.backbone.architecture ): raise ValueError( "initialize_from requires the same backbone architecture" ) - policy.backbone.load_state_dict(source.backbone.state_dict(), strict=True) - del source - dataset_config = policy.execution_config.dataset_training[policy.interface_name] - interface = policy.interface - dataset_runtime = DatasetRuntime( - name=policy.interface_name, - config=dataset_config, - interface_name=policy.interface_name, - objective=PolicyObjective(), - parts={}, - criteria=_criterion_modules(dataset_config, torch.device(device)), - loss_weights=None, - class_weights=None, - frozen_parameter_ids=frozenset(), - runtime_metadata=InterfaceRuntimeMetadata( - interface.target_decoder_ids, - interface.target_n_classes, - interface.target_global_to_decoder, - ), - ) - loss_service = LossService() - - def likelihood(batch): - batch = batch.to(device) - features, metadata = policy.model_inputs(batch.model_inputs) - prepared = PreparedBatch(features, batch.targets, metadata) - output = policy.network( - features, metadata, interface_name=policy.interface_name - ) - return loss_service.calculate( - output, prepared, dataset_runtime, policy.network + run.network.backbone.load_state_dict( + source_network.backbone.state_dict(), strict=True ) - optimization = OptimizationRuntime.create( - config, device, policy.network.parameters(), phase_epochs=epochs - ) - integrations = integrations or IntegrationManager() - generator = torch.Generator().manual_seed(seed) - train_loader = DataLoader( - training, - batch_size=config.batch_size, - sampler=EpisodeWindowSampler(training, generator), - num_workers=config.num_workers, - generator=generator, - collate_fn=collate_sequence_batches, - ) - validation_loader = DataLoader( - validation, - batch_size=config.batch_size, - collate_fn=collate_sequence_batches, - num_workers=config.num_workers, + device = torch.device(config.device) + cuda_devices = ( + [device.index if device.index is not None else torch.cuda.current_device()] + if device.type == "cuda" + else [] + ) + with torch.random.fork_rng(devices=cuda_devices): + torch.manual_seed(config.seed) + run = RunBuilder().build( + config, + ExecutionEnvironment(0, 0, 1, device, False), + integrations or IntegrationManager(), + configure=configure, ) - metrics = [] - best_loss = float("inf") - best_epoch = 0 - best_weights = None - global_batch = 0 - first_epoch = 0 - random_state = RandomStateManager(torch.device(device)) - store = RunCheckpointStore(run_config, run_config.model_name) - if resume_path is not None: - checkpoint = store.load(resume_path) - previous = checkpoint.training_config - if not isinstance(previous, ResolvedPolicyTrainConfig): - raise ValueError("policy resume requires a policy training checkpoint") - - if _resume_contract(previous) != _resume_contract(run_config): - raise ValueError( - "policy resume configuration changes the training contract" - ) - expected_data = _loader_contract(training, validation) - if checkpoint.loader_state.parts != expected_data: + try: + TrainingEngine().run(run) + if run.state.best_model_state_dict is None: raise ValueError( - "policy resume trajectory content or episode order changed" + "policy training did not produce finite validation likelihood" ) - policy.network.load_state_dict( - checkpoint.model.model_state_dict, strict=True - ) - saved_interface = checkpoint.model.model_config.values["interfaces"][ - policy.interface_name - ] - policy.set_normalization_statistics( - saved_interface["selected_columns_statistics"] - ) - optimization.optimizer.load_state_dict(checkpoint.optimization.optimizer) - optimization.load_non_optimizer_state(checkpoint.optimization) - state = checkpoint.run_state - first_epoch = state["next_epoch"] - metrics = list(state["metrics"]) - best_loss, best_epoch, best_weights = ( - state["best_loss"], - state["best_epoch"], - state["best_weights"], - ) - global_batch = state["global_batch"] - generator.set_state(state["data_generator"]) - random_state.restore( - RandomState.from_state_dict(checkpoint.random_state.states[0]) - ) - if integrations.enabled: - integrations.load_state_dict(checkpoint.integration_state.values) - del checkpoint, state - optimization.optimizer.zero_grad(set_to_none=True) - - def run_epoch(loader, epoch, *, updating): - nonlocal global_batch - policy.train(updating) - total, decisions = 0.0, 0 - with torch.set_grad_enabled(updating): - for batch_index, batch in enumerate(loader): - result = likelihood(batch) - if not torch.isfinite(result.backward_loss): - raise ValueError("policy action likelihood became non-finite") - total += sum( - float(value) for value in result.accounting_sums.values() - ) - decisions += int(result.accounting_count) - if updating: - identity = StepIdentity( - epoch, - batch_index, - global_batch, - optimization.optimizer_step, - 0, - 1, - 0, - 1, - ) - optimization.accumulate( - result.backward_loss, identity, integrations, policy.network - ) - step = optimization.complete_step( - policy.network, identity, integrations, UpdatePolicy() - ) - if step.stop_requested or not step.applied: - raise RuntimeError( - "local policy optimizer did not apply its update" - ) - global_batch += 1 - return total / decisions - - def weights(): - return { - name: value.detach().cpu().clone() - for name, value in policy.network.state_dict().items() - } - - for epoch in range(first_epoch, epochs): - training_loss = run_epoch(train_loader, epoch, updating=True) - validation_loss = run_epoch(validation_loader, epoch, updating=False) - metrics.append( + run.network.load_state_dict(run.state.best_model_state_dict, strict=True) + policy = CategoricalPolicy.from_network(run.network, config, name) + policy.eval() + metrics = tuple( { - "epoch": float(epoch), - "training_nll": training_loss, - "validation_nll": validation_loss, + "epoch": float(item["epoch"] - 1), + "training_nll": item["training"][name], + "validation_nll": item["validation"][source.source], } + for item in run.state.epoch_metrics ) - logger.info( - f"Policy epoch {epoch + 1}/{epochs}: " - f"action_nll={training_loss:.6f}, " - f"validation_action_nll={validation_loss:.6f}" - ) - if validation_loss < best_loss: - best_loss = validation_loss - best_epoch = epoch - best_weights = weights() - optimization.step_scheduler(validation_loss) - checkpoint = RunCheckpoint( - format_version=RUN_CHECKPOINT_FORMAT_VERSION, - model=build_model_artifact( - policy.network, - policy.execution_config, - state_dict=weights(), - ), - optimization=optimization.state_dict(), - run_state={ - "algorithm": "return_conditioned", - "next_epoch": epoch + 1, - "metrics": metrics, - "best_loss": best_loss, - "best_epoch": best_epoch, - "best_weights": best_weights, - "global_batch": global_batch, - "data_generator": generator.get_state(), - }, - random_state=DistributedRandomState( - states=(random_state.capture_local().state_dict(),) - ), - loader_state=LoaderState(parts=_loader_contract(training, validation)), - integration_state=IntegrationState(values=integrations.state_dict()), - training_config=run_config, + return PolicyTrainingResult( + policy, + metrics, + run.state.best_epoch - 1, + run.optimization.optimizer_step, + tuple(training.episode_ids), + tuple(validation.episode_ids), ) - store.save(checkpoint, "latest") - del checkpoint - if best_weights is None: - raise ValueError( - "policy training did not produce finite validation likelihood" - ) - policy.network.load_state_dict(best_weights, strict=True) - policy.eval() - return PolicyTrainingResult( - policy, - tuple(metrics), - best_epoch, - optimization.optimizer_step, - train_ids, - validation_ids, - ) - - -def _loader_contract( - training: PolicySequenceDataset, validation: PolicySequenceDataset -) -> dict: - return { - "policy": { - "training": training.content_keys, - "validation": validation.content_keys, - "training_episode_ids": training.episode_ids, - "validation_episode_ids": validation.episode_ids, - } - } - - -def _resume_contract(config: ResolvedPolicyTrainConfig) -> dict: - # Device, output paths, workers, and epoch budget may change; learning semantics may not. - values = config.model_dump(mode="json") - for key in ( - "device", - "model_path", - "report_path", - "initialize_from", - "project_root", - ): - values.pop(key) - for phase in values["training_plan"]["phases"]: - phase.pop("epochs") - for key in ("resume", "num_workers", "episode_cache_size"): - values["optimization"].pop(key) - return values + finally: + run.distributed.finalize() diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index d1f29d6b..fe6e7aa4 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -2,20 +2,17 @@ from sequifier.config.policy_config import ( CategoricalPolicyConfig, - PolicyOptimizationConfig, PolicySchemaConfig, ReturnDefinition, ) from sequifier.config.policy_infer_config import PolicyInferenceConfig from sequifier.config.policy_train_config import ( PolicyTableColumns, - PolicyTrainConfig, - ResolvedPolicyTrainConfig, - ReturnConditionedPhaseConfig, SequenceSourceConfig, - resolve_policy_train_config, ) from sequifier.config.sequence_import_config import SequenceImportConfig +from sequifier.config.train_config import PolicyTrainConfig +from sequifier.config.training_plan import ReturnConditionedPhaseConfig from sequifier.io.batch import SequifierBatch, SequifierInputs, collate_sequence_batches from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.io.policy_window import PolicyWindowView, action_index @@ -47,10 +44,7 @@ "import_sequences", "PolicyInferenceConfig", "PolicyTrainConfig", - "PolicyOptimizationConfig", "ReturnConditionedPhaseConfig", - "ResolvedPolicyTrainConfig", - "resolve_policy_train_config", "run_policy_training", "CategoricalPolicyConfig", "ReturnDefinition", diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 6a8cc732..b54aa322 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -142,7 +142,11 @@ def _construct_loaders(self, run: TrainingRun) -> None: for dataset in run.datasets.values(): for part_name, part in dataset.parts.items(): part.loader("training") - if dataset.config.parts[part_name].validation_data_path is not None: + configured = dataset.config.parts.get(part_name) + if "validation" in part.loaders or ( + configured is not None + and configured.validation_data_path is not None + ): part.loader("validation") def build( @@ -150,6 +154,8 @@ def build( config: Any, execution: ExecutionEnvironment, integrations: IntegrationManager, + *, + configure: Callable[[TrainingRun, bool], None] | None = None, ) -> TrainingRun: strategy = self._strategy(config, execution) random_manager = RandomStateManager(execution.device) @@ -285,6 +291,8 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: loader_state=loader_state, context=context, ) + if configure is not None: + configure(run, loaded is not None) self._construct_loaders(run) if loaded is not None: restorer.restore_runtime(loaded, run) diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 1f7e8ada..27214643 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -16,11 +16,11 @@ from torch import Tensor, nn # noqa: E402 from sequifier.artifacts.model_artifact import load_execution_model # noqa: E402 -from sequifier.config.policy_train_config import ResolvedPolicyTrainConfig # noqa: E402 +from sequifier.config.train_config import PolicyTrainConfig # noqa: E402 +from sequifier.config.train_config import load_train_config # noqa: E402 from sequifier.config.train_config import ( # noqa: E402 ResolvedSequifierConfig as TrainModel, ) -from sequifier.config.train_config import load_train_config # noqa: E402 from sequifier.distributed.env import setup_distributed_env # noqa: E402 from sequifier.helpers import ( # noqa: E402 configure_determinism, @@ -160,7 +160,7 @@ def _mp_train_worker_wrapper( @beartype def run_training( - config: TrainModel | ResolvedPolicyTrainConfig, + config: TrainModel | PolicyTrainConfig, *, integration_specs: tuple[IntegrationSpec, ...] = (), integration_instances: tuple[Any, ...] = (), @@ -168,7 +168,7 @@ def run_training( ) -> None: """Launch canonical service-composed training locally or across workers.""" - if isinstance(config, ResolvedPolicyTrainConfig): + if isinstance(config, PolicyTrainConfig): if integration_specs or integration_instances or semantic_optimizer_grouping: raise ValueError( "policy training does not yet support custom integrations or parameter grouping" diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index ef706314..d595ed11 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -46,6 +46,11 @@ def validate_for_restore( model_config=checkpoint.model.model_config.values, ) current = self._resume_settings(current_config) + if current_config.global_training.training_objective == "policy": + for contract in (saved, current): + for phase in contract["training_plan"]: + phase.pop("epochs") + contract["resume_training"].pop("num_workers") mismatches = self._mismatches(saved, current) if mismatches: raise ValueError( diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 252b2db6..0545540a 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -174,6 +174,7 @@ def _update_monitor(self, run: Any, result: EvaluationResult) -> tuple[float, bo ) if improved: run.state.best_validation_loss = value + run.state.best_epoch = run.state.epoch run.state.epochs_without_improvement = 0 state = run.distributed.capture_model_state(run.network) if run.distributed.rank == 0: @@ -501,6 +502,22 @@ def flush_metrics( dataset.parts[runtime_batch.part].iterator_positions[ "training" ] = state.iterator_positions[position_key] + sums, count_tensor = run.loss.reduce_accounting( + loss.accounting_sums, + loss.accounting_count, + dataset.config.interface.target_columns, + ) + total, _ = run.loss.finalize_reduced_accounting( + sums, count_tensor, dataset, allow_empty=True + ) + count = float(count_tensor.item()) + state.training_sums[dataset.name] = ( + state.training_sums.get(dataset.name, 0.0) + + float(total.item()) * count + ) + state.training_counts[dataset.name] = ( + state.training_counts.get(dataset.name, 0.0) + count + ) batch_committed = True batch_in_progress = False window = metric_windows.setdefault( @@ -645,6 +662,19 @@ def flush_metrics( state.iterator_positions = {} state.source_scheduler_state = scheduler.state_dict() self._update_monitor(run, evaluation) + state.epoch_metrics.append( + { + "epoch": current_epoch, + "training": { + name: total / state.training_counts[name] + for name, total in state.training_sums.items() + if state.training_counts[name] + }, + "validation": evaluation.total_losses, + } + ) + state.training_sums.clear() + state.training_counts.clear() boundary = capture_runtime_boundary(run) boundary_identity = last_identity if current_epoch % config.global_training.save_interval_epochs == 0: diff --git a/src/sequifier/training/loss.py b/src/sequifier/training/loss.py index 9be8562a..4aac18e1 100644 --- a/src/sequifier/training/loss.py +++ b/src/sequifier/training/loss.py @@ -83,6 +83,10 @@ def prepare_batch( key: value.to(device, non_blocking=True) for key, value in batch.metadata.items() } + if interface.policy is not None: + from sequifier.io.policy_window import normalize_policy_features + + features = normalize_policy_features(features, metadata, interface) features, targets, metadata = dataset.objective.prepare_batch( features, targets, metadata, eval_seed=eval_seed ) @@ -280,6 +284,8 @@ def calculate( backward_loss: Tensor = total + network.regularization_loss( dataset.interface_name ) + if interface.policy is not None and not torch.isfinite(backward_loss): + raise ValueError("policy action likelihood became non-finite") accounting_dtype = ( torch.float32 if backward_loss.device.type == "mps" else torch.float64 ) diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 40aa3c1d..5ca78c67 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -170,6 +170,7 @@ class DatasetPartRuntime: iterator_positions: dict[str, int] = field(default_factory=dict) iterator_start_states: dict[str, torch.Tensor] = field(default_factory=dict) _restore_from_iterator_start: set[str] = field(default_factory=set) + data_contract: dict[str, Any] | None = None @property def loaders(self) -> dict[Literal["training", "validation"], DataLoader]: @@ -183,6 +184,8 @@ def loader_generators( @property def source_metadata(self) -> dict[str, Any]: + if self.data_contract is not None: + return self.data_contract part = self.factory.config.dataset_training[self.factory.dataset_name].parts[ self.name ] @@ -207,6 +210,7 @@ def iter_loader(self, split: Literal["training", "validation"]) -> Iterator[Any] def loader_state_dict(self) -> dict[str, Any]: return { + "data_contract": self.data_contract, "generators": { split: generator.get_state() for split, generator in self.factory.generators.items() @@ -221,6 +225,13 @@ def load_loader_state_dict( *, resume_from_iterator_start: bool, ) -> None: + if ( + self.data_contract is not None + and state.get("data_contract") != self.data_contract + ): + raise ValueError( + "Dataset content or episode order changed since the checkpoint" + ) for split, generator_state in state.get("generators", {}).items(): generator = self.factory.generators.get(split) if generator is None: diff --git a/src/sequifier/training/state.py b/src/sequifier/training/state.py index ea0c9893..340f1950 100644 --- a/src/sequifier/training/state.py +++ b/src/sequifier/training/state.py @@ -33,6 +33,10 @@ class RunState: best_validation_loss: float = float("inf") epochs_without_improvement: int = 0 best_model_state_dict: dict[str, Tensor] | None = None + best_epoch: int = 0 + epoch_metrics: list[dict[str, Any]] = field(default_factory=list) + training_sums: dict[str, float] = field(default_factory=dict) + training_counts: dict[str, float] = field(default_factory=dict) backbone_parent_revision_id: str | None = None def snapshot(self) -> RunStateSnapshot: diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 37af5541..8a122010 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -157,14 +157,8 @@ def _prepared_table_contents( Split target bounds select decisions. A preceding action may be recovered from stored context, but observations outside the split are not targets. """ - import torch - - from sequifier.helpers import ( - PANDAS_TO_TORCH_TYPES, - numpy_storage_to_pytorch, - read_data, - ) - from sequifier.io.pt_payload import load_pt_payload + from sequifier.helpers import PANDAS_TO_TORCH_TYPES, read_data + from sequifier.io.pt_payload import load_pt_payload, tabular_tensor_batch metadata = load_dataset_metadata(metadata_path) if metadata.depth_layouts: @@ -175,46 +169,24 @@ def _prepared_table_contents( for path in paths: if path.suffix.lower() == ".pt": batch = load_pt_payload(path, n_classes=metadata.n_classes) - sequences = batch.sequences - ids, starts, pads = ( - batch.sequence_ids, - batch.start_item_positions, - batch.left_pad_lengths, - ) - lows, highs = ( - batch.split_start_item_positions, - batch.split_end_item_positions, - ) else: data = read_data(str(path), path.suffix.lower().lstrip(".")) - available = sorted(set(data["inputCol"].to_list())) - sequences, pads = numpy_storage_to_pytorch( + batch = tabular_tensor_batch( data, { column: PANDAS_TO_TORCH_TYPES[dtype] for column, dtype in metadata.column_data_types.items() }, - available, + sorted(set(data["inputCol"].to_list())), metadata.window_length, ) - positions = ( - data.group_by(["sequenceId", "subsequenceId"]) - .agg( - pl.col("startItemPosition").first(), - pl.col("splitStartItemPosition").first(), - pl.col("splitEndItemPosition").first(), - ) - .sort(["sequenceId", "subsequenceId"]) - ) - ids, starts, lows, highs = ( - torch.tensor(positions[name].to_numpy()) - for name in ( - "sequenceId", - "startItemPosition", - "splitStartItemPosition", - "splitEndItemPosition", - ) - ) + sequences, ids, starts, pads = ( + batch.sequences, + batch.sequence_ids, + batch.start_item_positions, + batch.left_pad_lengths, + ) + lows, highs = batch.split_start_item_positions, batch.split_end_item_positions for row in range(len(ids)): for offset in range( int(pads[row]), next(iter(sequences.values())).shape[1] @@ -322,167 +294,184 @@ def decode(value, column): columns.action or self.policy.action_column, ) data = decisions - rows = list(data.iter_rows(named=True)) - if not rows: - raise ValueError("sequence source changed while loading") - action_column = columns.action or self.policy.action_column - observations, static_inputs = ( - {name: mapping.get(name, name) for name in names} - for names, mapping in ( - (self.policy.observation_columns, columns.observations), - (self.policy.static_columns, columns.static_inputs), + static_data = None + if self.source.static_data_path is not None: + static_data = ( + _scan((Path(self.source.static_data_path),)) + .filter(pl.col(columns.sequence_id) == self.sequence_id) + .collect() ) + return table_episode( + data, + self.policy, + columns, + sequence_id=self.sequence_id, + preceding=preceding, + static_data=static_data, + episode_end=self.source.episode_end, + episode_namespace=self.source.episode_namespace, + decode=decode, ) - inputs = set(observations.values()) - if self.source.static_data_path is None: - inputs.update(static_inputs.values()) - if action_column in inputs or columns.previous_action == action_column: - raise ValueError( - "current action labels require distinct input/previous-action columns" - ) - if ( - set(columns.observations) - observations.keys() - or set(columns.static_inputs) - static_inputs.keys() - ): - raise ValueError("table mappings must name declared policy features") - required = ( - inputs - | {columns.item_position, action_column} - | set(columns.successors.values()) + + +def table_episode( + data, + policy, + columns, + *, + sequence_id, + preceding=None, + static_data=None, + episode_end="unknown", + episode_namespace="", + decode=domain_scalar, +) -> SequenceEpisode: + """Canonical table-to-episode validation shared by training and inference.""" + data = ordered_sequence_rows(data, columns.item_position) + rows = list(data.iter_rows(named=True)) + if not rows: + raise ValueError("sequence source changed while loading") + action_column = columns.action or policy.action_column + observations, static_inputs = ( + {name: mapping.get(name, name) for name in names} + for names, mapping in ( + (policy.observation_columns, columns.observations), + (policy.static_columns, columns.static_inputs), ) - required.update( - filter( - None, - ( - columns.previous_action, - columns.bootstrap_allowed, - columns.group, - columns.episode_id, - ), - ) + ) + inputs = set(observations.values()) + if static_data is None: + inputs.update(static_inputs.values()) + if action_column in inputs or columns.previous_action == action_column: + raise ValueError( + "current action labels require distinct input/previous-action columns" + ) + if ( + set(columns.observations) - observations.keys() + or set(columns.static_inputs) - static_inputs.keys() + ): + raise ValueError("table mappings must name declared policy features") + required = ( + inputs + | {columns.item_position, action_column} + | set(columns.successors.values()) + ) + required.update( + filter( + None, + ( + columns.previous_action, + columns.bootstrap_allowed, + columns.group, + columns.episode_id, + ), + ) + ) + if required - set(data.columns): + raise ValueError( + f"policy table is missing columns: {sorted(required - set(data.columns))}" ) - if required - set(data.columns): - raise ValueError( - f"policy table is missing columns: {sorted(required - set(data.columns))}" - ) - def values(column, decoder=decode): - return tuple(decoder(row[column], column) for row in rows) + def values(column, decoder=decode): + return tuple(decoder(row[column], column) for row in rows) - def optional_values(column, decoder=decode): - if ( - column is None - or column not in data.columns - or data[column].null_count() == len(rows) - ): - return None - return values(column, decoder) - - def constant(entries, label): - unique = set(entries) - if len(unique) != 1: - raise ValueError(f"{label} must remain constant within an episode") - return next(iter(unique)) - - positions = values(columns.item_position, lambda value, _: value) - actions = values(action_column) - if columns.previous_action is not None: - preceding_actions = values( - columns.previous_action, - lambda value, _: None - if value is None - else decode(value, action_column), - ) - if preceding_actions[1:] != actions[:-1]: - raise ValueError( - "previous-action column must lag executed actions by exactly one decision" - ) - preceding = preceding_actions[0] - if self.source.static_data_path is not None: - static_rows = ( - _scan((Path(self.source.static_data_path),)) - .filter(pl.col(columns.sequence_id) == self.sequence_id) - .collect() - ) - if len(static_rows) != 1: - raise ValueError("static data requires exactly one row per sequenceId") - static_rows = list(static_rows.iter_rows(named=True)) - else: - static_rows = rows - static = { - name: constant( - (decode(row[column], column) for row in static_rows), - f"static input {name}", + def optional_values(column, decoder=decode): + if ( + column is None + or column not in data.columns + or data[column].null_count() == len(rows) + ): + return None + return values(column, decoder) + + def constant(entries, label): + unique = set(entries) + if len(unique) != 1: + raise ValueError(f"{label} must remain constant within an episode") + return next(iter(unique)) + + positions = values(columns.item_position, lambda value, _: value) + actions = values(action_column) + if columns.previous_action is not None: + preceding_actions = values( + columns.previous_action, + lambda value, _: None if value is None else decode(value, action_column), + ) + if preceding_actions[1:] != actions[:-1]: + raise ValueError( + "previous-action column must lag executed actions by exactly one decision" ) - for name, column in static_inputs.items() - } - # Strict boolean and length checks belong to TransitionColumns/SequenceEpisode. - transitions = { - field: optional_values(getattr(columns, field), lambda value, _: value) - for field in ("terminated", "truncated", "bootstrap_allowed") - } - transitions = TransitionColumns(**transitions).model_dump(exclude_none=True) - declared = self.source.episode_end - if declared != "unknown": - current = transitions.get(declared) - if current is not None and not current[-1]: - raise ValueError( - "episode_end conflicts with the recorded final boundary" - ) - transitions[declared] = (False,) * (len(rows) - 1) + (True,) - successors = None - if columns.successors: - if columns.successors.keys() != observations.keys(): - raise ValueError( - "successor mapping must cover every observation column" - ) - successors = [] - for row in rows: - present = [ - row[column] is not None for column in columns.successors.values() - ] - if any(present) and not all(present): - raise ValueError( - "a missing successor must mask all observation fields" - ) - successors.append( - { - name: decode(row[column], observations[name]) - for name, column in columns.successors.items() - } - if all(present) - else None - ) - groups = optional_values(columns.group, domain_scalar) - identity = ( - constant(values(columns.episode_id, domain_scalar), "episode ID") - if columns.episode_id - else self.sequence_id + preceding = preceding_actions[0] + if static_data is not None: + if len(static_data) != 1: + raise ValueError("static data requires exactly one row per sequenceId") + static_rows = list(static_data.iter_rows(named=True)) + else: + static_rows = rows + static = { + name: constant( + (decode(row[column], column) for row in static_rows), + f"static input {name}", ) - return SequenceEpisode( - episode_id=self.source.episode_namespace + str(identity), - item_positions=positions, - observations={ - name: values(column) for name, column in observations.items() - }, - actions=actions, - static_inputs=static, - starts_episode=positions[0] == 0 and preceding is None, - previous_action=preceding, - group_id=str(constant(groups, "collection group")) - if groups is not None - else None, - transitions=TransitionColumns( - **transitions, - rewards=optional_values( - columns.reward, - lambda value, column: numeric_scalar( - decode(value, column), "reward" - ), - ), - successors=tuple(successors) if successors is not None else None, + for name, column in static_inputs.items() + } + # Strict boolean and length checks belong to TransitionColumns/SequenceEpisode. + transitions = { + field: optional_values(getattr(columns, field), lambda value, _: value) + for field in ("terminated", "truncated", "bootstrap_allowed") + } + transitions = TransitionColumns(**transitions).model_dump(exclude_none=True) + declared = episode_end + if declared != "unknown": + current = transitions.get(declared) + if current is not None and not current[-1]: + raise ValueError("episode_end conflicts with the recorded final boundary") + transitions[declared] = (False,) * (len(rows) - 1) + (True,) + successors = None + if columns.successors: + if columns.successors.keys() != observations.keys(): + raise ValueError("successor mapping must cover every observation column") + successors = [] + for row in rows: + present = [ + row[column] is not None for column in columns.successors.values() + ] + if any(present) and not all(present): + raise ValueError("a missing successor must mask all observation fields") + successors.append( + { + name: decode(row[column], observations[name]) + for name, column in columns.successors.items() + } + if all(present) + else None + ) + groups = optional_values(columns.group, domain_scalar) + identity = ( + constant(values(columns.episode_id, domain_scalar), "episode ID") + if columns.episode_id + else sequence_id + ) + return SequenceEpisode( + episode_id=episode_namespace + str(identity), + item_positions=positions, + observations={name: values(column) for name, column in observations.items()}, + actions=actions, + static_inputs=static, + starts_episode=positions[0] == 0 and preceding is None, + previous_action=preceding, + group_id=str(constant(groups, "collection group")) + if groups is not None + else None, + transitions=TransitionColumns( + **transitions, + rewards=optional_values( + columns.reward, + lambda value, column: numeric_scalar(decode(value, column), "reward"), ), - ) + successors=tuple(successors) if successors is not None else None, + ), + ) def source_episodes(source: SequenceSourceConfig, policy: PolicySchemaConfig): @@ -565,11 +554,7 @@ def source_episodes(source: SequenceSourceConfig, policy: PolicySchemaConfig): def import_sequences(config) -> Path: - """Atomically commit raw Parquet shards and a checksummed source manifest. - - This is a lossless import, not a fit: split assignment, vocabularies, window - padding and return labels remain the responsibility of the training view. - """ + """Commit canonical columnar episodes as checksummed raw Parquet shards.""" import pyarrow as pa import pyarrow.parquet as pq @@ -580,29 +565,6 @@ def import_sequences(config) -> Path: "sequence import output_path must name a new dataset directory" ) destination.parent.mkdir(parents=True, exist_ok=True) - fields = config.policy.observation_columns + config.policy.static_columns - reserved = { - "sequenceId", - "itemPosition", - "episodeId", - "__executed_action", - "__reward", - "__terminated", - "__truncated", - "__previous_action", - "__bootstrap", - "__group", - } - reserved.update("__next_" + column for column in config.policy.observation_columns) - if set(fields) & reserved: - raise ValueError( - "policy feature names collide with sequence import metadata columns" - ) - references = source_episodes(source, config.policy) - if not references: - raise ValueError("sequence import requires at least one episode") - stage = Path(tempfile.mkdtemp(prefix=".sequifier-import-", dir=destination.parent)) - shards, records, schema, seen = [], [], None, set() columns = PolicyTableColumns( action="__executed_action", reward="__reward", @@ -610,58 +572,60 @@ def import_sequences(config) -> Path: truncated="__truncated", previous_action="__previous_action", bootstrap_allowed="__bootstrap", - group=None, + group="__group", episode_id="episodeId", successors={ - column: "__next_" + column for column in config.policy.observation_columns + name: "__next_" + name for name in config.policy.observation_columns }, ) + fields = config.policy.observation_columns + config.policy.static_columns + reserved = { + value for value in columns.model_dump().values() if isinstance(value, str) + } | set(columns.successors.values()) + if set(fields) & reserved: + raise ValueError( + "policy feature names collide with sequence import metadata columns" + ) + references = source_episodes(source, config.policy) + if not references: + raise ValueError("sequence import requires at least one episode") + real = ( + set(fields) + - set(config.policy.categorical_columns) + - config.policy.categorical_n_classes.keys() + ) + action_type = ( + pa.string() if isinstance(config.policy.action_values[0], str) else pa.float64() + ) + types = {name: pa.float64() for name in real | {"__reward"}} + types.update( + {name: pa.bool_() for name in ("__terminated", "__truncated", "__bootstrap")} + ) + types.update( + { + "__executed_action": action_type, + "__previous_action": action_type, + "__group": pa.string(), + } + ) + stage = Path(tempfile.mkdtemp(prefix=".sequifier-import-", dir=destination.parent)) + tables, shards, seen, schema = [], [], set(), None def flush(): - nonlocal records, schema - if not records: - return - if schema is None: - inferred = pa.Table.from_pylist(records).schema - real = ( - set(fields) - - set(config.policy.categorical_columns) - - config.policy.categorical_n_classes.keys() - ) - numeric = real | {"__reward"} - if not any(isinstance(value, str) for value in config.policy.action_values): - numeric |= {"__executed_action", "__previous_action"} - numeric |= {"__next_" + column for column in real} - types = {name: pa.float64() for name in numeric} - types.update( + nonlocal schema + if tables: + table = pa.concat_tables(tables, promote_options="permissive") + schema = schema or table.schema + name = f"part-{len(shards):05d}.parquet" + pq.write_table(table.cast(schema), stage / name) + shards.append( { - name: pa.bool_() - for name in ("__terminated", "__truncated", "__bootstrap") + "path": name, + "sha256": _checksum(stage / name), + "decisions": table.num_rows, } ) - types["__group"] = pa.string() - types.setdefault( - "__previous_action", inferred.field("__executed_action").type - ) - types.update( - { - "__next_" + column: types.get(column, inferred.field(column).type) - for column in config.policy.observation_columns - } - ) - schema = pa.schema( - [ - pa.field(field.name, types.get(field.name, field.type)) - for field in inferred - ] - ) - name = f"part-{len(shards):05d}.parquet" - path = stage / name - pq.write_table(pa.Table.from_pylist(records, schema=schema), path) - shards.append( - {"path": name, "sha256": _checksum(path), "decisions": len(records)} - ) - records = [] + tables.clear() try: for index, reference in enumerate(references): @@ -669,45 +633,46 @@ def flush(): if episode.episode_id in seen: raise ValueError("duplicate episode IDs in sequence import") seen.add(episode.episode_id) - for position, item in enumerate(episode.item_positions): - transition = ( - episode.transitions.at(position) if episode.transitions else {} - ) - successor = transition.get("successors") - records.append( - { - "sequenceId": index, - "itemPosition": item, - "episodeId": episode.episode_id, - **episode.observation(position), - **episode.static_inputs, - "__executed_action": episode.actions[position], - "__previous_action": episode.previous_action - if position == 0 - else episode.actions[position - 1], - **{ - key: transition.get(field) - for key, field in { - "__reward": "rewards", - "__terminated": "terminated", - "__truncated": "truncated", - "__bootstrap": "bootstrap_allowed", - }.items() - }, - "__group": episode.group_id, - **{ - "__next_" + column: successor[column] - if successor is not None - else None - for column in config.policy.observation_columns - }, - } + length = len(episode.actions) + transitions = episode.transitions or TransitionColumns() + values = { + **episode.observations, + **{ + name: [value] * length + for name, value in episode.static_inputs.items() + }, + "sequenceId": [index] * length, + "itemPosition": episode.item_positions, + "episodeId": [episode.episode_id] * length, + "__executed_action": episode.actions, + "__previous_action": (episode.previous_action, *episode.actions[:-1]), + "__group": [episode.group_id] * length, + **{ + column: getattr(transitions, field) or [None] * length + for field, column in { + "rewards": "__reward", + "terminated": "__terminated", + "truncated": "__truncated", + "bootstrap_allowed": "__bootstrap", + }.items() + }, + } + arrays = { + name: pa.array(entries, type=types.get(name)) + for name, entries in values.items() + } + for name, column in columns.successors.items(): + arrays[column] = pa.array( + [ + successor[name] if successor is not None else None + for successor in (transitions.successors or [None] * length) + ], + type=arrays[name].type, ) + tables.append(pa.table(arrays)) if (index + 1) % config.episodes_per_shard == 0: flush() flush() - # Group is optional globally; all-null group columns do not invent a group. - columns = columns.model_copy(update={"group": "__group"}) manifest = { "format": "sequifier.sequence-data", "version": 1, From 090c9ac7743b47033b63289ea21cc289ca4d4209 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 16:27:01 +0100 Subject: [PATCH 09/71] consolidate --- .gitattributes | 1 + src/sequifier/artifacts/loading.py | 11 +- src/sequifier/artifacts/model_artifact.py | 29 ++- src/sequifier/artifacts/model_config.py | 222 +++++++--------- src/sequifier/artifacts/model_export.py | 88 +------ src/sequifier/config/infer_config.py | 19 +- src/sequifier/config/policy_infer_config.py | 21 +- src/sequifier/config/train_config.py | 29 +-- src/sequifier/infer.py | 92 ++----- src/sequifier/io/policy_dataset.py | 9 + src/sequifier/io/policy_window.py | 7 +- src/sequifier/io/preprocessing.py | 34 +++ src/sequifier/preprocess.py | 141 +++------- src/sequifier/rl/checkpoint.py | 6 +- src/sequifier/rl/inference.py | 3 +- src/sequifier/rl/policy.py | 155 ++++++----- src/sequifier/rl/training.py | 80 +++--- src/sequifier/rollout/preprocessing.py | 40 ++- src/sequifier/train.py | 10 +- src/sequifier/training/engine.py | 14 +- src/sequifier/training/runtime.py | 136 +++++----- src/sequifier/trajectory/importers.py | 8 +- tests/test_consolidation.py | 269 ++++++++++++++++++++ 23 files changed, 761 insertions(+), 663 deletions(-) create mode 100644 .gitattributes create mode 100644 tests/test_consolidation.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..4f392e96 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +src/sequifier/consolidated-docs.md linguist-generated=true diff --git a/src/sequifier/artifacts/loading.py b/src/sequifier/artifacts/loading.py index a392811a..70e65b9a 100644 --- a/src/sequifier/artifacts/loading.py +++ b/src/sequifier/artifacts/loading.py @@ -9,7 +9,7 @@ import torch from torch import nn -from sequifier.artifacts.model_artifact import ModelArtifact, restore_model_artifact +from sequifier.artifacts.model_artifact import read_artifact, restore_model_artifact from sequifier.model.parameter_catalog import ParameterCatalog @@ -42,14 +42,9 @@ def load_model_for_analysis( if config is not None: raise ValueError("Current Sequifier artifacts embed their model configuration.") artifact_path = Path(path).expanduser().resolve() - payload = torch.load(artifact_path, map_location="cpu", weights_only=False) - artifact_type = payload.get("artifact_type") if isinstance(payload, dict) else None - if artifact_type == "sequifier_run_checkpoint": - payload = payload["model"] - if not isinstance(payload, dict): - raise ValueError(f"Unsupported Sequifier artifact: {artifact_path}") + artifact, artifact_type = read_artifact(artifact_path) network, resolved_config, artifact = restore_model_artifact( - ModelArtifact.from_state_dict(payload), + artifact, device=options.device, interface_name=interface_name, ) diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index e68a0f63..7090276d 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -171,29 +171,36 @@ def restore_model_artifact( return built.network, config, artifact -def read_execution_artifact(path: str | Path) -> ModelArtifact: - """Read the portable execution payload from either supported artifact kind.""" +def read_artifact(path: str | Path) -> tuple[ModelArtifact, str]: + """Read and validate an execution artifact, retaining its source kind.""" payload = torch.load( Path(path).expanduser().resolve(), map_location="cpu", weights_only=False ) - if ( - isinstance(payload, dict) - and payload.get("artifact_type") == "sequifier_run_checkpoint" - ): + artifact_type = payload.get("artifact_type") if isinstance(payload, dict) else None + if artifact_type == "sequifier_run_checkpoint": payload = payload["model"] - if not isinstance(payload, dict): - raise ValueError("Sequifier model artifact must be a mapping") - return ModelArtifact.from_state_dict(payload) + if not isinstance(payload, dict) or not isinstance(artifact_type, str): + raise ValueError( + "Sequifier model artifact must be a mapping with an artifact_type" + ) + return ModelArtifact.from_state_dict(payload), artifact_type + + +def read_execution_artifact(path: str | Path) -> ModelArtifact: + """Read the portable execution payload from either supported artifact kind.""" + return read_artifact(path)[0] def load_execution_model( - path: str | Path, + path: str | Path | ModelArtifact, *, device: str = "cpu", interface_name: str | None = None, ) -> tuple[Any, Any, ModelArtifact]: return restore_model_artifact( - read_execution_artifact(path), device=device, interface_name=interface_name + path if isinstance(path, ModelArtifact) else read_execution_artifact(path), + device=device, + interface_name=interface_name, ) diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index d73033dc..bd23595b 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -2,16 +2,11 @@ from __future__ import annotations -from pydantic import TypeAdapter - from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig from sequifier.config.train_config import ( BackboneComponentConfig, DatasetFreezingSpecModel, - DecoderComponentConfig, - FeatureLayoutRegistryModel, GlobalTrainingSpecModel, - IngestionComponentConfig, ModelInterfaceSpecModel, ModelSpecModel, ResolvedDatasetTrainingSpec, @@ -62,7 +57,7 @@ def resolved_config_from_model_config( torch_compile="none", ) authored_interfaces = {} - resolved_datasets = {} + resolved_interfaces = {} fallback_storage_layout = StoredWindowLayout( window_length=context_length + max(1, target_offset), max_target_offset=max(1, target_offset), @@ -85,104 +80,55 @@ def resolved_config_from_model_config( else fallback_storage_layout ) resolve_window_view(storage_layout, window_view) - ingestion = TypeAdapter(IngestionComponentConfig).validate_python( - interface["ingestion"] - ) - decoder = TypeAdapter(DecoderComponentConfig).validate_python( - interface["decoder"] - ) - feature_layout = ( - FeatureLayoutRegistryModel.model_validate(interface["feature_layout"]) - if interface.get("feature_layout") is not None - else None - ) - authored_interfaces[name] = ModelInterfaceSpecModel( - policy=( - PolicySchemaConfig.model_validate( - { - key: value - for key, value in interface["policy"].items() - if key in PolicySchemaConfig.model_fields - } - ) - if interface.get("policy") is not None - else None - ), - input_columns=interface["input_columns"], - static_conditioning=interface.get("static_conditioning"), - preconditioning_image_encoder=interface.get( - "preconditioning_image_encoder" - ), - target_columns=interface["target_columns"], - categorical_hashing=interface.get("categorical_hashing", {}), - categorical_decoder_special_tokens=interface.get( - "categorical_decoder_special_tokens", {} - ), - feature_layout=feature_layout, - ingestion=ingestion, - decoder=decoder, - ) + authored = { + key: value + for key, value in interface.items() + if key in ModelInterfaceSpecModel.model_fields + } + policy_spec = interface.get("policy") + if policy_spec is not None: + authored["policy"] = PolicySchemaConfig.model_validate( + { + key: value + for key, value in policy_spec.items() + if key in PolicySchemaConfig.model_fields + } + ) + authored_interfaces[name] = ModelInterfaceSpecModel.model_validate(authored) + if policy_spec is not None: + policy_spec = {"categorical_encoding": "legacy", **policy_spec} target_decoder_ids = interface.get("target_decoder_ids", {}) target_n_classes = interface.get( "target_n_classes", {column: len(ids) for column, ids in target_decoder_ids.items()}, ) - global_to_decoder = interface.get("target_global_to_decoder", {}) - policy_spec = interface.get("policy") - if policy_spec is not None: - policy_spec = dict(policy_spec) - # Existing policy artifacts used decoder IDs as input vocabulary IDs. - policy_spec.setdefault("categorical_encoding", "legacy") - resolved = ResolvedModelInterface( - policy=CategoricalPolicyConfig.model_validate(policy_spec) - if policy_spec is not None - else None, - depth_layouts=interface.get("depth_layouts", {}), - tensor_payload_version=interface.get("tensor_payload_version", 1), - split_paths=interface.get("split_paths", []), - prediction_aligned_splits=interface.get("prediction_aligned_splits", []), - prediction_alignment_steps=interface.get("prediction_alignment_steps", 1), - window_lengths=interface.get("window_lengths"), - prediction_length=interface.get("prediction_length"), - target_offset=interface.get("target_offset"), - name=name, - input_columns=interface["input_columns"], - static_conditioning=interface.get("static_conditioning"), - static_features_path=interface.get("static_features_path"), - image_sources=interface.get("image_sources", {}), - preconditioning_image=interface.get("preconditioning_image"), - preconditioning_image_encoder=interface.get( - "preconditioning_image_encoder" - ), - static_categorical_columns=interface.get("static_categorical_columns", []), - static_real_columns=interface.get("static_real_columns", []), - target_columns=interface["target_columns"], - categorical_hashing=interface.get("categorical_hashing", {}), - target_column_types=interface["target_column_types"], - column_data_types=interface["column_data_types"], - categorical_columns=interface["categorical_columns"], - real_columns=interface["real_columns"], - categorical_decoder_special_tokens=interface.get( - "categorical_decoder_special_tokens", {} - ), - feature_layout=feature_layout, - ingestion=ingestion, - decoder=decoder, - n_classes=interface.get("n_classes", target_n_classes), - id_maps=interface.get("id_maps", {}), - special_token_ids=interface.get( - "special_token_ids", SPECIAL_TOKEN_IDS.ids_by_label - ), - selected_columns_statistics=interface.get( - "selected_columns_statistics", {} - ), - normalize_real_columns=interface.get("normalize_real_columns", True), - target_decoder_ids=target_decoder_ids, - target_n_classes=target_n_classes, - target_global_to_decoder=global_to_decoder, - categorical_hash_contracts=interface.get("categorical_hash_contracts", {}), - storage_layout=storage_layout, - window_view=window_view, + resolved = ResolvedModelInterface.model_validate( + { + **{ + key: value + for key, value in interface.items() + if key in ResolvedModelInterface.model_fields + }, + "name": name, + "policy": CategoricalPolicyConfig.model_validate(policy_spec) + if policy_spec is not None + else None, + "n_classes": interface.get("n_classes", target_n_classes), + "target_decoder_ids": target_decoder_ids, + "target_n_classes": target_n_classes, + "target_global_to_decoder": interface.get( + "target_global_to_decoder", {} + ), + "id_maps": interface.get("id_maps", {}), + "special_token_ids": interface.get( + "special_token_ids", SPECIAL_TOKEN_IDS.ids_by_label + ), + "categorical_decoder_special_tokens": interface.get( + "categorical_decoder_special_tokens", {} + ), + "storage_layout": storage_layout, + "window_view": window_view, + } ) if resolved.depth_layouts and resolved.tensor_payload_version not in {2, 3}: raise ValueError( @@ -200,44 +146,52 @@ def resolved_config_from_model_config( expected_schema.to_dict(), sort_keys=True ): raise ValueError(f"Execution schema contradicts interface {name!r}") - criteria = { - target: ( - "CrossEntropyLoss" - if resolved.target_column_types[target] == "categorical" - else "MSELoss" - ) - for target in resolved.target_columns - } - resolved_datasets[name] = ResolvedDatasetTrainingSpec( - name=name, - model_interface=name, - interface=resolved, - parts={}, - criterion=criteria, - class_share_log_columns=[], - freeze=DatasetFreezingSpecModel(), - ) - model = ModelSpecModel( - backbone=backbone, - interfaces=authored_interfaces, - ) - config = ResolvedSequifierConfig( - project_root=".", - model_name="loaded-model", + resolved_interfaces[name] = resolved + config = execution_config( + ModelSpecModel(backbone=backbone, interfaces=authored_interfaces), + resolved_interfaces, + global_spec, device=device, - seed=0, - global_training=global_spec, - model=model, - dataset_training=resolved_datasets, - training_plan=[], - evaluation_sources=[], - evaluation_monitor=None, - export_generative_model=True, - export_embedding_model=False, embedding_layer_names=values.get( "embedding_layer_names", ["backbone.final_norm"] ), - export_onnx=False, - export_pt=False, ) return config, interface_name + + +def execution_config(model, interfaces, global_training, *, device="cpu", **options): + """Compose the same execution contract for loaded and newly resolved models.""" + return ResolvedSequifierConfig( + global_training=global_training, + model=model, + dataset_training={ + name: ResolvedDatasetTrainingSpec( + name=name, + model_interface=name, + interface=interface, + parts={}, + criterion={ + target: "CrossEntropyLoss" if kind == "categorical" else "MSELoss" + for target, kind in interface.target_column_types.items() + }, + class_share_log_columns=[], + freeze=DatasetFreezingSpecModel(), + ) + for name, interface in interfaces.items() + }, + **{ + "project_root": ".", + "model_name": "loaded-model", + "seed": 0, + "device": device, + "training_plan": [], + "evaluation_sources": [], + "evaluation_monitor": None, + "export_generative_model": True, + "export_embedding_model": False, + "export_onnx": False, + "export_pt": False, + "embedding_layer_names": ["backbone.final_norm"], + **options, + }, + ) diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index 3d0f564c..0a854f56 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -16,81 +16,21 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: interface = dataset.interface if interface.name in interfaces: continue - interfaces[interface.name] = { - "execution_schema": ExecutionSchema.from_interface( - interface, training_config.global_training.context_length - ).to_dict(), - "depth_layouts": interface.depth_layouts.model_dump(mode="json"), - "tensor_payload_version": interface.tensor_payload_version, - "split_paths": interface.split_paths, - "prediction_aligned_splits": interface.prediction_aligned_splits, - "prediction_alignment_steps": interface.prediction_alignment_steps, - "window_lengths": interface.window_lengths, - "prediction_length": interface.prediction_length, - "target_offset": interface.target_offset, - "input_columns": interface.input_columns, - "static_conditioning": ( - interface.static_conditioning.model_dump(mode="python") - if interface.static_conditioning is not None - else None - ), - "static_features_path": interface.static_features_path, - "image_sources": { - name: source.model_dump(mode="python", exclude={"path"}) - for name, source in interface.image_sources.items() - }, - "preconditioning_image": ( - interface.preconditioning_image.model_dump( - mode="python", exclude={"path"} - ) - if interface.preconditioning_image is not None - else None - ), - "preconditioning_image_encoder": ( - interface.preconditioning_image_encoder.model_dump(mode="python") - if interface.preconditioning_image_encoder is not None - else None - ), - "static_categorical_columns": interface.static_categorical_columns, - "static_real_columns": interface.static_real_columns, - "target_columns": interface.target_columns, - "categorical_hashing": { - column: config.model_dump(mode="python") - for column, config in interface.categorical_hashing.items() - }, - "categorical_hash_contracts": interface.categorical_hash_contracts, - "target_column_types": interface.target_column_types, - "column_data_types": interface.column_data_types, - "categorical_columns": interface.categorical_columns, - "real_columns": interface.real_columns, - "categorical_decoder_special_tokens": ( - interface.categorical_decoder_special_tokens - ), - "feature_layout": ( - interface.feature_layout.model_dump(mode="python") - if interface.feature_layout is not None - else None - ), - "ingestion": interface.ingestion.model_dump( - mode="python", - ), - "decoder": interface.decoder.model_dump( - mode="python", - ), - "n_classes": interface.n_classes, - "id_maps": interface.id_maps, - "special_token_ids": interface.special_token_ids, - "selected_columns_statistics": (interface.selected_columns_statistics), - "normalize_real_columns": interface.normalize_real_columns, - "target_decoder_ids": interface.target_decoder_ids, - "target_n_classes": interface.target_n_classes, - "target_global_to_decoder": interface.target_global_to_decoder, - "storage_layout": { - "window_length": (interface.storage_layout.window_length), - "max_target_offset": interface.storage_layout.max_target_offset, - "version": interface.storage_layout.version, + values = interface.model_dump( + mode="python", + exclude={ + "name": True, + "window_view": True, + "policy": True, + "image_sources": {"__all__": {"path"}}, + "preconditioning_image": {"path"}, }, - } + ) + values["execution_schema"] = ExecutionSchema.from_interface( + interface, training_config.global_training.context_length + ).to_dict() + values["depth_layouts"] = interface.depth_layouts.model_dump(mode="json") + interfaces[interface.name] = values if interface.policy is not None: interfaces[interface.name]["policy"] = interface.policy.model_dump( mode="json" diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 83041e54..1c878017 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -402,7 +402,8 @@ def load_inferer_config( key: value for key, value in args_config.items() if key != "skip_metadata" } authored_values = merge_config_fragments((config_values, cli_values)) - policy_config = policy_inference_config(authored_values) + artifacts = {} + policy_config = policy_inference_config(authored_values, artifacts) if policy_config is not None: return policy_config authored_values, extracted_metadata_values = extract_inline_metadata( @@ -495,8 +496,6 @@ def load_inferer_config( selected_metadata = part.metadata selected_metadata_path = part.metadata_config_path - import torch - from sequifier.artifacts.model_config import resolved_config_from_model_config autoregressive_decoder_targets: set[str] = set() @@ -505,14 +504,7 @@ def load_inferer_config( if not isinstance(model_path, str): continue if model_path.lower().endswith(".pt"): - payload = torch.load( - normalize_path(model_path, project_root), - map_location="cpu", - weights_only=False, - ) - if payload.get("artifact_type") == "sequifier_run_checkpoint": - payload = payload["model"] - model_config = payload.get("model_config") + model_config = artifacts[model_path].model_config.values elif model_path.lower().endswith(".onnx"): import onnx @@ -642,9 +634,11 @@ def load_inferer_config( layout_is_authoritative, ) - return resolve_inference_config( + resolved = resolve_inference_config( config, metadata, allow_wider_artifact_folder=artifact_only_metadata ) + resolved._artifacts = artifacts + return resolved @beartype @@ -1023,6 +1017,7 @@ class ResolvedInferenceConfig(_InferenceConfigBase[str, list[str], dict[str, str window_view: ModelWindowView dataset_metadata: Optional[DatasetMetadata] = Field(default=None, exclude=True) _image_store: Any = PrivateAttr(default=None) + _artifacts: dict[str, Any] = PrivateAttr(default_factory=dict) @model_validator(mode="after") def validate_numeric_rollout(self): diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index cf4f73cb..99da3bb0 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -3,14 +3,16 @@ from pathlib import Path from typing import Literal -from pydantic import Field, model_validator +from pydantic import Field, PrivateAttr, model_validator -from sequifier.artifacts.model_artifact import read_execution_artifact +from sequifier.artifacts.model_artifact import ModelArtifact, read_execution_artifact from sequifier.config.policy_config import PolicyConfigBase from sequifier.config.rollout_config import RolloutConfig class PolicyInferenceConfig(PolicyConfigBase): + _artifacts: dict[str, ModelArtifact] = PrivateAttr(default_factory=dict) + project_root: str = "." model_path: str model_type: Literal["policy", "generative"] = "generative" @@ -51,8 +53,11 @@ def validate_execution(self): return self -def policy_inference_config(values: dict) -> PolicyInferenceConfig | None: +def policy_inference_config( + values: dict, artifacts: dict[str, ModelArtifact] | None = None +) -> PolicyInferenceConfig | None: """Dispatch from the persisted interface contract, without training YAML.""" + artifacts = {} if artifacts is None else artifacts model_path = values.get("model_path") paths = [model_path] if isinstance(model_path, str) else model_path or [] selected = values.get("model_interface") @@ -60,7 +65,11 @@ def policy_inference_config(values: dict) -> PolicyInferenceConfig | None: for path in paths: if not isinstance(path, str) or not path.lower().endswith(".pt"): continue - artifact = read_execution_artifact(Path(values.get("project_root", ".")) / path) + if path not in artifacts: + artifacts[path] = read_execution_artifact( + Path(values.get("project_root", ".")) / path + ) + artifact = artifacts[path] for name, interface in artifact.model_config.values["interfaces"].items(): if interface.get("policy") is not None and ( selected is None or selected == name @@ -75,4 +84,6 @@ def policy_inference_config(values: dict) -> PolicyInferenceConfig | None: authored = dict(values) authored["model_path"] = paths[0] authored["model_interface"] = policy_routes[0] - return PolicyInferenceConfig.model_validate(authored) + config = PolicyInferenceConfig.model_validate(authored) + config._artifacts = artifacts + return config diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 2118db6f..ae047a8e 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -2488,35 +2488,16 @@ def interface_build_view( seed=config.seed, global_training=config.global_training, training_objective=config.global_training.training_objective, - input_columns=interface.input_columns, static_columns=( interface.static_conditioning.columns if interface.static_conditioning else [] ), - static_features_path=interface.static_features_path, - image_sources=interface.image_sources, - preconditioning_image=interface.preconditioning_image, - preconditioning_image_encoder=interface.preconditioning_image_encoder, - target_columns=interface.target_columns, - categorical_hashing=interface.categorical_hashing, - categorical_hash_contracts=interface.categorical_hash_contracts, - target_column_types=interface.target_column_types, - column_data_types=interface.column_data_types, - categorical_columns=interface.categorical_columns, - real_columns=interface.real_columns, - categorical_decoder_special_tokens=interface.categorical_decoder_special_tokens, - depth_layouts=interface.depth_layouts, - tensor_payload_version=interface.tensor_payload_version, - feature_layout=interface.feature_layout, - n_classes=interface.n_classes, - id_maps=interface.id_maps, - special_token_ids=interface.special_token_ids, - target_decoder_ids=interface.target_decoder_ids, - target_n_classes=interface.target_n_classes, - target_global_to_decoder=interface.target_global_to_decoder, - storage_layout=interface.storage_layout, - window_view=interface.window_view, + **{ + name: getattr(interface, name) + for name in SelectedInterfaceConfig.__dataclass_fields__ + if name in ResolvedModelInterface.model_fields + }, model=SelectedModelComponents( backbone=config.model.backbone, ingestion=interface.ingestion, diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index fb796708..edb16129 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -12,8 +12,8 @@ from beartype.typing import Iterator from loguru import logger +from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.config.infer_config import ResolvedInferenceConfig, load_inferer_config -from sequifier.config.train_config import ResolvedSequifierConfig as TrainModel from sequifier.helpers import ( PANDAS_TO_TORCH_TYPES, configure_determinism, @@ -27,6 +27,7 @@ validate_stored_window_width, write_data, ) +from sequifier.io.preprocessing import denormalize_values from sequifier.objectives import get_objective_class from sequifier.special_tokens import ( ONNX_CATEGORICAL_TARGET_CODECS_KEY, @@ -648,75 +649,21 @@ def infer_worker( value = getattr(config, key, None) if value is not None: route_args[key] = value + artifact = getattr(config, "_artifacts", {}).get(model_path) if model_path.lower().endswith(".pt"): - model_state = torch.load( - normalize_path(model_path, config.project_root), - map_location="cpu", - weights_only=False, - ) - embedded_model_config = model_state.get("model_config") - if embedded_model_config is not None: - interface_name = route_args.get("model_interface") - if ( - interface_name is None - and len(embedded_model_config.get("interfaces", {})) == 1 - ): - interface_name = next(iter(embedded_model_config["interfaces"])) - if interface_name is not None: - interface = embedded_model_config["interfaces"].get(interface_name) - if interface is None: - raise ValueError( - f"Unknown PT model interface {interface_name!r}" - ) - target_decoder_ids = interface.get("target_decoder_ids", {}) - elif model_state.get("training_config") is not None: - training_config = TrainModel.model_validate( - model_state["training_config"] + if artifact is None: + artifact = read_execution_artifact( + normalize_path(model_path, config.project_root) ) - selected_dataset = route_args.get("dataset") - selected_interface = route_args.get("model_interface") - datasets = training_config.dataset_training - if selected_dataset is not None: - if selected_dataset not in datasets: - raise ValueError( - f"Unknown inference dataset {selected_dataset!r}" - ) - dataset_config = datasets[selected_dataset] - if ( - selected_interface is not None - and dataset_config.model_interface != selected_interface - ): - raise ValueError( - f"Dataset {selected_dataset!r} maps to interface " - f"{dataset_config.model_interface!r}, not " - f"{selected_interface!r}" - ) - elif selected_interface is not None: - dataset_config = next( - ( - dataset - for dataset in datasets.values() - if dataset.model_interface == selected_interface - ), - None, - ) - if dataset_config is None: - raise ValueError( - "No execution route for model interface " - f"{selected_interface!r}" - ) - elif len(datasets) == 1: - dataset_config = next(iter(datasets.values())) - else: - raise ValueError( - "A dataset or model_interface selection is required for " - "multi-dataset checkpoint inference" - ) - target_decoder_ids = dict(dataset_config.interface.target_decoder_ids) - else: - raise ValueError( - "PyTorch artifact has neither model_config nor a resolved " - "training_config." + interfaces = artifact.model_config.values["interfaces"] + interface_name = route_args.get("model_interface") + if interface_name is None and len(interfaces) == 1: + interface_name = next(iter(interfaces)) + if interface_name is not None: + if interface_name not in interfaces: + raise ValueError(f"Unknown PT model interface {interface_name!r}") + target_decoder_ids = interfaces[interface_name].get( + "target_decoder_ids", {} ) if is_folder_input: @@ -770,6 +717,7 @@ def infer_worker( training_objective=config.training_objective, normalize_real_columns=normalize_real_columns, target_decoder_ids=target_decoder_ids, + artifact=artifact, ) column_data_types = _torch_column_types(config) @@ -1578,6 +1526,7 @@ def __init__( training_objective: Optional[str] = None, normalize_real_columns: bool = True, target_decoder_ids: Optional[dict[str, list[int]]] = None, + artifact: Any = None, ): """Load a PT or ONNX backend and postprocessing state.""" self.model_type = model_type @@ -1694,6 +1643,7 @@ def __init__( self.args_config, self.device, self.infer_with_dropout, + artifact=artifact, ) route_model = getattr( self.inference_model, @@ -1733,9 +1683,9 @@ def invert_normalization( """Invert target-column Z-score normalization.""" if not self.normalize_real_columns: return values - std = self.selected_columns_statistics[target_column]["std"] - mean = self.selected_columns_statistics[target_column]["mean"] - return (values * (std + 1e-9)) + mean + return denormalize_values( + values, self.selected_columns_statistics[target_column] + ) @beartype def _exclude_mask_token( diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index c4af613b..c9c7137d 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -191,6 +191,15 @@ def __init__( def __len__(self): return self.cumulative_lengths[-1] if self.cumulative_lengths else 0 + @property + def data_contract(self): + return hashlib.sha256( + json.dumps([self.content_keys, self.episode_ids]).encode() + ).hexdigest() + + def sampler(self, generator): + return EpisodeWindowSampler(self, generator) + def fit_categorical_config(self) -> CategoricalPolicyConfig: """Fit only declared raw categorical features, using the training episodes.""" values = self.config.model_dump() diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py index 976ab3bf..f3796791 100644 --- a/src/sequifier/io/policy_window.py +++ b/src/sequifier/io/policy_window.py @@ -7,6 +7,7 @@ from sequifier.config.policy_config import CategoricalPolicyConfig from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view from sequifier.io.batch import SequifierInputs +from sequifier.io.preprocessing import normalize_values from sequifier.io.values import domain_fields, domain_scalar, numeric_scalar from sequifier.special_tokens import SPECIAL_TOKEN_IDS @@ -19,9 +20,9 @@ def normalize_policy_features(features, metadata, interface): features = dict(features) mask = metadata["attention_valid_mask"] for column, statistics in interface.selected_columns_statistics.items(): - features[column] = ( - (features[column] - statistics["mean"]) / (statistics["std"] + 1e-9) - ).masked_fill(~mask, 0.0) + features[column] = normalize_values(features[column], statistics).masked_fill( + ~mask, 0.0 + ) if interface.policy.return_conditioned: features[DESIRED_RETURN] = ( features[DESIRED_RETURN] / interface.policy.returns.scale diff --git a/src/sequifier/io/preprocessing.py b/src/sequifier/io/preprocessing.py index 39099a8e..7fa5c0c8 100644 --- a/src/sequifier/io/preprocessing.py +++ b/src/sequifier/io/preprocessing.py @@ -2,6 +2,7 @@ import math import warnings +from collections.abc import Mapping from typing import Union import numpy as np @@ -119,3 +120,36 @@ def create_id_map(data: pl.DataFrame, column: str) -> dict[Union[str, int], int] else: id_map = {id_: i + SPECIAL_TOKEN_IDS.user_start for i, id_ in enumerate(ids)} return dict(id_map) + + +NORMALIZATION_EPSILON = 1e-9 + + +def normalization_statistics(statistics: Mapping) -> dict[str, dict[str, float]]: + """Validate persisted statistics once, independently of the array backend.""" + result = {} + for column, values in statistics.items(): + mean, std = values.get("mean"), values.get("std") + if ( + isinstance(mean, bool) + or isinstance(std, bool) + or not isinstance(mean, (int, float)) + or not isinstance(std, (int, float)) + or not math.isfinite(mean) + or not math.isfinite(std) + or std < 0 + ): + raise ValueError( + f"{column}: normalization requires finite means and nonnegative scales" + ) + result[column] = {"mean": float(mean), "std": float(std)} + return result + + +def normalize_values(values, statistics): + """Standardize scalar, NumPy, Torch, or Polars values with fixed statistics.""" + return (values - statistics["mean"]) / (statistics["std"] + NORMALIZATION_EPSILON) + + +def denormalize_values(values, statistics): + return values * (statistics["std"] + NORMALIZATION_EPSILON) + statistics["mean"] diff --git a/src/sequifier/preprocess.py b/src/sequifier/preprocess.py index 2a994571..d334016e 100644 --- a/src/sequifier/preprocess.py +++ b/src/sequifier/preprocess.py @@ -27,20 +27,30 @@ load_preprocessor_config, ) from sequifier.helpers import ( + FLOAT_TYPE_ORDER, PANDAS_TO_TORCH_TYPES, StoredWindowLayout, +) +from sequifier.helpers import _highest_ranked_type as _highest_ranked_type +from sequifier.helpers import _resolve_integer_sequence_type +from sequifier.helpers import ( + _smallest_float_covering_integer_range as _smallest_float_covering_integer_range, +) +from sequifier.helpers import ( assign_sequence_to_split, canonicalize_polars_dtype_name, is_float_dtype_name, is_integer_dtype_name, polars_dtype_from_name, read_data, + resolve_unified_polars_numeric_dtype, ) from sequifier.io.images import _time_ms, validate_sources from sequifier.io.preprocessing import ( _finite_mean_and_std, create_id_map, get_combined_statistics, + normalize_values, ) from sequifier.io.pt_payload import StoredTensorBatch, load_pt_payload, save_pt_payload from sequifier.io.sample_order import ( @@ -68,7 +78,6 @@ STABLE_HASH_PRIME = 1099511628211 UINT64_MASK = (1 << 64) - 1 -FLOAT_TYPE_ORDER = ("Float16", "Float32", "Float64") INTEGER_TYPE_ORDER = ( "Int8", "UInt8", @@ -79,26 +88,7 @@ "Int64", "UInt64", ) -INTEGER_TYPE_INFO = { - "Int8": np.iinfo(np.int8), - "Int16": np.iinfo(np.int16), - "Int32": np.iinfo(np.int32), - "Int64": np.iinfo(np.int64), - "UInt8": np.iinfo(np.uint8), - "UInt16": np.iinfo(np.uint16), - "UInt32": np.iinfo(np.uint32), - "UInt64": np.iinfo(np.uint64), -} -FLOAT_TYPE_INFO = { - "Float16": np.finfo(np.float16), - "Float32": np.finfo(np.float32), - "Float64": np.finfo(np.float64), -} -FLOAT_EXACT_INTEGER_LIMITS = { - "Float16": 2**11, - "Float32": 2**24, - "Float64": 2**53, -} + INT64_INFO = np.iinfo(np.int64) @@ -674,98 +664,36 @@ def _apply_output_type_casting( return data.with_columns(casts) -@beartype -def _highest_ranked_type(types: list[str], order: tuple[str, ...]) -> str: - return max(types, key=lambda type_: order.index(type_)) - - -@beartype -def _smallest_float_covering_integer_range(integer_type: str) -> str: - integer_info = INTEGER_TYPE_INFO[integer_type] - largest_magnitude = max(abs(int(integer_info.min)), int(integer_info.max)) - for float_type in FLOAT_TYPE_ORDER: - if ( - largest_magnitude <= float(FLOAT_TYPE_INFO[float_type].max) - and largest_magnitude <= FLOAT_EXACT_INTEGER_LIMITS[float_type] - ): - return float_type - return "Float64" - - -@beartype -def _resolve_integer_sequence_type(integer_types: list[str]) -> Any: - if not integer_types: - raise ValueError("Cannot resolve an integer sequence type without integers") - - min_value = min(int(INTEGER_TYPE_INFO[type_].min) for type_ in integer_types) - max_value = max(int(INTEGER_TYPE_INFO[type_].max) for type_ in integer_types) - - if min_value >= 0 and all(type_.startswith("UInt") for type_ in integer_types): - for dtype_name in ("UInt8", "UInt16", "UInt32", "UInt64"): - info = INTEGER_TYPE_INFO[dtype_name] - if max_value <= int(info.max): - return polars_dtype_from_name(dtype_name) - - for dtype_name in ("Int8", "Int16", "Int32", "Int64"): - info = INTEGER_TYPE_INFO[dtype_name] - if min_value >= int(info.min) and max_value <= int(info.max): - return polars_dtype_from_name(dtype_name) - - raise ValueError(f"Cannot resolve a safe integer dtype for {integer_types}") - - @beartype def _resolve_unified_parquet_type(column_data_types: dict[str, str]) -> Any: - if not column_data_types: - raise ValueError("column_data_types cannot be empty") - - normalized_types = [ - ( - "Int64" - if _is_temporal_dtype_name(type_) - else canonicalize_polars_dtype_name(type_) - ) - for type_ in column_data_types.values() - ] - float_types = [type_ for type_ in normalized_types if is_float_dtype_name(type_)] - integer_types = [ - type_ for type_ in normalized_types if is_integer_dtype_name(type_) - ] - - if not float_types: - if len(set(integer_types)) > 1: + normalized = { + column: "Int64" + if _is_temporal_dtype_name(dtype) + else canonicalize_polars_dtype_name(dtype) + for column, dtype in column_data_types.items() + } + resolved = resolve_unified_polars_numeric_dtype(normalized) + types = set(normalized.values()) + floats = [dtype for dtype in types if is_float_dtype_name(dtype)] + if not floats: + if len(types) > 1: logger.warning( "Multiple integer column_data_types were specified for Parquet output; " "using a unified integer schema." ) - return _resolve_integer_sequence_type(integer_types) - - resolved_float = _highest_ranked_type(float_types, FLOAT_TYPE_ORDER) - if len(set(normalized_types)) > 1: - logger.warning( - "Multiple column_data_types were specified for Parquet output; " - f"using unified sequence dtype {resolved_float}." - ) - - if integer_types: - required_float = _highest_ranked_type( - [ - _smallest_float_covering_integer_range(integer_type) - for integer_type in integer_types - ], - FLOAT_TYPE_ORDER, - ) - if FLOAT_TYPE_ORDER.index(required_float) > FLOAT_TYPE_ORDER.index( - resolved_float - ): + else: + requested = max(floats, key=FLOAT_TYPE_ORDER.index) + if len(types) > 1: + logger.warning( + "Multiple column_data_types were specified for Parquet output; " + f"using unified sequence dtype {requested}." + ) + if str(resolved) != requested: logger.warning( "An integer column_type has a range exceeding " - f"{resolved_float}; upgrading unified Parquet sequence dtype " - f"to {required_float}." + f"{requested}; upgrading unified Parquet sequence dtype to {resolved}." ) - resolved_float = required_float - - return polars_dtype_from_name(resolved_float) + return resolved @beartype @@ -3610,9 +3538,8 @@ def _apply_column_statistics( col_types[col] = "Float64" if normalize_real_columns: expressions.append( - ( - (pl.col(col) - selected_columns_statistics[col]["mean"]) - / (selected_columns_statistics[col]["std"] + 1e-9) + normalize_values( + pl.col(col), selected_columns_statistics[col] ).alias(col) ) diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index b8304048..01a3a4b8 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -5,6 +5,7 @@ import torch from sequifier.artifacts.model_artifact import ( + ModelArtifact, build_model_artifact, load_execution_model, save_model_artifact, @@ -39,7 +40,10 @@ def save_policy( def load_policy( - path: str | Path, *, device: str = "cpu", interface_name: str | None = None + path: str | Path | ModelArtifact, + *, + device: str = "cpu", + interface_name: str | None = None, ) -> CategoricalPolicy: network, config, _ = load_execution_model( path, device=device, interface_name=interface_name diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index aa721937..c55116a2 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -26,7 +26,8 @@ def infer_policy(config: PolicyInferenceConfig) -> None: configure_determinism(config.seed, config.deterministic) policy = load_policy( - normalize_path(config.model_path, config.project_root), + config._artifacts.get(config.model_path) + or normalize_path(config.model_path, config.project_root), device=config.device, interface_name=config.model_interface, ) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 0426921e..0160829b 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -1,13 +1,14 @@ """Policy alignment adapter over Sequifier's configurable model system.""" -import math from collections.abc import Mapping import torch from torch import Tensor, nn -from sequifier.artifacts.model_config import resolved_config_from_model_config -from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.artifacts.model_config import execution_config +from sequifier.config.metadata import DatasetMetadata +from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig +from sequifier.config.train_config import GlobalTrainingSpecModel, _resolve_interface from sequifier.io.batch import SequifierInputs from sequifier.io.policy_window import ( DESIRED_RETURN, @@ -16,18 +17,22 @@ action_target, normalize_policy_features, ) +from sequifier.io.preprocessing import normalization_statistics from sequifier.model.execution_schema import ExecutionSchema from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.special_tokens import SPECIAL_TOKEN_IDS, SPECIAL_TOKEN_NAMES def policy_execution_config( - config: CategoricalPolicyConfig, model, device: str = "cpu" + config: CategoricalPolicyConfig, + model, + device: str = "cpu", + *, + global_training=None, + statistics=None, + **run_options, ): - """Resolve policy features through the same execution schema as forecasts. - - Actions are decoder IDs; the extra input-only ID is the episode-start token. - Returns and previous actions remain distinct from the current action target. - """ + """Resolve policy alignment directly into the ordinary execution contract.""" columns = config.observation_columns + config.static_columns + [PREVIOUS_ACTION] if config.return_conditioned: columns.append(DESIRED_RETURN) @@ -36,54 +41,77 @@ def policy_execution_config( real = [column for column in columns if column not in categorical] decoder_ids = action_global_ids(config) action_width = decoder_ids[-1] + 1 - lookup = [-1] * action_width - for local_id, global_id in enumerate(decoder_ids): - lookup[global_id] = local_id - n_classes = { - **config.categorical_n_classes, - PREVIOUS_ACTION: action_width + 1, - target: action_width, - } - interface = model.interfaces[config.interface_name] - values = { - "training_objective": "policy", - "context_length": config.context_length, - "target_offset": 0, - "backbone": model.backbone.model_dump(mode="json"), - "interfaces": { - config.interface_name: { - "policy": config.model_dump(mode="json"), - "input_columns": columns, - "target_columns": [target], - "column_data_types": { - column: "Int64" if column in categorical else "Float32" - for column in columns + [target] - }, - "categorical_columns": categorical, - "real_columns": real, - "target_column_types": {target: "categorical"}, - "n_classes": n_classes, - "target_decoder_ids": {target: decoder_ids}, - "target_global_to_decoder": {target: lookup}, - "id_maps": { - **config.id_maps, - target: { - str(value): global_id - for value, global_id in zip(config.action_values, decoder_ids) - }, - }, - "normalize_real_columns": True, - "selected_columns_statistics": { - column: {"mean": 0.0, "std": 1.0} - for column in real - if column != DESIRED_RETURN - }, - "ingestion": interface.ingestion.model_dump(mode="json"), - "decoder": interface.decoder.model_dump(mode="json"), - } + global_training = global_training or GlobalTrainingSpecModel( + training_objective="policy", + context_length=config.context_length, + target_offset=0, + inference_batch_size=1, + batch_size=1, + learning_rate=1e-3, + torch_compile="none", + ) + interface = model.interfaces[config.interface_name].model_copy( + update={ + "input_columns": columns, + "target_columns": [target], + "policy": PolicySchemaConfig.model_validate( + config.model_dump(include=set(PolicySchemaConfig.model_fields)) + ), + } + ) + metadata = DatasetMetadata( + window_length=config.context_length + 1, + column_data_types={ + column: "Int64" if column in categorical else "Float32" + for column in columns + [target] + }, + n_classes={ + **config.categorical_n_classes, + PREVIOUS_ACTION: action_width + 1, + target: action_width, + }, + id_maps={ + **config.id_maps, + target: { + str(value): global_id + for value, global_id in zip(config.action_values, decoder_ids) + }, }, - } - return resolved_config_from_model_config(values, device=device)[0] + selected_columns_statistics=statistics + if statistics is not None + else { + column: {"mean": 0.0, "std": 1.0} + for column in real + if column != DESIRED_RETURN + }, + ) + # Legacy action IDs can occupy reserved-token slots. Resolve the same IDs + # through the ordinary codec, then retain the persisted policy contract. + spec = interface.model_copy( + update={ + "categorical_decoder_special_tokens": { + target: [ + name + for name in SPECIAL_TOKEN_NAMES + if getattr(SPECIAL_TOKEN_IDS, name) in decoder_ids + ] + } + } + ) + resolved = _resolve_interface( + config.interface_name, spec, metadata, global_training, metadata.storage_layout + ) + resolved.policy = config + resolved.categorical_columns = categorical + resolved.categorical_decoder_special_tokens = {} + resolved.selected_columns_statistics = metadata.selected_columns_statistics + return execution_config( + model.model_copy(update={"interfaces": {config.interface_name: interface}}), + {config.interface_name: resolved}, + global_training, + device=device, + **run_options, + ) class CategoricalPolicy(nn.Module): @@ -124,15 +152,12 @@ def set_normalization_statistics( ) if set(statistics) != expected: raise ValueError("normalization statistics must match real policy columns") - fitted = {} - for column, values in statistics.items(): - mean, std = float(values["mean"]), float(values["std"]) - if not math.isfinite(mean) or not math.isfinite(std) or std < 0: - raise ValueError( - "policy normalization requires finite means and nonnegative scales" - ) - fitted[column] = {"mean": mean, "std": std} - self.interface.selected_columns_statistics = fitted + self.interface.selected_columns_statistics = normalization_statistics( + { + column: {key: float(values[key]) for key in ("mean", "std")} + for column, values in statistics.items() + } + ) def model_inputs( self, inputs: SequifierInputs diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index af2ba556..dc6b925d 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -1,12 +1,8 @@ """Policy datasets adapted to Sequifier's ordinary TrainingRun and TrainingEngine.""" -import hashlib -import json from dataclasses import dataclass -from typing import Literal import torch -from torch.utils.data import DataLoader from sequifier.artifacts.model_artifact import load_execution_model from sequifier.config.train_config import ( @@ -16,8 +12,7 @@ ResolvedTrainingSource, ) from sequifier.integration.callbacks import IntegrationManager -from sequifier.io.batch import collate_sequence_batches -from sequifier.io.policy_dataset import EpisodeWindowSampler, PolicySequenceDataset +from sequifier.io.policy_dataset import PolicySequenceDataset from sequifier.rl.policy import CategoricalPolicy, policy_execution_config from sequifier.runtime.builder import RunBuilder, TrainingRun from sequifier.runtime.context import ExecutionEnvironment @@ -56,58 +51,43 @@ def train_policy( fitted = training.fit_categorical_config() for dataset in (training, validation): dataset.use_config(fitted) - config = policy_execution_config(fitted, run_config.model, run_config.device) - config.project_root, config.model_name, config.seed = ( - run_config.project_root, - run_config.model_name, - run_config.seed, - ) - config.global_training = run_config.global_training name, part_name = fitted.interface_name, "episodes" - interface = config.interface(name) - interface.selected_columns_statistics = training.normalization_statistics() source = ResolvedTrainingSource( source=f"{name}.{part_name}", dataset=name, part=part_name ) - config.training_plan = [ - ResolvedTrainingPhase( - name=phase.name, - algorithm="return_conditioned", - epochs=phase.epochs, - mode="sequential", - sources=[source], - ) - ] - config.evaluation_sources = [source] - config.evaluation_monitor = EvaluationMonitorSpecModel( - source=source.source, metric="loss", mode="min" + config = policy_execution_config( + fitted, + run_config.model, + run_config.device, + global_training=run_config.global_training, + statistics=training.normalization_statistics(), + project_root=run_config.project_root, + model_name=run_config.model_name, + seed=run_config.seed, + training_plan=[ + ResolvedTrainingPhase( + name=phase.name, + algorithm="return_conditioned", + epochs=phase.epochs, + mode="sequential", + sources=[source], + ) + ], + evaluation_sources=[source], + evaluation_monitor=EvaluationMonitorSpecModel( + source=source.source, metric="loss", mode="min" + ), ) def configure(run: TrainingRun, resuming: bool) -> None: - factory = PartLoaderFactory(config, name, part_name) - contract = {} - partitions: dict[Literal["training", "validation"], PolicySequenceDataset] = { - "training": training, - "validation": validation, - } - for split, dataset in partitions.items(): - generator = torch.Generator().manual_seed(config.seed) - factory.generators[split] = generator - factory.loaders[split] = DataLoader( - dataset, - batch_size=config.global_training.batch_size, - sampler=EpisodeWindowSampler(dataset, generator) - if split == "training" - else None, - num_workers=config.global_training.num_workers, - generator=generator, - collate_fn=collate_sequence_batches, - ) - contract[split] = hashlib.sha256( - json.dumps([dataset.content_keys, dataset.episode_ids]).encode() - ).hexdigest() run.datasets.resolve(name).parts[part_name] = DatasetPartRuntime( - part_name, factory, data_contract=contract + part_name, + PartLoaderFactory( + config, + name, + part_name, + {"training": training, "validation": validation}, + ), ) if not resuming and run_config.initialize_from is not None: source_network, source_config, _ = load_execution_model( diff --git a/src/sequifier/rollout/preprocessing.py b/src/sequifier/rollout/preprocessing.py index e241c8b2..44957560 100644 --- a/src/sequifier/rollout/preprocessing.py +++ b/src/sequifier/rollout/preprocessing.py @@ -1,9 +1,13 @@ """Fixed numeric training transforms; never fit on seeds or adapter values.""" -import math - import numpy as np +from sequifier.io.preprocessing import ( + NORMALIZATION_EPSILON, + denormalize_values, + normalization_statistics, + normalize_values, +) from sequifier.rollout.history import validate_seed from sequifier.trajectory.schema import SeedHistory @@ -11,35 +15,24 @@ class FixedNumericPreprocessing: def __init__(self, columns, statistics, normalize, dtypes): self.normalize = normalize - self.statistics = {} + self.statistics = ( + normalization_statistics( + {column: statistics.get(column, {}) for column in columns} + ) + if normalize + else {} + ) self.dtypes = {column: np.dtype(dtypes[column]) for column in columns} for column in columns: if self.dtypes[column].kind != "f": raise ValueError( f"rollout field {column!r} must have a floating numeric dtype" ) - if normalize: - stats = statistics.get(column, {}) - mean, std = stats.get("mean"), stats.get("std") - if ( - isinstance(mean, bool) - or isinstance(std, bool) - or not isinstance(mean, (int, float)) - or not isinstance(std, (int, float)) - or not math.isfinite(mean) - or not math.isfinite(std) - or std < 0 - ): - raise ValueError( - f"rollout requires finite training mean/std for {column!r}" - ) - self.statistics[column] = {"mean": float(mean), "std": float(std)} def decode(self, column, values): values = np.asarray(values, dtype=np.float64) if self.normalize: - stats = self.statistics[column] - values = values * (stats["std"] + 1e-9) + stats["mean"] + values = denormalize_values(values, self.statistics[column]) if not np.isfinite(values).all(): raise ValueError(f"rollout field {column!r} decoded to non-finite values") return values @@ -47,8 +40,7 @@ def decode(self, column, values): def encode(self, column, values): values = np.asarray(values, dtype=np.float64) if self.normalize: - stats = self.statistics[column] - values = (values - stats["mean"]) / (stats["std"] + 1e-9) + values = normalize_values(values, self.statistics[column]) with np.errstate(over="ignore", invalid="ignore"): values = values.astype(self.dtypes[column]) if not np.isfinite(values).all(): @@ -110,5 +102,5 @@ def to_dict(self): "model_dtypes": { column: dtype.name for column, dtype in self.dtypes.items() }, - "epsilon": 1e-9, + "epsilon": NORMALIZATION_EPSILON, } diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 27214643..10f4fd8c 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -15,7 +15,10 @@ from loguru import logger as loguru_logger # noqa: E402 from torch import Tensor, nn # noqa: E402 -from sequifier.artifacts.model_artifact import load_execution_model # noqa: E402 +from sequifier.artifacts.model_artifact import ( # noqa: E402 + ModelArtifact, + load_execution_model, +) from sequifier.config.train_config import PolicyTrainConfig # noqa: E402 from sequifier.config.train_config import load_train_config # noqa: E402 from sequifier.config.train_config import ( # noqa: E402 @@ -236,13 +239,16 @@ def load_inference_model( args_config: dict[str, Any], device: str, infer_with_dropout: bool, + artifact: ModelArtifact | None = None, ) -> LoadedInferenceModel: """Load only the new portable-model or exact-run artifact formats.""" del training_config_path interface_name = args_config.get("model_interface") or args_config.get("dataset") network, config, _ = load_execution_model( - model_path, device=device, interface_name=interface_name + artifact if artifact is not None else model_path, + device=device, + interface_name=interface_name, ) if interface_name is None: if len(network.interfaces) != 1: diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 0545540a..b88fb9bc 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -294,7 +294,7 @@ def flush_metrics( if window is None: return dataset = run.datasets.resolve(dataset_name) - total, targets = run.loss.finalize_accounting( + total, targets = run.loss.finalize_reduced_accounting( window["sums"], window["count"], dataset, allow_empty=True ) run.metrics.record_training( @@ -525,21 +525,17 @@ def flush_metrics( { "sums": { name: value.new_zeros(()) - for name, value in loss.accounting_sums.items() + for name, value in sums.items() }, - "count": loss.accounting_count.new_zeros(()).to( - next(iter(loss.accounting_sums.values())).dtype - ), + "count": count_tensor.new_zeros(()), "batches": 0, "seconds": 0.0, "part": runtime_batch.part, }, ) - for name, value in loss.accounting_sums.items(): + for name, value in sums.items(): window["sums"][name] += value - window["count"] += loss.accounting_count.to( - window["count"].dtype - ) + window["count"] += count_tensor window["batches"] += 1 window["seconds"] += time.perf_counter() - started if window["part"] != runtime_batch.part: diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 5ca78c67..ec11891e 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -22,7 +22,7 @@ dataset_part_view, ) from sequifier.helpers import normalize_path -from sequifier.io.batch import SequifierBatch +from sequifier.io.batch import SequifierBatch, collate_sequence_batches from sequifier.io.sequifier_dataset_from_file import SequifierDatasetFromFile from sequifier.io.sequifier_dataset_from_folder_parquet import ( SequifierDatasetFromFolderParquet, @@ -66,6 +66,7 @@ class PartLoaderFactory: config: ResolvedSequifierConfig dataset_name: str part_name: str + dataset_splits: dict[str, Any] = field(default_factory=dict) loaders: dict[Literal["training", "validation"], DataLoader] = field( default_factory=dict, init=False, @@ -81,67 +82,75 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: if split in self.loaders: return self.loaders[split] - dataset_config: Any = dataset_part_view( - self.config, self.dataset_name, self.part_name - ) - if split != "training" or not _training_plan_uses_part( - self.config, self.dataset_name, self.part_name - ): - dataset_config = replace( - dataset_config, - curriculum_training=False, - curriculum_column=None, - ) - part = self.config.dataset_training[self.dataset_name].parts[self.part_name] - path = ( - part.training_data_path - if split == "training" - else part.validation_data_path - ) - if path is None: - raise ValueError( - f"Part {self.dataset_name}.{self.part_name} has no {split} path" + sequence_dataset = bool(self.dataset_splits) + if sequence_dataset: + dataset = self.dataset_splits[split] + else: + dataset_config: Any = dataset_part_view( + self.config, self.dataset_name, self.part_name ) - shuffle = split == "training" - if part.storage_form == "file": - if global_spec.distributed: - raise ValueError( - "Distributed training is not supported with single-file parts" - ) - split_paths = [ - normalize_path(split_path, self.config.project_root) - for split_path in part.metadata.split_paths - ] - if path in split_paths: + if split != "training" or not _training_plan_uses_part( + self.config, self.dataset_name, self.part_name + ): dataset_config = replace( dataset_config, - storage_layout=part.metadata.layout_for_split( - split_paths.index(path) - ), + curriculum_training=False, + curriculum_column=None, ) - dataset = SequifierDatasetFromFile(path, dataset_config, shuffle=shuffle) - elif global_spec.read_format == "pt": - cls = ( - SequifierDatasetFromFolderPt - if global_spec.load_full_data_to_ram - else SequifierDatasetFromFolderPtLazy - ) - dataset = cls(path, dataset_config, shuffle=shuffle) - elif global_spec.read_format == "parquet": - cls = ( - SequifierDatasetFromFolderParquet - if global_spec.load_full_data_to_ram - else SequifierDatasetFromFolderParquetLazy - ) - dataset = cls(path, dataset_config, shuffle=shuffle) - else: - raise ValueError( - f"Folder parts do not support read_format={global_spec.read_format!r}" + part = self.config.dataset_training[self.dataset_name].parts[self.part_name] + path = ( + part.training_data_path + if split == "training" + else part.validation_data_path ) + if path is None: + raise ValueError( + f"Part {self.dataset_name}.{self.part_name} has no {split} path" + ) + shuffle = split == "training" + if part.storage_form == "file": + if global_spec.distributed: + raise ValueError( + "Distributed training is not supported with single-file parts" + ) + split_paths = [ + normalize_path(split_path, self.config.project_root) + for split_path in part.metadata.split_paths + ] + if path in split_paths: + dataset_config = replace( + dataset_config, + storage_layout=part.metadata.layout_for_split( + split_paths.index(path) + ), + ) + dataset = SequifierDatasetFromFile( + path, dataset_config, shuffle=shuffle + ) + elif global_spec.read_format == "pt": + cls = ( + SequifierDatasetFromFolderPt + if global_spec.load_full_data_to_ram + else SequifierDatasetFromFolderPtLazy + ) + dataset = cls(path, dataset_config, shuffle=shuffle) + elif global_spec.read_format == "parquet": + cls = ( + SequifierDatasetFromFolderParquet + if global_spec.load_full_data_to_ram + else SequifierDatasetFromFolderParquetLazy + ) + dataset = cls(path, dataset_config, shuffle=shuffle) + else: + raise ValueError( + f"Folder parts do not support read_format={global_spec.read_format!r}" + ) generator = self.generators.get(split) if generator is None: generator = torch.Generator().manual_seed( self.config.seed + if sequence_dataset + else self.config.seed + (10_001 if split == "training" else 10_002) + list(self.config.dataset_training).index(self.dataset_name) * 101 + list(self.config.dataset_training[self.dataset_name].parts).index( @@ -151,12 +160,18 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: self.generators[split] = generator loader = DataLoader( dataset, - batch_size=None, - sampler=None, + batch_size=global_spec.batch_size if sequence_dataset else None, + sampler=dataset.sampler(generator) + if sequence_dataset and split == "training" + else None, + collate_fn=collate_sequence_batches if sequence_dataset else None, num_workers=global_spec.num_workers, - pin_memory=self.config.device not in {"mps", "cpu"}, - prefetch_factor=4 if global_spec.num_workers > 0 else None, - persistent_workers=global_spec.num_workers > 0, + pin_memory=not sequence_dataset + and self.config.device not in {"mps", "cpu"}, + prefetch_factor=4 + if global_spec.num_workers > 0 and not sequence_dataset + else None, + persistent_workers=global_spec.num_workers > 0 and not sequence_dataset, generator=generator, ) self.loaders[split] = loader @@ -186,6 +201,11 @@ def loader_generators( def source_metadata(self) -> dict[str, Any]: if self.data_contract is not None: return self.data_contract + if self.factory.dataset_splits: + return { + split: dataset.data_contract + for split, dataset in self.factory.dataset_splits.items() + } part = self.factory.config.dataset_training[self.factory.dataset_name].parts[ self.name ] diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 8a122010..30d4bd28 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -17,6 +17,7 @@ PolicyTableColumns, SequenceSourceConfig, ) +from sequifier.io.preprocessing import denormalize_values from sequifier.io.sequence import ( SequenceEpisode, TransitionColumns, @@ -274,10 +275,9 @@ def decode(value, column): metadata.normalize_real_columns and column in metadata.selected_columns_statistics ): - stats = metadata.selected_columns_statistics[column] - value = ( - numeric_scalar(value, column) * (stats["std"] + 1e-9) - + stats["mean"] + value = denormalize_values( + numeric_scalar(value, column), + metadata.selected_columns_statistics[column], ) return value diff --git a/tests/test_consolidation.py b/tests/test_consolidation.py new file mode 100644 index 00000000..8e9129f2 --- /dev/null +++ b/tests/test_consolidation.py @@ -0,0 +1,269 @@ +"""Behavioral checks for shared policy execution, loading, and preprocessing.""" + +from copy import deepcopy + +import numpy as np +import polars as pl +import pytest +import torch +import yaml + +from sequifier.artifacts.model_artifact import build_model_artifact, save_model_artifact +from sequifier.artifacts.model_config import resolved_config_from_model_config +from sequifier.artifacts.model_export import model_execution_config +from sequifier.config.infer_config import ResolvedInferenceConfig, load_inferer_config +from sequifier.config.policy_infer_config import PolicyInferenceConfig +from sequifier.config.train_config import PolicyTrainConfig +from sequifier.infer import infer_worker +from sequifier.io.policy_dataset import PolicySequenceDataset +from sequifier.io.policy_window import PolicyWindowView +from sequifier.io.preprocessing import denormalize_values, normalize_values +from sequifier.io.pt_payload import StoredTensorBatch, save_pt_payload +from sequifier.io.sequence import SequenceEpisode, TransitionColumns +from sequifier.make import policy_train_config_string +from sequifier.model.factory import build_transformer_network +from sequifier.rl.checkpoint import load_policy, save_policy +from sequifier.rl.inference import infer_policy +from sequifier.rl.policy import CategoricalPolicy, policy_execution_config +from sequifier.rl.training import train_policy +from sequifier.rollout.preprocessing import FixedNumericPreprocessing + + +def config_at(root, epochs=2): + values = yaml.safe_load(policy_train_config_string) + values["project_root"] = str(root) + values["global_training"].update(batch_size=2, log_interval=2) + values["training_plan"]["phases"][0]["epochs"] = epochs + return PolicyTrainConfig.model_validate(values) + + +def episodes(config, prefix): + return PolicySequenceDataset( + [ + SequenceEpisode( + episode_id=f"{prefix}-{index}", + item_positions=(0, 1, 2), + observations={"OBSERVATION_COLUMN": (1.0 + index, 2.0, 3.0)}, + actions=(0.0, 1.0, 0.0), + transitions=TransitionColumns( + rewards=(1.0, 1.0, 1.0), terminated=(False, False, True) + ), + ) + for index in range(2) + ], + config.policy, + ) + + +@pytest.mark.parametrize("encoding", ["global", "legacy"]) +@pytest.mark.parametrize("conditioned", [True, False]) +@pytest.mark.parametrize("collision", [True, False]) +def test_policy_artifact_round_trip_keeps_predictions( + tmp_path, encoding, conditioned, collision +): + authored = config_at(tmp_path) + schema = authored.policy.model_copy( + update={ + "categorical_encoding": encoding, + "return_conditioned": conditioned, + "action_column": "OBSERVATION_COLUMN" if collision else "action", + } + ) + model = authored.model.model_copy(deep=True) + if not conditioned: + ingestion = model.interfaces["actor"].ingestion + assert ingestion.type == "embedding" + dims = ingestion.feature_embedding_dims + assert dims is not None + dims.pop("__policy_return") + dims["__policy_previous_action"] += 1 + config = policy_execution_config(schema, model) + network = build_transformer_network(config, device=torch.device("cpu")).network + policy = CategoricalPolicy.from_network(network, config, "actor").eval() + inputs = PolicyWindowView(schema).inputs( + [{"OBSERVATION_COLUMN": 2.0}], [2], [3.0] if conditioned else None, {} + ) + exported = model_execution_config(config) + restored, _ = resolved_config_from_model_config(exported, device="cpu") + assert model_execution_config(restored) == exported + path = tmp_path / "policy.pt" + save_policy(policy, path) + with torch.no_grad(): + torch.testing.assert_close( + load_policy(path)(inputs), policy(inputs), rtol=0, atol=0 + ) + + +@pytest.mark.parametrize("boundary", ["epoch", "batch"]) +def test_policy_resume_matches_uninterrupted_training(tmp_path, boundary): + threads = torch.get_num_threads() + torch.set_num_threads(1) + try: + complete = config_at(tmp_path / "complete") + complete.global_training.save_interval_batches = 1 + complete.global_training.save_interval_val_loss = False + expected = train_policy( + episodes(complete, "train"), episodes(complete, "val"), complete + ) + if boundary == "epoch": + partial = config_at(tmp_path / "resumed", epochs=1) + train_policy(episodes(partial, "train"), episodes(partial, "val"), partial) + values = deepcopy(partial.model_dump()) + values["global_training"]["resume"] = {"policy": "if_exists"} + else: + values = deepcopy(complete.model_dump()) + values["project_root"] = str(tmp_path / "resumed") + checkpoint = next((tmp_path / "complete").rglob("*-epoch-1-batch-1.pt")) + values["global_training"]["resume"] = { + "policy": "required", + "checkpoint_path": str(checkpoint), + } + values["training_plan"]["phases"][0]["epochs"] = 2 + resumed = PolicyTrainConfig.model_validate(values) + actual = train_policy( + episodes(resumed, "train"), episodes(resumed, "val"), resumed + ) + assert actual.metrics == expected.metrics + assert actual.optimizer_steps == expected.optimizer_steps + assert actual.best_epoch == expected.best_epoch + for name, tensor in expected.policy.network.state_dict().items(): + torch.testing.assert_close( + actual.policy.network.state_dict()[name], tensor, rtol=0, atol=0 + ) + finally: + torch.set_num_threads(threads) + + +@pytest.mark.parametrize("checkpoint", [True, False]) +def test_policy_inference_reads_artifact_once(tmp_path, monkeypatch, checkpoint): + authored = config_at(tmp_path) + config = policy_execution_config(authored.policy, authored.model) + network = build_transformer_network(config, device=torch.device("cpu")).network + path = tmp_path / "policy.pt" + save_policy(CategoricalPolicy.from_network(network, config, "actor"), path) + if checkpoint: + artifact = build_model_artifact(network, config) + torch.save( + { + "artifact_type": "sequifier_run_checkpoint", + "model": artifact.state_dict(), + }, + path, + ) + pl.DataFrame({"OBSERVATION_COLUMN": [1.0, 2.0]}).write_csv(tmp_path / "data.csv") + inference = tmp_path / "infer.yaml" + inference.write_text( + yaml.safe_dump( + { + "project_root": str(tmp_path), + "model_path": "policy.pt", + "data_path": "data.csv", + "desired_return": 3.0, + "output_path": "predictions.csv", + "output_probabilities": True, + } + ) + ) + reads = [] + original_load = torch.load + + def counted_load(*args, **kwargs): + reads.append(args[0]) + return original_load(*args, **kwargs) + + monkeypatch.setattr(torch, "load", counted_load) + resolved = load_inferer_config(str(inference), {}, False) + assert isinstance(resolved, PolicyInferenceConfig) + infer_policy(resolved) + assert len(reads) == 1 + assert "_artifacts" not in resolved.model_dump() + predictions = pl.read_csv(tmp_path / "predictions.csv") + assert len(predictions) == 2 + assert set(predictions["action"]) <= {0.0, 1.0} + np.testing.assert_allclose( + predictions["action__probability_0"] + predictions["action__probability_1"], + 1.0, + rtol=1e-6, + ) + + +def test_shared_normalization_preserves_padding_and_zero_scale(): + statistics = {"x": {"mean": 7.0, "std": 0.0}} + values = np.array([7.0, 7.000000001], dtype=np.float64) + expected = normalize_values(values, statistics["x"]) + np.testing.assert_allclose(denormalize_values(expected, statistics["x"]), values) + torch.testing.assert_close( + normalize_values(torch.from_numpy(values), statistics["x"]), + torch.from_numpy(expected), + ) + frame = pl.DataFrame({"x": values}).select( + normalize_values(pl.col("x"), statistics["x"]) + ) + np.testing.assert_allclose(frame["x"], expected) + codec = FixedNumericPreprocessing(["x"], statistics, True, {"x": "float64"}) + seed = codec.seed_from_model( + {"x": np.array([[0.0, expected[1]]])}, np.array([[False, True]]), ["x"], [] + ) + features, metadata = codec.model_inputs(seed) + assert features["x"][0, 0] == 0.0 + assert not metadata["attention_valid_mask"][0, 0] + np.testing.assert_allclose(features["x"][0, 1], expected[1]) + + +def test_generative_inference_reads_artifact_once(tmp_path, monkeypatch): + authored = config_at(tmp_path) + config = policy_execution_config(authored.policy, authored.model) + config.interface().policy = None + config.model.interfaces["actor"].policy = None + network = build_transformer_network(config, device=torch.device("cpu")).network + save_model_artifact(build_model_artifact(network, config), tmp_path / "model.pt") + payload = StoredTensorBatch( + sequences={ + "OBSERVATION_COLUMN": torch.ones(1, 5), + "__policy_previous_action": torch.full((1, 5), 3, dtype=torch.int64), + "__policy_return": torch.ones(1, 5), + "action": torch.full((1, 5), 3, dtype=torch.int64), + }, + sequence_ids=torch.zeros(1, dtype=torch.int64), + subsequence_ids=torch.zeros(1, dtype=torch.int64), + start_item_positions=torch.zeros(1, dtype=torch.int64), + left_pad_lengths=torch.zeros(1, dtype=torch.int64), + split_start_item_positions=torch.zeros(1, dtype=torch.int64), + split_end_item_positions=torch.full((1,), 5, dtype=torch.int64), + ) + save_pt_payload(payload, tmp_path / "data.pt") + inference = tmp_path / "infer.yaml" + inference.write_text( + yaml.safe_dump( + { + "project_root": str(tmp_path), + "model_path": "model.pt", + "data_path": "data.pt", + "model_type": "generative", + "read_format": "pt", + "device": "cpu", + } + ) + ) + reads = [] + original_load = torch.load + + def counted_load(path, *args, **kwargs): + if str(path).endswith("model.pt"): + reads.append(path) + return original_load(path, *args, **kwargs) + + monkeypatch.setattr(torch, "load", counted_load) + monkeypatch.setattr(torch, "compile", lambda model: model) + resolved = load_inferer_config(str(inference), {}, False) + assert isinstance(resolved, ResolvedInferenceConfig) + metadata = resolved.dataset_metadata + assert metadata is not None + infer_worker( + resolved, {}, metadata.id_maps, metadata.selected_columns_statistics, None, True + ) + assert len(reads) == 1 + predictions = pl.read_csv( + tmp_path / "outputs" / "predictions" / "model" / "part-000.csv" + ) + assert len(predictions) > 0 From f3d148e8dd42a277348c1eec8e7334a34817264a Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Sat, 7 Nov 2026 16:38:37 +0100 Subject: [PATCH 10/71] consolidate --- src/sequifier/artifacts/model_artifact.py | 20 +++++++++----------- src/sequifier/rl/distributions.py | 9 +++++++++ src/sequifier/rl/inference.py | 16 +++------------- src/sequifier/rl/runtime.py | 12 +++--------- src/sequifier/trajectory/returns.py | 2 +- src/sequifier/trajectory/schema.py | 3 ++- 6 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index 7090276d..240e54e2 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -138,12 +138,7 @@ def load_model_artifact( ) -> tuple[Any, Any, ModelArtifact]: """Load a portable artifact and return network, resolved config, and schema.""" - payload = torch.load( - Path(path).expanduser().resolve(), - map_location=torch.device(device), - weights_only=False, - ) - artifact = ModelArtifact.from_state_dict(payload) + artifact, _ = read_artifact(path, expected_type="sequifier_model") return restore_model_artifact( artifact, device=device, interface_name=interface_name ) @@ -171,12 +166,18 @@ def restore_model_artifact( return built.network, config, artifact -def read_artifact(path: str | Path) -> tuple[ModelArtifact, str]: +def read_artifact( + path: str | Path, *, expected_type: str | None = None +) -> tuple[ModelArtifact, str]: """Read and validate an execution artifact, retaining its source kind.""" payload = torch.load( Path(path).expanduser().resolve(), map_location="cpu", weights_only=False ) artifact_type = payload.get("artifact_type") if isinstance(payload, dict) else None + if expected_type is not None and artifact_type != expected_type: + if expected_type == "sequifier_run_checkpoint": + raise ValueError("Artifact is not a Sequifier run checkpoint.") + raise ValueError("Artifact is not a portable Sequifier model.") if artifact_type == "sequifier_run_checkpoint": payload = payload["model"] if not isinstance(payload, dict) or not isinstance(artifact_type, str): @@ -205,10 +206,7 @@ def load_execution_model( def model_artifact_from_run_checkpoint(path: str | Path) -> ModelArtifact: - payload = torch.load(Path(path), map_location="cpu", weights_only=False) - if payload.get("artifact_type") != "sequifier_run_checkpoint": - raise ValueError("Artifact is not a Sequifier run checkpoint.") - return ModelArtifact.from_state_dict(payload["model"]) + return read_artifact(path, expected_type="sequifier_run_checkpoint")[0] def load_weights_from_run_checkpoint( diff --git a/src/sequifier/rl/distributions.py b/src/sequifier/rl/distributions.py index f32c0623..b79ef21a 100644 --- a/src/sequifier/rl/distributions.py +++ b/src/sequifier/rl/distributions.py @@ -4,6 +4,12 @@ from torch import Tensor +def action_generator(device: torch.device, seed: int) -> torch.Generator: + return torch.Generator( + device=device if device.type == "cuda" else "cpu" + ).manual_seed(seed) + + class CategoricalActionDistribution: def __init__(self, logits: Tensor): if logits.ndim < 2 or logits.shape[-1] < 2 or not torch.isfinite(logits).all(): @@ -16,6 +22,9 @@ def __init__(self, logits: Tensor): def mode(self) -> Tensor: return self.logits.argmax(dim=-1) + def select(self, deterministic: bool, generator: torch.Generator) -> Tensor: + return self.mode() if deterministic else self.sample(generator) + def sample(self, generator: torch.Generator | None = None) -> Tensor: probabilities = self.distribution.probs # MPS has no device-specific Generator; sample on its generator's device. diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index c55116a2..57e13be9 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -14,7 +14,7 @@ from sequifier.io.sequence import ordered_sequence_rows from sequifier.io.values import domain_scalar from sequifier.rl.checkpoint import load_policy -from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.rl.distributions import CategoricalActionDistribution, action_generator from sequifier.rl.runtime import collect_policy_rollout from sequifier.rollout.factories import load_adapter from sequifier.rollout.history import numeric_scalar @@ -112,9 +112,7 @@ def budget(row): else [indexed.slice(i, 1) for i in range(len(indexed))] ) outputs = [] - generator = torch.Generator( - device=policy.device if policy.device.type == "cuda" else "cpu" - ).manual_seed(config.seed) + generator = action_generator(policy.device, config.seed) pending_inputs = [] pending_rows = [] @@ -124,15 +122,7 @@ def flush(): with torch.no_grad(): inputs = collate_model_inputs(pending_inputs).to(policy.device) decision = CategoricalActionDistribution(policy(inputs)[:, -1, :]) - actions = ( - ( - decision.mode() - if config.deterministic - else decision.sample(generator) - ) - .cpu() - .tolist() - ) + actions = decision.select(config.deterministic, generator).cpu().tolist() probabilities = ( decision.distribution.probs.cpu().tolist() if config.output_probabilities diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index 5d5f508f..2666803b 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -5,7 +5,7 @@ from sequifier.config.rollout_config import RolloutConfig from sequifier.io.policy_window import PolicyWindowView from sequifier.io.values import DomainValue, domain_fields -from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.rl.distributions import CategoricalActionDistribution, action_generator from sequifier.rl.policy import CategoricalPolicy from sequifier.rollout.contracts import EnvironmentAdapter, ResetRequest from sequifier.rollout.coordinator import collect_rollout @@ -51,9 +51,7 @@ def reset(self, request: ResetRequest) -> None: ) self.previous_action_trace = [] self.log_probabilities = [] - self.generator = torch.Generator( - device=self.policy.device if self.policy.device.type == "cuda" else "cpu" - ).manual_seed(request.seed) + self.generator = action_generator(self.policy.device, request.seed) self.pending = None self.ended = False self.active = True @@ -91,11 +89,7 @@ def __call__(self, history: SeedHistory) -> dict[str, DomainValue]: distribution = CategoricalActionDistribution( self.policy(inputs.to(self.policy.device))[:, -1, :] ) - action = ( - distribution.mode() - if self.deterministic - else distribution.sample(self.generator) - ) + action = distribution.select(self.deterministic, self.generator) log_probability = float(distribution.log_prob(action)[0]) index = int(action[0]) self.pending = (observation, index, log_probability) diff --git a/src/sequifier/trajectory/returns.py b/src/sequifier/trajectory/returns.py index a9795120..bb78aad2 100644 --- a/src/sequifier/trajectory/returns.py +++ b/src/sequifier/trajectory/returns.py @@ -2,7 +2,7 @@ from collections.abc import Sequence -from sequifier.rollout.history import numeric_scalar +from sequifier.io.values import numeric_scalar def return_to_go(rewards: Sequence[float], gamma: float = 1.0) -> list[float]: diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index 3ccb440f..05d19e47 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -16,6 +16,7 @@ from sequifier.config.rollout_config import InputBinding from sequifier.io.values import DomainValue +from sequifier.trajectory.returns import remaining_return class Record(BaseModel): @@ -214,7 +215,7 @@ def _validate_policy_execution(trajectory: Trajectory) -> None: "policy return budgets must include the initial and every successor budget" ) for index, step in enumerate(trajectory.transitions): - expected = (budgets[index] - step.reward) / spec.returns.gamma + expected = remaining_return(budgets[index], step.reward, spec.returns.gamma) if not math.isclose( budgets[index + 1], expected, rel_tol=1e-12, abs_tol=1e-12 ): From 65e2a3a4133b74c844646400a5b4e49698f16dc3 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 09:45:50 +0100 Subject: [PATCH 11/71] rename --- documentation/configs/train.md | 2 +- .../configs/train-policy-from-table.yaml | 2 +- .../configs/train-return-conditioned.yaml | 2 +- ...nforcement-learning-implementation-plan.md | 8 +- documentation/policy-learning.md | 8 +- release-notes/unreleased-policy-training.md | 2 +- src/sequifier/config/train_config.py | 8 +- src/sequifier/config/training_plan.py | 10 +- src/sequifier/consolidated-docs.md | 10 +- src/sequifier/make.py | 2 +- src/sequifier/rl/learner.py | 2 +- src/sequifier/rl/training.py | 2 +- src/sequifier/rl_api.py | 4 +- tests/test_consolidation.py | 269 ------------------ 14 files changed, 35 insertions(+), 296 deletions(-) delete mode 100644 tests/test_consolidation.py diff --git a/documentation/configs/train.md b/documentation/configs/train.md index dda3ba6b..a66681b8 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -770,7 +770,7 @@ gradients are absent. ## Policy sequence training -`sequifier train` accepts one `return_conditioned` phase. Put the observation/action +`sequifier train` accepts one `policy_imitation` phase. Put the observation/action schema in `model.interfaces..policy`, a sequence source in each `dataset_training..parts.`, and optimizer/context settings in `global_training` with `training_objective: policy`. Reference sources with diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml index d7703648..5b02c79f 100644 --- a/documentation/example_inputs/configs/train-policy-from-table.yaml +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -50,7 +50,7 @@ global_training: training_plan: phases: - name: behavior_cloning - algorithm: return_conditioned + algorithm: policy_imitation policy: actor epochs: 80 sources: [{source: demonstrations.recorded}] diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index 32b0f5c5..94b55469 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -69,7 +69,7 @@ global_training: training_plan: phases: - name: offline_policy - algorithm: return_conditioned + algorithm: policy_imitation epochs: 80 sources: - source: demonstrations.recorded diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md index 23bf55ae..79f3afe4 100644 --- a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md +++ b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md @@ -18,7 +18,7 @@ acceptance-tested; no tests were written or run, as requested. IQL and the remaining broader stages are still planned; RC is an independent route. Third implementation slice: `sequifier train` resolves a single -`return_conditioned` phase with named trajectory sources and policies, an +`policy_imitation` phase with named trajectory sources and policies, an algorithm runtime boundary, grouped episode splitting, best-action-NLL selection, actor-only artifacts and training reports. Synthetic demo code, demo-specific configurations/public exports and the experimental `policy` command are removed. @@ -93,7 +93,7 @@ rollout/adapters/{gymnasium,tensor_vector}.py trajectory/{schema,store,importers,dataset,relabel}.py rl/{policy,distributions,models,normalization,metrics}.py rl/{runtime,checkpoint,transfer}.py -rl/algorithms/{return_conditioned,iql,ppo}.py +rl/algorithms/{policy_imitation,iql,ppo}.py rl/{ppo_buffer,offline_sampling}.py rollout_api.py Stable external adapter/collector API rl_api.py Stable policy and learner API @@ -267,7 +267,7 @@ Initialize actor from a compatible supervised policy artifact; initialize critic ### 5.2 Return-conditioned sequence modeling -Implement a named algorithm runtime `return_conditioned` using the policy-aligned window view. This is Decision-Transformer-style return conditioning using Sequifier's multivariate rows; do not claim identical interleaved return/state/action token architecture. +Implement a named algorithm runtime `policy_imitation` using the policy-aligned window view. This is Decision-Transformer-style return conditioning using Sequifier's multivariate rows; do not claim identical interleaved return/state/action token architecture. For complete recorded episodes compute: @@ -319,7 +319,7 @@ RC actor to IQL transfer requires a declared conditioning policy: either retain Replace resolved phase branching with a discriminated union normalized from authored config: - `supervised`: existing sources, epochs, and source scheduling, unchanged defaults. -- `return_conditioned`: trajectory sources, policy interface, epochs, return definition. +- `policy_imitation`: trajectory sources, policy interface, epochs, return definition. - `iql`: fixed trajectory sources, updates, actor/V/Q specifications, algorithm settings. - `ppo`: environment source, policy/value specifications, environment-step budget, rollout and update settings (Stage 3). diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 71ce22cd..d1529386 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -8,6 +8,10 @@ optionally, a desired return. The current action is the supervised target. ## Train from ordinary sequence tables +The phase algorithm is `policy_imitation` for both behavior cloning and +return-conditioned imitation. The policy's `return_conditioned` setting selects +between them. + Use the existing `sequenceId` and `itemPosition` conventions. Each row describes **the current observation, the action executed at that observation, and the reward/boundary resulting from that action**. One sequence is one episode or an @@ -65,7 +69,7 @@ global_training: training_plan: phases: - name: behavior_cloning - algorithm: return_conditioned + algorithm: policy_imitation policy: actor epochs: 80 sources: [{source: demonstrations.recorded}] @@ -213,7 +217,7 @@ part containing a sequence source specification. Reference it with the ordinary `{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these parts do not need fabricated criteria. Model/interface references are validated. The shared phase schema preserves supervised defaults; this learner still supports -one `return_conditioned` phase, with BC selected on its policy. Mixed algorithms, +one `policy_imitation` phase, with BC selected on its policy. Mixed algorithms, backbone repositories, image/depth policies and online learning remain planned. Use `initialize_from` for backbone artifact transfer. diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index e3edc1cc..1197575a 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -1,6 +1,6 @@ # Integrated policy training and inference (unreleased) -`sequifier train` accepts one trajectory-backed `return_conditioned` phase; +`sequifier train` accepts one trajectory-backed `policy_imitation` phase; `return_conditioned: false` selects behavior cloning. Policies now use the standard configurable ingestion, backbone, decoder, and network implementation. Encoded categorical observations and matching forecast-backbone transfer are diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index ae047a8e..061f5725 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -822,7 +822,7 @@ def resolve(self): if self.model_name in {".", ".."}: raise ValueError("model_name must name a single directory") if len(self.training_plan.phases) != 1: - raise ValueError("policy training requires one return_conditioned phase") + raise ValueError("policy training requires one policy_imitation phase") phase = self.training_plan.phases[0] if phase.policy not in self.model.interfaces: raise ValueError(f"unknown policy interface {phase.policy!r}") @@ -1319,7 +1319,7 @@ class ResolvedTrainingSource(BaseModel): class ResolvedTrainingPhase(BaseModel): model_config = ConfigDict(extra="forbid") - algorithm: Literal["supervised", "return_conditioned"] = "supervised" + algorithm: Literal["supervised", "policy_imitation"] = "supervised" name: str epochs: int @@ -2334,9 +2334,9 @@ def load_train_config_with_source( if isinstance(phase, dict) } if algorithms - {"supervised"}: - if algorithms != {"return_conditioned"} or len(phases) != 1: + if algorithms != {"policy_imitation"} or len(phases) != 1: raise ValueError( - "This increment supports one return_conditioned phase; mixed plans, " + "This increment supports one policy_imitation phase; mixed plans, " "IQL and PPO are not yet implemented." ) if sensitive or metadata_inline_values or skip_metadata: diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py index 1674984f..c9890ce7 100644 --- a/src/sequifier/config/training_plan.py +++ b/src/sequifier/config/training_plan.py @@ -65,9 +65,9 @@ def validate_mode_fields(self): return self -class ReturnConditionedPhaseConfig(BaseModel): +class PolicyImitationPhaseConfig(BaseModel): model_config = ConfigDict(extra="forbid") - algorithm: Literal["return_conditioned"] + algorithm: Literal["policy_imitation"] name: str = Field(default="policy", min_length=1) epochs: int = Field(gt=0) sources: list[TrainingSourceSpecModel] = Field(min_length=1) @@ -91,7 +91,7 @@ def trajectory_sources(self) -> list[str]: policy: str = Field(min_length=1) -Phase = TypeVar("Phase", TrainingPhaseSpecModel, ReturnConditionedPhaseConfig) +Phase = TypeVar("Phase", TrainingPhaseSpecModel, PolicyImitationPhaseConfig) class TrainingPlanModel(BaseModel, Generic[Phase]): @@ -133,8 +133,8 @@ def validate_unique_names(self): SupervisedTrainingPlanModel = TrainingPlanModel[TrainingPhaseSpecModel] -PolicyTrainingPlanModel = TrainingPlanModel[ReturnConditionedPhaseConfig] +PolicyTrainingPlanModel = TrainingPlanModel[PolicyImitationPhaseConfig] AlgorithmPhase = Annotated[ - TrainingPhaseSpecModel | ReturnConditionedPhaseConfig, + TrainingPhaseSpecModel | PolicyImitationPhaseConfig, Field(discriminator="algorithm"), ] diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 3e7488b3..0a41a309 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -1721,7 +1721,7 @@ gradients are absent. ## Policy sequence training -`sequifier train` accepts one `return_conditioned` phase. Put the observation/action +`sequifier train` accepts one `policy_imitation` phase. Put the observation/action schema in `model.interfaces..policy`, a sequence source in each `dataset_training..parts.`, and optimizer/context settings in `global_training` with `training_objective: policy`. Reference sources with @@ -2347,6 +2347,10 @@ optionally, a desired return. The current action is the supervised target. ## Train from ordinary sequence tables +The phase algorithm is `policy_imitation` for both behavior cloning and +return-conditioned imitation. The policy's `return_conditioned` setting selects +between them. + Use the existing `sequenceId` and `itemPosition` conventions. Each row describes **the current observation, the action executed at that observation, and the reward/boundary resulting from that action**. One sequence is one episode or an @@ -2404,7 +2408,7 @@ global_training: training_plan: phases: - name: behavior_cloning - algorithm: return_conditioned + algorithm: policy_imitation policy: actor epochs: 80 sources: [{source: demonstrations.recorded}] @@ -2552,7 +2556,7 @@ part containing a sequence source specification. Reference it with the ordinary `{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these parts do not need fabricated criteria. Model/interface references are validated. The shared phase schema preserves supervised defaults; this learner still supports -one `return_conditioned` phase, with BC selected on its policy. Mixed algorithms, +one `policy_imitation` phase, with BC selected on its policy. Mixed algorithms, backbone repositories, image/depth policies and online learning remain planned. Use `initialize_from` for backbone artifact transfer. diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 80c7d4cc..99059055 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -133,7 +133,7 @@ resume: {policy: never} training_plan: phases: - - {algorithm: return_conditioned, policy: actor, epochs: 80, sources: [{source: demonstrations.recorded}]} + - {algorithm: policy_imitation, policy: actor, epochs: 80, sources: [{source: demonstrations.recorded}]} """ policy_infer_config_string = """project_root: . diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index a7ebe973..95ee5231 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -103,7 +103,7 @@ def dataset(names: list[str]) -> PolicySequenceDataset: report = { "format": "sequifier.policy-training-report", "version": 1, - "algorithm": "return_conditioned", + "algorithm": "policy_imitation", "config": config.model_dump(mode="json"), "sources": sources, "policy": result.policy.config.model_dump(mode="json"), diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index dc6b925d..bdc8c760 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -67,7 +67,7 @@ def train_policy( training_plan=[ ResolvedTrainingPhase( name=phase.name, - algorithm="return_conditioned", + algorithm="policy_imitation", epochs=phase.epochs, mode="sequential", sources=[source], diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index fe6e7aa4..e7008ac2 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -12,7 +12,7 @@ ) from sequifier.config.sequence_import_config import SequenceImportConfig from sequifier.config.train_config import PolicyTrainConfig -from sequifier.config.training_plan import ReturnConditionedPhaseConfig +from sequifier.config.training_plan import PolicyImitationPhaseConfig from sequifier.io.batch import SequifierBatch, SequifierInputs, collate_sequence_batches from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.io.policy_window import PolicyWindowView, action_index @@ -44,7 +44,7 @@ "import_sequences", "PolicyInferenceConfig", "PolicyTrainConfig", - "ReturnConditionedPhaseConfig", + "PolicyImitationPhaseConfig", "run_policy_training", "CategoricalPolicyConfig", "ReturnDefinition", diff --git a/tests/test_consolidation.py b/tests/test_consolidation.py deleted file mode 100644 index 8e9129f2..00000000 --- a/tests/test_consolidation.py +++ /dev/null @@ -1,269 +0,0 @@ -"""Behavioral checks for shared policy execution, loading, and preprocessing.""" - -from copy import deepcopy - -import numpy as np -import polars as pl -import pytest -import torch -import yaml - -from sequifier.artifacts.model_artifact import build_model_artifact, save_model_artifact -from sequifier.artifacts.model_config import resolved_config_from_model_config -from sequifier.artifacts.model_export import model_execution_config -from sequifier.config.infer_config import ResolvedInferenceConfig, load_inferer_config -from sequifier.config.policy_infer_config import PolicyInferenceConfig -from sequifier.config.train_config import PolicyTrainConfig -from sequifier.infer import infer_worker -from sequifier.io.policy_dataset import PolicySequenceDataset -from sequifier.io.policy_window import PolicyWindowView -from sequifier.io.preprocessing import denormalize_values, normalize_values -from sequifier.io.pt_payload import StoredTensorBatch, save_pt_payload -from sequifier.io.sequence import SequenceEpisode, TransitionColumns -from sequifier.make import policy_train_config_string -from sequifier.model.factory import build_transformer_network -from sequifier.rl.checkpoint import load_policy, save_policy -from sequifier.rl.inference import infer_policy -from sequifier.rl.policy import CategoricalPolicy, policy_execution_config -from sequifier.rl.training import train_policy -from sequifier.rollout.preprocessing import FixedNumericPreprocessing - - -def config_at(root, epochs=2): - values = yaml.safe_load(policy_train_config_string) - values["project_root"] = str(root) - values["global_training"].update(batch_size=2, log_interval=2) - values["training_plan"]["phases"][0]["epochs"] = epochs - return PolicyTrainConfig.model_validate(values) - - -def episodes(config, prefix): - return PolicySequenceDataset( - [ - SequenceEpisode( - episode_id=f"{prefix}-{index}", - item_positions=(0, 1, 2), - observations={"OBSERVATION_COLUMN": (1.0 + index, 2.0, 3.0)}, - actions=(0.0, 1.0, 0.0), - transitions=TransitionColumns( - rewards=(1.0, 1.0, 1.0), terminated=(False, False, True) - ), - ) - for index in range(2) - ], - config.policy, - ) - - -@pytest.mark.parametrize("encoding", ["global", "legacy"]) -@pytest.mark.parametrize("conditioned", [True, False]) -@pytest.mark.parametrize("collision", [True, False]) -def test_policy_artifact_round_trip_keeps_predictions( - tmp_path, encoding, conditioned, collision -): - authored = config_at(tmp_path) - schema = authored.policy.model_copy( - update={ - "categorical_encoding": encoding, - "return_conditioned": conditioned, - "action_column": "OBSERVATION_COLUMN" if collision else "action", - } - ) - model = authored.model.model_copy(deep=True) - if not conditioned: - ingestion = model.interfaces["actor"].ingestion - assert ingestion.type == "embedding" - dims = ingestion.feature_embedding_dims - assert dims is not None - dims.pop("__policy_return") - dims["__policy_previous_action"] += 1 - config = policy_execution_config(schema, model) - network = build_transformer_network(config, device=torch.device("cpu")).network - policy = CategoricalPolicy.from_network(network, config, "actor").eval() - inputs = PolicyWindowView(schema).inputs( - [{"OBSERVATION_COLUMN": 2.0}], [2], [3.0] if conditioned else None, {} - ) - exported = model_execution_config(config) - restored, _ = resolved_config_from_model_config(exported, device="cpu") - assert model_execution_config(restored) == exported - path = tmp_path / "policy.pt" - save_policy(policy, path) - with torch.no_grad(): - torch.testing.assert_close( - load_policy(path)(inputs), policy(inputs), rtol=0, atol=0 - ) - - -@pytest.mark.parametrize("boundary", ["epoch", "batch"]) -def test_policy_resume_matches_uninterrupted_training(tmp_path, boundary): - threads = torch.get_num_threads() - torch.set_num_threads(1) - try: - complete = config_at(tmp_path / "complete") - complete.global_training.save_interval_batches = 1 - complete.global_training.save_interval_val_loss = False - expected = train_policy( - episodes(complete, "train"), episodes(complete, "val"), complete - ) - if boundary == "epoch": - partial = config_at(tmp_path / "resumed", epochs=1) - train_policy(episodes(partial, "train"), episodes(partial, "val"), partial) - values = deepcopy(partial.model_dump()) - values["global_training"]["resume"] = {"policy": "if_exists"} - else: - values = deepcopy(complete.model_dump()) - values["project_root"] = str(tmp_path / "resumed") - checkpoint = next((tmp_path / "complete").rglob("*-epoch-1-batch-1.pt")) - values["global_training"]["resume"] = { - "policy": "required", - "checkpoint_path": str(checkpoint), - } - values["training_plan"]["phases"][0]["epochs"] = 2 - resumed = PolicyTrainConfig.model_validate(values) - actual = train_policy( - episodes(resumed, "train"), episodes(resumed, "val"), resumed - ) - assert actual.metrics == expected.metrics - assert actual.optimizer_steps == expected.optimizer_steps - assert actual.best_epoch == expected.best_epoch - for name, tensor in expected.policy.network.state_dict().items(): - torch.testing.assert_close( - actual.policy.network.state_dict()[name], tensor, rtol=0, atol=0 - ) - finally: - torch.set_num_threads(threads) - - -@pytest.mark.parametrize("checkpoint", [True, False]) -def test_policy_inference_reads_artifact_once(tmp_path, monkeypatch, checkpoint): - authored = config_at(tmp_path) - config = policy_execution_config(authored.policy, authored.model) - network = build_transformer_network(config, device=torch.device("cpu")).network - path = tmp_path / "policy.pt" - save_policy(CategoricalPolicy.from_network(network, config, "actor"), path) - if checkpoint: - artifact = build_model_artifact(network, config) - torch.save( - { - "artifact_type": "sequifier_run_checkpoint", - "model": artifact.state_dict(), - }, - path, - ) - pl.DataFrame({"OBSERVATION_COLUMN": [1.0, 2.0]}).write_csv(tmp_path / "data.csv") - inference = tmp_path / "infer.yaml" - inference.write_text( - yaml.safe_dump( - { - "project_root": str(tmp_path), - "model_path": "policy.pt", - "data_path": "data.csv", - "desired_return": 3.0, - "output_path": "predictions.csv", - "output_probabilities": True, - } - ) - ) - reads = [] - original_load = torch.load - - def counted_load(*args, **kwargs): - reads.append(args[0]) - return original_load(*args, **kwargs) - - monkeypatch.setattr(torch, "load", counted_load) - resolved = load_inferer_config(str(inference), {}, False) - assert isinstance(resolved, PolicyInferenceConfig) - infer_policy(resolved) - assert len(reads) == 1 - assert "_artifacts" not in resolved.model_dump() - predictions = pl.read_csv(tmp_path / "predictions.csv") - assert len(predictions) == 2 - assert set(predictions["action"]) <= {0.0, 1.0} - np.testing.assert_allclose( - predictions["action__probability_0"] + predictions["action__probability_1"], - 1.0, - rtol=1e-6, - ) - - -def test_shared_normalization_preserves_padding_and_zero_scale(): - statistics = {"x": {"mean": 7.0, "std": 0.0}} - values = np.array([7.0, 7.000000001], dtype=np.float64) - expected = normalize_values(values, statistics["x"]) - np.testing.assert_allclose(denormalize_values(expected, statistics["x"]), values) - torch.testing.assert_close( - normalize_values(torch.from_numpy(values), statistics["x"]), - torch.from_numpy(expected), - ) - frame = pl.DataFrame({"x": values}).select( - normalize_values(pl.col("x"), statistics["x"]) - ) - np.testing.assert_allclose(frame["x"], expected) - codec = FixedNumericPreprocessing(["x"], statistics, True, {"x": "float64"}) - seed = codec.seed_from_model( - {"x": np.array([[0.0, expected[1]]])}, np.array([[False, True]]), ["x"], [] - ) - features, metadata = codec.model_inputs(seed) - assert features["x"][0, 0] == 0.0 - assert not metadata["attention_valid_mask"][0, 0] - np.testing.assert_allclose(features["x"][0, 1], expected[1]) - - -def test_generative_inference_reads_artifact_once(tmp_path, monkeypatch): - authored = config_at(tmp_path) - config = policy_execution_config(authored.policy, authored.model) - config.interface().policy = None - config.model.interfaces["actor"].policy = None - network = build_transformer_network(config, device=torch.device("cpu")).network - save_model_artifact(build_model_artifact(network, config), tmp_path / "model.pt") - payload = StoredTensorBatch( - sequences={ - "OBSERVATION_COLUMN": torch.ones(1, 5), - "__policy_previous_action": torch.full((1, 5), 3, dtype=torch.int64), - "__policy_return": torch.ones(1, 5), - "action": torch.full((1, 5), 3, dtype=torch.int64), - }, - sequence_ids=torch.zeros(1, dtype=torch.int64), - subsequence_ids=torch.zeros(1, dtype=torch.int64), - start_item_positions=torch.zeros(1, dtype=torch.int64), - left_pad_lengths=torch.zeros(1, dtype=torch.int64), - split_start_item_positions=torch.zeros(1, dtype=torch.int64), - split_end_item_positions=torch.full((1,), 5, dtype=torch.int64), - ) - save_pt_payload(payload, tmp_path / "data.pt") - inference = tmp_path / "infer.yaml" - inference.write_text( - yaml.safe_dump( - { - "project_root": str(tmp_path), - "model_path": "model.pt", - "data_path": "data.pt", - "model_type": "generative", - "read_format": "pt", - "device": "cpu", - } - ) - ) - reads = [] - original_load = torch.load - - def counted_load(path, *args, **kwargs): - if str(path).endswith("model.pt"): - reads.append(path) - return original_load(path, *args, **kwargs) - - monkeypatch.setattr(torch, "load", counted_load) - monkeypatch.setattr(torch, "compile", lambda model: model) - resolved = load_inferer_config(str(inference), {}, False) - assert isinstance(resolved, ResolvedInferenceConfig) - metadata = resolved.dataset_metadata - assert metadata is not None - infer_worker( - resolved, {}, metadata.id_maps, metadata.selected_columns_statistics, None, True - ) - assert len(reads) == 1 - predictions = pl.read_csv( - tmp_path / "outputs" / "predictions" / "model" / "part-000.csv" - ) - assert len(predictions) > 0 From 6dfa06761a454297085b0013543f2f0edfbd6b5f Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 09:49:40 +0100 Subject: [PATCH 12/71] rename --- documentation/policy-learning.md | 2 +- src/sequifier/consolidated-docs.md | 2 +- src/sequifier/rl/distributions.py | 2 +- src/sequifier/rl_api.py | 4 ++-- src/sequifier/rollout/coordinator.py | 2 +- src/sequifier/trajectory/importers.py | 2 +- src/sequifier/trajectory/schema.py | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index d1529386..26c3434a 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -403,7 +403,7 @@ training example labels only its last position, so every decision contributes once. Current actions, current rewards, and future observations are excluded from inputs; return-to-go is the intentional future-derived condition. -Policy trajectories use `sequifier.numeric-policy-trajectory`, version 1. +Policy trajectories use `sequifier.scalar-policy-trajectory`, version 1. Forecast trajectories can be demonstrations with explicit `action_semantics: executed_predictions` and a finite `action_values` vocabulary. They retain the latest reset observation and start previous-action history with diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 0a41a309..47afedca 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -2742,7 +2742,7 @@ training example labels only its last position, so every decision contributes once. Current actions, current rewards, and future observations are excluded from inputs; return-to-go is the intentional future-derived condition. -Policy trajectories use `sequifier.numeric-policy-trajectory`, version 1. +Policy trajectories use `sequifier.scalar-policy-trajectory`, version 1. Forecast trajectories can be demonstrations with explicit `action_semantics: executed_predictions` and a finite `action_values` vocabulary. They retain the latest reset observation and start previous-action history with diff --git a/src/sequifier/rl/distributions.py b/src/sequifier/rl/distributions.py index b79ef21a..cebeef2b 100644 --- a/src/sequifier/rl/distributions.py +++ b/src/sequifier/rl/distributions.py @@ -51,7 +51,7 @@ def entropy(self) -> Tensor: return self.distribution.entropy() -def masked_action_likelihood( +def masked_action_nll_loss( distribution: CategoricalActionDistribution, actions: Tensor, valid_mask: Tensor ) -> Tensor: if ( diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index e7008ac2..ce56c10b 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -20,7 +20,7 @@ from sequifier.rl.checkpoint import load_policy, save_policy from sequifier.rl.distributions import ( CategoricalActionDistribution, - masked_action_likelihood, + masked_action_nll_loss, ) from sequifier.rl.learner import run_policy_training from sequifier.rl.policy import CategoricalPolicy @@ -55,7 +55,7 @@ "action_index", "collect_policy_rollout", "load_policy", - "masked_action_likelihood", + "masked_action_nll_loss", "remaining_return", "return_to_go", "save_policy", diff --git a/src/sequifier/rollout/coordinator.py b/src/sequifier/rollout/coordinator.py index d9acf932..db83173a 100644 --- a/src/sequifier/rollout/coordinator.py +++ b/src/sequifier/rollout/coordinator.py @@ -166,7 +166,7 @@ def execute(predictions, step_id): last = transitions[-1] return Trajectory( format=( - "sequifier.numeric-policy-trajectory" + "sequifier.scalar-policy-trajectory" if config.mode == "policy" else "sequifier.numeric-forecast-trajectory" ), diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 30d4bd28..6c4e4ea5 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -76,7 +76,7 @@ def sequence_from_trajectory( ) -> SequenceEpisode: episode = validate_trajectory(trajectory) if ( - episode.format != "sequifier.numeric-policy-trajectory" + episode.format != "sequifier.scalar-policy-trajectory" and action_semantics != "executed_predictions" ): raise ValueError( diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index 05d19e47..46aaa31b 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -63,7 +63,7 @@ class Transition(Record): class Trajectory(Record): format: Literal[ - "sequifier.numeric-forecast-trajectory", "sequifier.numeric-policy-trajectory" + "sequifier.numeric-forecast-trajectory", "sequifier.scalar-policy-trajectory" ] = "sequifier.numeric-forecast-trajectory" version: Literal[1] = 1 episode_id: str = Field(min_length=1) @@ -147,7 +147,7 @@ def validate_temporal_contract(self): self.total_return, expected_return, rel_tol=1e-12, abs_tol=1e-12 ): raise ValueError("total_return does not equal the sum of rewards") - if self.format == "sequifier.numeric-policy-trajectory": + if self.format == "sequifier.scalar-policy-trajectory": if ( sum(self.seed.attention_valid_mask) != 1 or len(self.prediction_columns) != 1 From 231a1bccdc144578152779930ff7ca67d727a1ec Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 10:30:06 +0100 Subject: [PATCH 13/71] iql --- documentation/configs/train.md | 12 +- .../example_inputs/configs/train-bc-toy.yaml | 45 ++ .../configs/train-iql-from-table.yaml | 76 +++ .../example_inputs/configs/train-iql-toy.yaml | 57 ++ .../example_inputs/iql_toy_control.py | 140 +++++ .../plans/offline-iql-implementation-plan.md | 222 ++++++++ ...nforcement-learning-implementation-plan.md | 9 + documentation/policy-learning.md | 73 ++- release-notes/unreleased-policy-training.md | 13 +- src/sequifier/config/train_config.py | 14 +- src/sequifier/config/training_plan.py | 69 ++- src/sequifier/consolidated-docs.md | 85 ++- src/sequifier/io/iql_dataset.py | 130 +++++ src/sequifier/io/policy_window.py | 18 + src/sequifier/make.py | 58 ++ src/sequifier/rl/iql.py | 526 ++++++++++++++++++ src/sequifier/rl/learner.py | 7 +- src/sequifier/rl/models.py | 72 +++ src/sequifier/rl/policy.py | 8 + src/sequifier/rl/training.py | 2 + src/sequifier/rl_api.py | 6 +- src/sequifier/train.py | 2 +- 22 files changed, 1613 insertions(+), 31 deletions(-) create mode 100644 documentation/example_inputs/configs/train-bc-toy.yaml create mode 100644 documentation/example_inputs/configs/train-iql-from-table.yaml create mode 100644 documentation/example_inputs/configs/train-iql-toy.yaml create mode 100644 documentation/example_inputs/iql_toy_control.py create mode 100644 documentation/plans/offline-iql-implementation-plan.md create mode 100644 src/sequifier/io/iql_dataset.py create mode 100644 src/sequifier/rl/iql.py create mode 100644 src/sequifier/rl/models.py diff --git a/documentation/configs/train.md b/documentation/configs/train.md index a66681b8..776b6afb 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -1,7 +1,7 @@ # Train Command Guide -For trajectory-backed categorical policy training and inference through the same -commands, see [policy learning](../policy-learning.md). +For trajectory-backed categorical BC, return-conditioned imitation, and offline +IQL through the same commands, see [policy learning](../policy-learning.md). `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated @@ -770,7 +770,7 @@ gradients are absent. ## Policy sequence training -`sequifier train` accepts one `policy_imitation` phase. Put the observation/action +`sequifier train` accepts one `policy_imitation` or `iql` phase. Put the observation/action schema in `model.interfaces..policy`, a sequence source in each `dataset_training..parts.`, and optimizer/context settings in `global_training` with `training_objective: policy`. Reference sources with @@ -781,10 +781,12 @@ Return-conditioned training additionally requires rewards and a known return endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories all use the same episode representation. Models support CPU/CUDA/MPS, backbone transfer via `initialize_from`, and shared-engine checkpoint resume via -`global_training.resume`. Mixed phases, IQL/PPO and image/depth policies remain +`global_training.resume`. IQL additionally requires complete transition data +and an unconditioned categorical policy. Mixed phases, PPO and image/depth policies remain planned. The former compact registries and configuration aliases have been consolidated into the shared model/dataset authoring surface. See the [complete table example](../example_inputs/configs/train-policy-from-table.yaml), -[return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml) +[return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml), +[IQL example](../example_inputs/configs/train-iql-from-table.yaml), and [policy learning guide](../policy-learning.md). diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml new file mode 100644 index 00000000..f6789d82 --- /dev/null +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -0,0 +1,45 @@ +project_root: . +model_name: toy-bc +device: cpu +seed: 1010 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128} + positional_encoding_scope: global + interfaces: + actor: + input_columns: [position] + target_columns: [action] + policy: + observation_columns: [position] + action_column: action + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {position: 1, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +dataset_training: + demonstrations: + model_interface: actor + parts: + recorded: {paths: [data/offline-transitions.csv]} +global_training: + training_objective: policy + context_length: 16 + batch_size: 64 + learning_rate: 0.0003 + validation_fraction: 0.2 +training_plan: + phases: + - name: behavior_cloning + algorithm: policy_imitation + policy: actor + sources: [{source: demonstrations.recorded}] + epochs: 30 diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml new file mode 100644 index 00000000..68c2524d --- /dev/null +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -0,0 +1,76 @@ +project_root: . +model_name: offline-iql +device: cpu +seed: 1010 +# Optional compatible, unconditioned BC actor artifact: +# initialize_from: models/behavior-cloning/behavior-cloning.pt + +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128} + positional_encoding_scope: global + interfaces: + actor: + input_columns: [position, velocity] + target_columns: [action] + policy: + observation_columns: [position, velocity] + action_column: action + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: + position: 1 + velocity: 1 + __policy_previous_action: 6 + decoder: {type: linear, prediction_length: 1, support: 1} + +dataset_training: + demonstrations: + model_interface: actor + parts: + recorded: + paths: [data/offline-transitions.parquet] + columns: + sequence_id: sequenceId + item_position: itemPosition + action: action + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {position: next_position, velocity: next_velocity} + +global_training: + training_objective: policy + context_length: 16 + batch_size: 64 + learning_rate: 0.005 + validation_fraction: 0.2 + resume: {policy: if_exists} + +training_plan: + phases: + - name: offline_improvement + algorithm: iql + policy: actor + sources: [{source: demonstrations.recorded}] + updates: 10000 + batch_size: 64 + gamma: 0.99 + expectile: 0.7 + inverse_temperature: 3.0 + max_weight: 100.0 + target_update_rate: 0.005 + actor_learning_rate: 0.0003 + value_learning_rate: 0.0003 + critic_learning_rate: 0.0003 + evaluation_interval: 100 + checkpoint_interval: 100 diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml new file mode 100644 index 00000000..184d4eb7 --- /dev/null +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -0,0 +1,57 @@ +project_root: . +model_name: toy-iql +device: cpu +seed: 1010 +initialize_from: models/toy-bc/toy-bc.pt +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128} + positional_encoding_scope: global + interfaces: + actor: + input_columns: [position] + target_columns: [action] + policy: + observation_columns: [position] + action_column: action + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {position: 1, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +dataset_training: + demonstrations: + model_interface: actor + parts: + recorded: + paths: [data/offline-transitions.csv] + columns: + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {position: next_position} +global_training: + training_objective: policy + context_length: 16 + batch_size: 64 + learning_rate: 0.005 + validation_fraction: 0.2 + resume: {policy: if_exists} +training_plan: + phases: + - name: offline_improvement + algorithm: iql + policy: actor + sources: [{source: demonstrations.recorded}] + updates: 10000 + batch_size: 64 + evaluation_interval: 100 + checkpoint_interval: 100 diff --git a/documentation/example_inputs/iql_toy_control.py b/documentation/example_inputs/iql_toy_control.py new file mode 100644 index 00000000..8a3a18e8 --- /dev/null +++ b/documentation/example_inputs/iql_toy_control.py @@ -0,0 +1,140 @@ +"""Small offline control dataset and separate BC/IQL rollout comparison. + +Generate data with ``python iql_toy_control.py generate data/offline-transitions.csv``. +After training both actors, run +``python iql_toy_control.py evaluate path/to/bc.pt path/to/iql.pt``. +This is an example benchmark, not part of the test suite. +""" + +import csv +import random +import statistics +import sys +from pathlib import Path + +from sequifier.config.rollout_config import RolloutConfig +from sequifier.rl.checkpoint import load_policy +from sequifier.rl.runtime import collect_policy_rollout +from sequifier.rollout.contracts import StepResult +from sequifier.trajectory.schema import SeedHistory + +HORIZON = 12 + + +class ToyControlAdapter: + protocol_version = 1 + + def reset(self, request): + self.position = int(request.seed_history["position"][-1]) + + def step(self, request): + if int(request.observation["position"]) != self.position: + raise ValueError("control state and policy observation disagree") + action = request.predictions["action"] + self.position += -1 if action == "left" else 1 + done = self.position == 0 + return StepResult( + external_inputs={"position": float(self.position)}, + reward=1.0 if done else -0.05, + terminated=done, + truncated=request.step_id + 1 == HORIZON and not done, + ) + + def close(self): + pass + + +def generate(path): + rng = random.Random(1010) + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter( + stream, + fieldnames=[ + "sequenceId", + "itemPosition", + "position", + "action", + "reward", + "terminated", + "truncated", + "bootstrap_allowed", + "next_position", + ], + ) + writer.writeheader() + for episode in range(500): + position = rng.choice((-4, -3, 3, 4)) + quality = 0.85 if episode % 2 else 0.25 + for step in range(HORIZON): + good = "left" if position > 0 else "right" + action = ( + good + if rng.random() < quality + else ("right" if good == "left" else "left") + ) + successor = position + (-1 if action == "left" else 1) + terminated = successor == 0 + truncated = step + 1 == HORIZON and not terminated + writer.writerow( + { + "sequenceId": episode, + "itemPosition": step, + "position": float(position), + "action": action, + "reward": 1.0 if terminated else -0.05, + "terminated": terminated, + "truncated": truncated, + "bootstrap_allowed": not terminated, + "next_position": float(successor), + } + ) + position = successor + if terminated or truncated: + break + + +def evaluate(paths): + config = RolloutConfig.model_validate( + { + "mode": "policy", + "adapter": {"factory": "iql_toy_control:ToyControlAdapter"}, + "bindings": [ + {"input": "position", "source": "environment", "key": "position"} + ], + "trajectory_path": "unused.json", + "seed_index": 0, + "max_episode_steps": HORIZON, + } + ) + for path in paths: + policy = load_policy(path) + returns = [] + for episode, position in enumerate((-4, -3, 3, 4) * 25): + seed = SeedHistory( + observations={"position": [float(position)]}, + attention_valid_mask=[True], + static_inputs={}, + ) + trajectory = collect_policy_rollout( + policy, + ToyControlAdapter(), + seed, + config, + HORIZON, + random_seed=episode, + ) + returns.append(trajectory.total_return) + print(path, "mean", statistics.mean(returns), "std", statistics.stdev(returns)) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "generate": + generate(sys.argv[2]) + elif len(sys.argv) >= 4 and sys.argv[1] == "evaluate": + evaluate(sys.argv[2:]) + else: + raise SystemExit( + "usage: iql_toy_control.py generate FILE | evaluate BC_MODEL IQL_MODEL" + ) diff --git a/documentation/plans/offline-iql-implementation-plan.md b/documentation/plans/offline-iql-implementation-plan.md new file mode 100644 index 00000000..885ef61c --- /dev/null +++ b/documentation/plans/offline-iql-implementation-plan.md @@ -0,0 +1,222 @@ +# Offline IQL implementation plan for the current policy-learning slice + +Status: implementation added, verification deferred at user request, 2026-11-09. +This is a scoped follow-up to +[the rollout and reinforcement-learning plan](rollout-and-reinforcement-learning-implementation-plan.md), +especially sections 3, 5.1, 5.3–5.5, and 8. The target is an **offline IQL path +as complete as the current scalar, single-categorical-action BC and +return-conditioned imitation path**: train from the existing supported sequence +sources, select and save an actor, resume training, load it for inference, and +execute it with the existing single-environment coordinator. The original plan's +continuous/joint actions, media, vector/remote environments, DDP, PPO, and +mixed-algorithm runs remain later work. + +## 1. Baseline in the diff since `2d01d407cd2134c8197b1a18f0034d01486f18a9` + +The diff provides the dependencies for this slice, but no IQL learner: + +- `sequifier train` accepts one `policy_imitation` phase. It trains categorical + BC or return-conditioned imitation using the ordinary model, optimization, + evaluation, and format-v2 run-checkpoint services. The best actor is a normal + portable model artifact. `sequifier infer` can make raw table decisions and + execute a policy through the shared rollout coordinator. +- CSV/Parquet, prepared scalar histories, and version-1 JSON rollout logs adapt + to `SequenceEpisode`. Optional `TransitionColumns` already represent rewards, + boundary flags, successors, and bootstrap flags. Episode splitting, training-only + fitting, bounded episode caching, and causal windows exist. +- Rollout collection has a persistent **single** local adapter, explicit input + bindings, terminal action execution, scalar reward, final successor, and atomic + one-episode JSON output. It is not a general vector simulator or an evaluation + service. +- `CategoricalPolicy` and `CategoricalActionDistribution` support one categorical + action. There is no V, Q, target-Q, transition sampler, IQL optimizer, or + multi-network checkpoint. Policy training's config and command dispatch reject + IQL and more than one phase. + +No repository test suite is present in this checkout; the release notes say the +tests moved to `../sequifier-test/tests/sequifier-refactor-unit`. Verification +below must discover and run that suite where available, and add focused tests in +the owning test repository or an agreed in-repository location. + +## 2. Scope and contract decisions + +1. **Action and model scope.** One fixed categorical action vocabulary, existing + scalar/categorical observation and static fields, finite context, one actor + interface, and independent V/Q1/Q2 modules. The actor is unconditioned + (`return_conditioned: false`). A return-conditioned actor is rejected by IQL + until an explicit conditioning or distillation policy is implemented. The + ordinary BC actor can warm-start IQL after exact schema, vocabulary, + normalization, architecture, and context-length checks; critics start fresh. +2. **Data scope.** A fixed offline training partition and separate held-out + partition from the current `SequenceSourceConfig` formats. IQL requires a + reward, action, known termination/truncation and bootstrap decision at every + sampled step, plus a genuine successor whenever bootstrap is allowed. BC's + ability to use reward-free data stays intact. Collection cutoffs and fragments + are eligible when their history and successor are known; unknown boundaries + are rejected for IQL. No new online data is added during the phase. +3. **Runtime scope.** One `iql` phase via `sequifier train`, with Python entry + point parity, single process on the currently supported policy devices, + actor-only inference/rollouts, and exact offline resume from complete update + boundaries. No same-run BC→IQL phase scheduling is required for this slice: + BC artifact warm-start is the explicit bridge. Do not imply the full + multi-phase or distributed claims of the parent plan are complete. +4. **Reward and evaluation scope.** Existing sources contain one scalar reward. + Preserve it in raw units; expose an optional fixed, checkpointed scale only + if needed. Select checkpoints using a declared held-out metric. Include + held-out losses/diagnostics and a repeatable local rollout evaluation on a + small control fixture, so learning quality is measured independently of + Bellman loss. A general training-time rollout monitor is not part of this + increment. + +## 3. Dependency-ordered work packages + +### A. Lock transition and successor-history semantics + +Extend `io/sequence.py` and `trajectory/importers.py` with an IQL eligibility +validator. Require transition columns to be complete for selected IQL steps; +reject absent rewards or boundary/bootstrap flags, non-finite reward, ambiguous +action timing, bootstrap after termination, bootstrap with a missing final +successor, and a reset observation passed as a successor. Preserve the current +ability to record both `terminated` and `truncated`, with termination taking +precedence. Make validation errors include source, episode ID, and step. + +Add an IQL transition view in `io/` that indexes decisions across episodes without +materializing all context windows. It returns `h_t`, recorded `a_t`, scalar +`r_t`, `h_next`, bootstrap mask, and valid/padding masks. Reuse +`PolicyWindowView` for `h_t`; add a successor-window method that appends the +**actual** successor observation and uses `a_t` as its previous action. For an +interior step, the explicit successor must equal the next row when both are +present. For a terminal step without a successor, return a masked placeholder +and never evaluate V on it. Do not cross episode or fragment boundaries by +looking up the next dataset item. Static context and trained codecs are shared +between both windows. Fit vocabularies and normalization on training episodes +only, then freeze them for validation and inference. + +Sampling is uniform over eligible transitions, with a persisted RNG state and +dataset identity (content hash, episode/split IDs, codec hash). Keep the existing +bounded episode cache and construct only the two context-sized windows per +sample. Continue to guard against duplicate paths, IDs, content, and collection +groups across train/validation partitions. + +**Gate A:** hand-authored episodes prove first/interior/final alignment, truncation +bootstrap from the final state, terminal no-bootstrap, fragment context, and +rejection of missing/contradictory capabilities. Changing `a_t`, reward, or future +observation cannot change actor/V inputs at time `t`. + +### B. Add actor, V, and twin-Q model primitives + +In `rl/models.py`, compose V, Q1, and Q2 from the existing ingestion/backbone +building blocks. V accepts only history. Each Q encodes history, then fuses the +**recorded current action** through a separate categorical action encoder before +its scalar head. Q1 and Q2 have independent learned parameters. Actor, V, Q1, +and Q2 share immutable codecs/specifications but no trainable parameters or +mutable normalization buffers. Make parameter ownership and optimizer assignment +explicit and validate disjointness before training. + +Expose a small policy runtime boundary in `rl/policy.py`/ +`rl/distributions.py`: deterministic/stochastic `act` and +`evaluate_actions` return the same action IDs/domain labels and joint log +probability for the recorded categorical action. Keep special/start IDs excluded +from actions. This is a one-component instance of the parent plan's contract; +continuous, ordered, and joint distributions are deferred. Keep probability +math in FP32 and reject invalid action labels. + +**Gate B:** shape, action-codec, finite-output, parameter-disjointness, and +gradient-ownership tests. Actor/V are invariant to changing current action +labels and future observations; Q responds to its explicit action input. + +### C. Add the IQL phase and optimizer transactions + +Add a discriminated `IQLPhaseConfig` to `config/training_plan.py` and route it +through `config/train_config.py`, `train.py`, and `rl/learner.py`. Keep the +existing supervised and imitation defaults. Require one IQL phase for this +slice, named training/validation sources, actor/V/Q specs, `updates`, `batch_size`, +`gamma`, `expectile`, `inverse_temperature`, `max_weight`, target-Q Polyak rate, +separate optimizer/scheduler plans, clipping limits, evaluation/checkpoint +intervals, and optional BC artifact initialization. Validate all ranges, +references, action/schema compatibility, unsupported distribution modes and +distributed/FSDP settings **before** a long run starts. Do not invent epochs or +supervised targets for an IQL update budget. + +Add `rl/algorithms/iql.py` behind an algorithm runtime that owns preparation, +one update, evaluation, checkpoint state, and named counters. Reuse the existing +optimizer transaction and random-state services where their semantics fit; +extend them to assign independent V, Q, and actor optimizers and scheduler +clocks. Controller/observer events must identify algorithm, network role, +algorithm update, and optimizer step so one-optimizer integrations cannot +silently act on a different network. The supervised loss path stays untouched. + +For each sampled batch, take a consistent pre-update snapshot of detached +targets/advantages: + +```text +qbar = min(Q1_target(h_t,a_t), Q2_target(h_t,a_t)) +delta = stop_gradient(qbar) - V(h_t) +L_V = mean(|expectile - 1[delta < 0]| * delta²) +y = r_t + gamma * bootstrap_t * stop_gradient(V(h_next)) +L_Q = mean((Q1(h_t,a_t)-y)² + (Q2(h_t,a_t)-y)²) +w = min(exp(inverse_temperature * stop_gradient(qbar - V(h_t))), max_weight) +L_actor = -mean(stop_gradient(w) * log pi(a_t|h_t)) +``` + +Cap the exponent input at `log(max_weight)` for stability. Apply V, Q, and actor +optimizer transactions to those targets without gradient paths across networks. +Update target Qs by Polyak averaging **only after a successful Q step**. Define +one IQL update as all three attempted transactions and record applied/skipped +counters separately; handle AMP skips without advancing the wrong scheduler or +target. Stop on non-finite targets/losses instead of silently clipping them. + +**Gate C:** hand-computed expectile asymmetry, Bellman bootstrap, twin minimum, +actor weights/cap, and target updates match reference numbers. Only the intended +network receives gradients in each transaction. A one-batch deterministic run +produces finite metrics and saves an actor that `load_policy` can execute. + +### D. Bundle IQL state without changing portable actor artifacts + +Add an algorithm-owned checkpoint bundle in `rl/checkpoint.py` or a dedicated +module. It contains actor/V/Q1/Q2 and target-Q weights, three +optimizer/scheduler/scaler states, update and applied-step counters, sampler and +all device RNG states, frozen normalization/codecs, source/split/content hashes, +phase config, and artifact-initialization provenance. Save atomically after a +complete update. Restore strictly; missing critics or mismatched data/specs +must fail exact resume. Retain reading of existing supervised/imitation +format-v2 checkpoints, and do not reinterpret one as an IQL resume bundle. + +Export **only** the actor through the current `sequifier_model` path. Record IQL +provenance without embedding critics or optimizers. Inference, `load_policy`, +and the single-environment coordinator should execute it with the existing +codec/action mapping. Artifact warm-start and exact IQL resume remain distinct. + +**Gate D:** interrupted versus uninterrupted fixed-seed training matches +sample order, counters, and weights within device-appropriate tolerances; +corrupt/mismatched bundles fail; existing format-v2 resume and model loading +still work; actor outputs match before and after export. + +### E. Evaluate, document, and release the slice + +Write a structured IQL report with source/split IDs, exclusions, update counts, +actor NLL, V/Q ranges, Bellman residual, advantage and weight quantiles, +weight-cap fraction, best-checkpoint criterion, and artifact paths. Held-out +diagnostics must not fit codecs or update networks. Provide a deterministic +small local control environment and mixed-quality offline behavior dataset. +Compare IQL against BC on fixed seeds using rollout return and action quality; +report mean, spread, dataset-generation rules, and tolerances. The local +rollout evaluation can initially be a separate explicit evaluation command/job +that consumes each saved actor and the same adapter protocol; it need not be a +generalized training monitor. + +Add one complete `train-iql` YAML example, Python API example, and documentation +for required transition columns, cutoff/truncation handling, actor warm-start, +resume, inference, unsupported combinations, and result interpretation. Update +`documentation/policy-learning.md`, config docs, `sequifier make` templates, +consolidated docs, and release notes. Run lint/type checks and the existing +suite, plus the focused gates above and a clean CLI train → checkpoint → reload +→ infer → rollout smoke run. + +**Gate E / done:** a user can import or point to an existing supported scalar +trajectory source, train a fixed-dataset categorical IQL actor with a named +phase, resume it exactly, inspect learning and held-out behavior, load the +actor as an ordinary model artifact, and execute it through the current local +coordinator. BC and return-conditioned workflows still pass their existing +checks. This completes **this IQL slice**, not all Stage 1/2 acceptance criteria +of the parent plan. diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md index 79f3afe4..ce750b4f 100644 --- a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md +++ b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md @@ -47,6 +47,15 @@ bindings or seed histories. Policy image/depth inputs, mixed-algorithm execution IQL/PPO and their algorithm-specific buffers remain planned. No tests or training runs were written or executed for this follow-up, as requested. +Offline IQL follow-up: one fixed-dataset, scalar, single-categorical-action +`iql` phase now uses transition-capable sequence sources, independent V and twin +Q networks, target-Q backups, advantage-weighted actor extraction, a complete +learner checkpoint, and ordinary actor-only model artifacts. See the +[scoped IQL implementation plan](offline-iql-implementation-plan.md) and +[policy learning guide](../policy-learning.md). This does not complete the +broader Stage 1/2 contracts below. No tests were written or run for this +follow-up, as requested. + Deliver this as one coordinated update, built and verified in the three dependency-ordered stages below. Completion means all three stages work together; stage boundaries are engineering checkpoints, not separate proposed releases. This document specifies new behavior and proposed API names, not functionality already present. ## 1. Outcome and scope diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 26c3434a..aa7f5411 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -10,7 +10,8 @@ optionally, a desired return. The current action is the supervised target. The phase algorithm is `policy_imitation` for both behavior cloning and return-conditioned imitation. The policy's `return_conditioned` setting selects -between them. +between them. Offline IQL uses a separate `iql` phase with an unconditioned +categorical actor. Use the existing `sequenceId` and `itemPosition` conventions. Each row describes **the current observation, the action executed at that observation, and the @@ -155,7 +156,8 @@ values are decoded using the original metadata, then fitted for the selected training partition. Missing reward/boundary information remains unknown. For a metadata-referenced prepared split, use `metadata_config_path` and `split_index` instead of `paths`. Scalar prepared histories are currently supported; -policy image/depth ingestion and IQL/PPO remain planned. +policy image/depth ingestion and PPO remain planned; IQL supports the scalar +transition-capable sources described below. A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy `manifest` listing version-1 JSON episodes. These are validated rollout recordings @@ -217,7 +219,7 @@ part containing a sequence source specification. Reference it with the ordinary `{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these parts do not need fabricated criteria. Model/interface references are validated. The shared phase schema preserves supervised defaults; this learner still supports -one `policy_imitation` phase, with BC selected on its policy. Mixed algorithms, +one `policy_imitation` or `iql` phase, with BC selected on its policy. Mixed algorithms, backbone repositories, image/depth policies and online learning remain planned. Use `initialize_from` for backbone artifact transfer. @@ -443,6 +445,69 @@ also accept policy model and run artifacts. The previous standalone numeric-policy artifacts (including versions 1 and 2) use a different architecture and must be retrained. This workflow currently supports one offline categorical actor phase. Mixed supervised/policy plans, -critics, IQL/PPO, distributed policy learning, custom run integrations, multimodal +PPO, distributed policy learning, custom run integrations, multimodal trajectory storage, and ONNX policy execution through the CLI remain outside this workflow. PyTorch policy execution supports CPU, CUDA, and MPS. + +## Offline IQL + +Use a single `iql` phase to improve an unconditioned categorical actor from a +fixed offline dataset. See the complete +[IQL configuration](example_inputs/configs/train-iql-from-table.yaml). The +actor can start from a compatible BC model with `initialize_from`. IQL creates +separate value, Q1, Q2, and target-Q networks; the portable model artifact +contains only the selected actor. The run checkpoint contains the full learner. +In this categorical slice, V and Q use independent copies of the configured +actor history encoder plus their own scalar heads; their architecture is not +authored in a separate model registry. + +Each selected decision needs a finite scalar reward, `terminated`, `truncated`, +and `bootstrap_allowed`. An ordinary nonterminal decision must bootstrap from +its genuine successor observation. An interior row may obtain that successor +from the following row; a final nonterminal or truncated row needs explicitly +mapped successor columns. A true terminal may omit its successor and cannot +bootstrap. Termination takes precedence when both boundary flags are true. +Unknown boundaries and missing rewards are errors for IQL even though BC can +use those demonstrations. Current observations and previous actions form the +actor/value history; Q alone receives the current recorded action. Splits, +categorical fitting, and numeric statistics come only from training episodes. + +`updates` is the number of optimizer cycles, each containing one value, critic, +and actor step. The phase controls uniform transition batch size, discount, +expectile, inverse temperature, maximum actor weight, target-Q update rate, +separate optimizer learning rates/specs and scheduler specs, gradient clipping, +and evaluation/checkpoint intervals. Schedulers advance after their own +optimizer update. The held-out action likelihood selects the portable actor; +the report also records value, Q, Bellman, advantage, and weight diagnostics. +The supported per-role optimizers are Adam, AdamW, SGD, and RMSprop. +`global_training.batch_size` and `global_training.learning_rate` remain required +by the shared policy config; the IQL phase's batch size and three learning rates +control its actual updates. +Use a separate application evaluation with `collect_policy_rollout` to compare +returns against BC in a local environment. +The [toy control example](example_inputs/iql_toy_control.py) generates a fixed +mixed-quality dataset for its [BC](example_inputs/configs/train-bc-toy.yaml) +and [IQL](example_inputs/configs/train-iql-toy.yaml) configs. Train BC first; +the IQL config warm-starts from its actor artifact. The example compares saved +actors over 100 fixed initial states and reports mean and standard deviation +rather than asserting an improvement for every seed. + +```console +python documentation/example_inputs/iql_toy_control.py generate data/offline-transitions.csv +sequifier train --config-path documentation/example_inputs/configs/train-bc-toy.yaml +sequifier train --config-path documentation/example_inputs/configs/train-iql-toy.yaml +python documentation/example_inputs/iql_toy_control.py evaluate models/toy-bc/toy-bc.pt models/toy-iql/toy-iql.pt +``` + +`global_training.resume: {policy: if_exists}` resumes the IQL learner from its +complete update checkpoint. `policy: required` needs an explicit +`checkpoint_path`. Exact resume validates the configuration, source content, +split, fitted vocabulary, and normalization. A BC model passed to +`initialize_from` is a warm start, not a learner checkpoint. IQL currently +supports scalar histories, one categorical action, and local execution; a +return-conditioned actor, distributed training, mixed precision, continuous or +joint actions, and mixed phase plans are rejected. + +Python callers can use `run_iql_training(config)` from `sequifier.rl_api` with a +validated `PolicyTrainConfig`. The return value contains the selected +`CategoricalPolicy`, recorded update metrics, and training/validation episode IDs. diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index 1197575a..96e64e1f 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -1,6 +1,13 @@ # Integrated policy training and inference (unreleased) -`sequifier train` accepts one trajectory-backed `policy_imitation` phase; +Offline IQL now accepts one fixed-dataset `iql` phase for an unconditioned, +single-categorical actor. It adds strict transition/successor validation, +independent value and twin-Q networks, target-Q updates, weighted actor +extraction, a complete learner checkpoint, and actor-only model export. +Continuous/joint actions, PPO, distributed IQL, and mixed plans remain future +work. No tests were written or run for this increment, as requested. + +For imitation, `sequifier train` accepts one trajectory-backed `policy_imitation` phase; `return_conditioned: false` selects behavior cloning. Policies now use the standard configurable ingestion, backbone, decoder, and network implementation. Encoded categorical observations and matching forecast-backbone transfer are @@ -19,8 +26,8 @@ and context-sized window construction. Behavior cloning retains observed state/action pairs from truncated and cutoff episodes without return labels. Grouped episode splitting and training-only normalization remain in place. -`sequifier make` includes policy train and infer templates. Mixed algorithm -phases, critics, IQL/PPO, distributed policy learning, multimodal trajectories, +`sequifier make` includes policy train, IQL train, and infer templates. Mixed algorithm +phases, PPO, distributed policy learning, multimodal trajectories, and CLI ONNX policy execution remain future work. Policy sequence representation follow-up: diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 061f5725..5e08c5a6 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -822,10 +822,16 @@ def resolve(self): if self.model_name in {".", ".."}: raise ValueError("model_name must name a single directory") if len(self.training_plan.phases) != 1: - raise ValueError("policy training requires one policy_imitation phase") + raise ValueError("policy training requires one phase") phase = self.training_plan.phases[0] if phase.policy not in self.model.interfaces: raise ValueError(f"unknown policy interface {phase.policy!r}") + if phase.algorithm == "iql": + selected_policy = self.model.interfaces[phase.policy].policy + if selected_policy is not None and selected_policy.return_conditioned: + raise ValueError("IQL requires an unconditioned categorical actor") + if self.global_training.data_parallelism is not None: + raise ValueError("IQL does not yet support distributed execution") policy = self.policy interface = self.model.interfaces[phase.policy] if ( @@ -2334,10 +2340,10 @@ def load_train_config_with_source( if isinstance(phase, dict) } if algorithms - {"supervised"}: - if algorithms != {"policy_imitation"} or len(phases) != 1: + if len(phases) != 1 or not algorithms <= {"policy_imitation", "iql"}: raise ValueError( - "This increment supports one policy_imitation phase; mixed plans, " - "IQL and PPO are not yet implemented." + "Policy training supports one policy_imitation or iql phase; " + "mixed plans and PPO are not yet implemented." ) if sensitive or metadata_inline_values or skip_metadata: raise ValueError( diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py index c9890ce7..e859e158 100644 --- a/src/sequifier/config/training_plan.py +++ b/src/sequifier/config/training_plan.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from sequifier.config.components import ComponentSpec from sequifier.config.identifiers import identifier as _identifier from sequifier.typechecking import beartype @@ -91,7 +92,69 @@ def trajectory_sources(self) -> list[str]: policy: str = Field(min_length=1) -Phase = TypeVar("Phase", TrainingPhaseSpecModel, PolicyImitationPhaseConfig) +class IQLPhaseConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + algorithm: Literal["iql"] + name: str = Field(default="offline_iql", min_length=1) + policy: str = Field(min_length=1) + sources: list[TrainingSourceSpecModel] = Field(min_length=1) + validation_sources: list[str] | None = Field(default=None, min_length=1) + updates: int = Field(gt=0) + batch_size: int = Field(default=64, gt=0) + gamma: float = Field(default=0.99, gt=0, le=1, allow_inf_nan=False) + expectile: float = Field(default=0.7, gt=0, lt=1, allow_inf_nan=False) + inverse_temperature: float = Field(default=3.0, gt=0, allow_inf_nan=False) + max_weight: float = Field(default=100.0, ge=1, allow_inf_nan=False) + target_update_rate: float = Field(default=0.005, gt=0, le=1, allow_inf_nan=False) + actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + critic_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + actor_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + value_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + critic_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + actor_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + value_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + critic_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) + evaluation_interval: int = Field(default=100, gt=0) + checkpoint_interval: int = Field(default=100, gt=0) + + @model_validator(mode="after") + def validate_sources(self): + if any( + source.weight is not None or source.batches_per_selection is not None + for source in self.sources + ): + raise ValueError("IQL sources use uniform transition sampling") + return self + + @property + def trajectory_sources(self) -> list[str]: + return [value.source for value in self.sources] + + +Phase = TypeVar( + "Phase", + bound=TrainingPhaseSpecModel | PolicyImitationPhaseConfig | IQLPhaseConfig, +) class TrainingPlanModel(BaseModel, Generic[Phase]): @@ -133,8 +196,8 @@ def validate_unique_names(self): SupervisedTrainingPlanModel = TrainingPlanModel[TrainingPhaseSpecModel] -PolicyTrainingPlanModel = TrainingPlanModel[PolicyImitationPhaseConfig] +PolicyTrainingPlanModel = TrainingPlanModel[PolicyImitationPhaseConfig | IQLPhaseConfig] AlgorithmPhase = Annotated[ - TrainingPhaseSpecModel | PolicyImitationPhaseConfig, + TrainingPhaseSpecModel | PolicyImitationPhaseConfig | IQLPhaseConfig, Field(discriminator="algorithm"), ] diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 47afedca..3faa4d5a 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -951,8 +951,8 @@ See [policy learning](../policy-learning.md#optional-committed-import) and # Train Command Guide -For trajectory-backed categorical policy training and inference through the same -commands, see [policy learning](../policy-learning.md). +For trajectory-backed categorical BC, return-conditioned imitation, and offline +IQL through the same commands, see [policy learning](../policy-learning.md). `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated @@ -1721,7 +1721,7 @@ gradients are absent. ## Policy sequence training -`sequifier train` accepts one `policy_imitation` phase. Put the observation/action +`sequifier train` accepts one `policy_imitation` or `iql` phase. Put the observation/action schema in `model.interfaces..policy`, a sequence source in each `dataset_training..parts.`, and optimizer/context settings in `global_training` with `training_objective: policy`. Reference sources with @@ -1732,12 +1732,14 @@ Return-conditioned training additionally requires rewards and a known return endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories all use the same episode representation. Models support CPU/CUDA/MPS, backbone transfer via `initialize_from`, and shared-engine checkpoint resume via -`global_training.resume`. Mixed phases, IQL/PPO and image/depth policies remain +`global_training.resume`. IQL additionally requires complete transition data +and an unconditioned categorical policy. Mixed phases, PPO and image/depth policies remain planned. The former compact registries and configuration aliases have been consolidated into the shared model/dataset authoring surface. See the [complete table example](../example_inputs/configs/train-policy-from-table.yaml), -[return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml) +[return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml), +[IQL example](../example_inputs/configs/train-iql-from-table.yaml), and [policy learning guide](../policy-learning.md). @@ -2349,7 +2351,8 @@ optionally, a desired return. The current action is the supervised target. The phase algorithm is `policy_imitation` for both behavior cloning and return-conditioned imitation. The policy's `return_conditioned` setting selects -between them. +between them. Offline IQL uses a separate `iql` phase with an unconditioned +categorical actor. Use the existing `sequenceId` and `itemPosition` conventions. Each row describes **the current observation, the action executed at that observation, and the @@ -2494,7 +2497,8 @@ values are decoded using the original metadata, then fitted for the selected training partition. Missing reward/boundary information remains unknown. For a metadata-referenced prepared split, use `metadata_config_path` and `split_index` instead of `paths`. Scalar prepared histories are currently supported; -policy image/depth ingestion and IQL/PPO remain planned. +policy image/depth ingestion and PPO remain planned; IQL supports the scalar +transition-capable sources described below. A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy `manifest` listing version-1 JSON episodes. These are validated rollout recordings @@ -2556,7 +2560,7 @@ part containing a sequence source specification. Reference it with the ordinary `{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these parts do not need fabricated criteria. Model/interface references are validated. The shared phase schema preserves supervised defaults; this learner still supports -one `policy_imitation` phase, with BC selected on its policy. Mixed algorithms, +one `policy_imitation` or `iql` phase, with BC selected on its policy. Mixed algorithms, backbone repositories, image/depth policies and online learning remain planned. Use `initialize_from` for backbone artifact transfer. @@ -2782,6 +2786,69 @@ also accept policy model and run artifacts. The previous standalone numeric-policy artifacts (including versions 1 and 2) use a different architecture and must be retrained. This workflow currently supports one offline categorical actor phase. Mixed supervised/policy plans, -critics, IQL/PPO, distributed policy learning, custom run integrations, multimodal +PPO, distributed policy learning, custom run integrations, multimodal trajectory storage, and ONNX policy execution through the CLI remain outside this workflow. PyTorch policy execution supports CPU, CUDA, and MPS. + +## Offline IQL + +Use a single `iql` phase to improve an unconditioned categorical actor from a +fixed offline dataset. See the complete +[IQL configuration](example_inputs/configs/train-iql-from-table.yaml). The +actor can start from a compatible BC model with `initialize_from`. IQL creates +separate value, Q1, Q2, and target-Q networks; the portable model artifact +contains only the selected actor. The run checkpoint contains the full learner. +In this categorical slice, V and Q use independent copies of the configured +actor history encoder plus their own scalar heads; their architecture is not +authored in a separate model registry. + +Each selected decision needs a finite scalar reward, `terminated`, `truncated`, +and `bootstrap_allowed`. An ordinary nonterminal decision must bootstrap from +its genuine successor observation. An interior row may obtain that successor +from the following row; a final nonterminal or truncated row needs explicitly +mapped successor columns. A true terminal may omit its successor and cannot +bootstrap. Termination takes precedence when both boundary flags are true. +Unknown boundaries and missing rewards are errors for IQL even though BC can +use those demonstrations. Current observations and previous actions form the +actor/value history; Q alone receives the current recorded action. Splits, +categorical fitting, and numeric statistics come only from training episodes. + +`updates` is the number of optimizer cycles, each containing one value, critic, +and actor step. The phase controls uniform transition batch size, discount, +expectile, inverse temperature, maximum actor weight, target-Q update rate, +separate optimizer learning rates/specs and scheduler specs, gradient clipping, +and evaluation/checkpoint intervals. Schedulers advance after their own +optimizer update. The held-out action likelihood selects the portable actor; +the report also records value, Q, Bellman, advantage, and weight diagnostics. +The supported per-role optimizers are Adam, AdamW, SGD, and RMSprop. +`global_training.batch_size` and `global_training.learning_rate` remain required +by the shared policy config; the IQL phase's batch size and three learning rates +control its actual updates. +Use a separate application evaluation with `collect_policy_rollout` to compare +returns against BC in a local environment. +The [toy control example](example_inputs/iql_toy_control.py) generates a fixed +mixed-quality dataset for its [BC](example_inputs/configs/train-bc-toy.yaml) +and [IQL](example_inputs/configs/train-iql-toy.yaml) configs. Train BC first; +the IQL config warm-starts from its actor artifact. The example compares saved +actors over 100 fixed initial states and reports mean and standard deviation +rather than asserting an improvement for every seed. + +```console +python documentation/example_inputs/iql_toy_control.py generate data/offline-transitions.csv +sequifier train --config-path documentation/example_inputs/configs/train-bc-toy.yaml +sequifier train --config-path documentation/example_inputs/configs/train-iql-toy.yaml +python documentation/example_inputs/iql_toy_control.py evaluate models/toy-bc/toy-bc.pt models/toy-iql/toy-iql.pt +``` + +`global_training.resume: {policy: if_exists}` resumes the IQL learner from its +complete update checkpoint. `policy: required` needs an explicit +`checkpoint_path`. Exact resume validates the configuration, source content, +split, fitted vocabulary, and normalization. A BC model passed to +`initialize_from` is a warm start, not a learner checkpoint. IQL currently +supports scalar histories, one categorical action, and local execution; a +return-conditioned actor, distributed training, mixed precision, continuous or +joint actions, and mixed phase plans are rejected. + +Python callers can use `run_iql_training(config)` from `sequifier.rl_api` with a +validated `PolicyTrainConfig`. The return value contains the selected +`CategoricalPolicy`, recorded update metrics, and training/validation episode IDs. diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py new file mode 100644 index 00000000..cdb3997b --- /dev/null +++ b/src/sequifier/io/iql_dataset.py @@ -0,0 +1,130 @@ +"""Fixed offline transitions with causal current and successor contexts.""" + +import hashlib +import json +from bisect import bisect_right +from dataclasses import dataclass + +import torch + +from sequifier.io.batch import SequifierInputs +from sequifier.io.policy_window import PolicyWindowView, action_index +from sequifier.io.tensors import TensorRecord +from sequifier.io.values import numeric_scalar + + +@dataclass(frozen=True) +class IQLBatch(TensorRecord): + current: SequifierInputs + successor: SequifierInputs + action: torch.Tensor + reward: torch.Tensor + bootstrap: torch.Tensor + + +class IQLTransitionDataset: + """Index eligible decisions while retaining the source's bounded episode cache.""" + + def __init__(self, policy_dataset): + if policy_dataset.config.return_conditioned: + raise ValueError("IQL requires an unconditioned actor") + self.policy_dataset = policy_dataset + self.view = PolicyWindowView(policy_dataset.config) + self.cumulative_lengths = policy_dataset.cumulative_lengths + self.episode_ids = policy_dataset.episode_ids + digest = hashlib.sha256() + for episode_index, episode in enumerate(policy_dataset.episodes): + reference = policy_dataset.episodes.sources[episode_index] + source = ( + getattr(reference, "path", None) + or getattr(reference, "paths", None) + or episode_index + ) + transitions = episode.transitions + if transitions is None or any( + getattr(transitions, field) is None + for field in ("rewards", "terminated", "truncated", "bootstrap_allowed") + ): + raise ValueError( + f"IQL source {source!r} episode {episode.episode_id!r} " + "needs complete transition columns" + ) + for step in range(len(episode.actions)): + location = ( + f"IQL source {source!r} episode {episode.episode_id!r} " + f"step {episode.item_positions[step]}" + ) + reward = numeric_scalar(transitions.rewards[step], f"{location} reward") + if not torch.isfinite(torch.tensor(reward, dtype=torch.float32)): + raise ValueError(f"{location}: reward exceeds finite float32 range") + terminated = transitions.terminated[step] + bootstrap = transitions.bootstrap_allowed[step] + successor = self._successor(episode, step) + if terminated and bootstrap: + raise ValueError(f"{location}: termination forbids bootstrap") + if not terminated and not bootstrap: + raise ValueError( + f"{location}: nonterminal transition must bootstrap" + ) + if bootstrap and successor is None: + raise ValueError(f"{location}: bootstrap needs the true successor") + if successor is not None and set(successor) != set( + episode.observations + ): + raise ValueError( + f"{location}: successor fields differ from observations" + ) + if successor is not None and step + 1 < len(episode.actions): + if successor != episode.observation(step + 1): + raise ValueError( + f"{location}: successor breaks episode continuity" + ) + digest.update( + json.dumps(episode.model_dump(mode="json"), sort_keys=True).encode() + ) + self.data_contract = digest.hexdigest() + + @staticmethod + def _successor(episode, step): + transitions = episode.transitions + if transitions is not None and transitions.successors is not None: + supplied = transitions.successors[step] + if supplied is not None: + return supplied + if step + 1 < len(episode.actions): + return episode.observation(step + 1) + return None + + def __len__(self): + return len(self.policy_dataset) + + def __getitem__(self, index): + if index < 0: + index += len(self) + if not 0 <= index < len(self): + raise IndexError(index) + episode_index = bisect_right(self.cumulative_lengths, index) + step = index - ( + self.cumulative_lengths[episode_index - 1] if episode_index else 0 + ) + episode = self.policy_dataset.episodes[episode_index] + transitions = episode.transitions + successor = self._successor(episode, step) + current = self.view.episode_inputs(episode, step, None) + next_inputs = self.view.successor_inputs( + episode, + step, + successor if successor is not None else episode.observation(step), + ) + return IQLBatch( + current=current, + successor=next_inputs, + action=torch.tensor( + [action_index(episode.actions[step], self.policy_dataset.config)], + dtype=torch.long, + ), + reward=torch.tensor([transitions.rewards[step]], dtype=torch.float32), + bootstrap=torch.tensor( + [transitions.bootstrap_allowed[step]], dtype=torch.bool + ), + ) diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py index f3796791..727274eb 100644 --- a/src/sequifier/io/policy_window.py +++ b/src/sequifier/io/policy_window.py @@ -118,6 +118,24 @@ def episode_inputs( episode.static_inputs, ) + def successor_inputs(self, episode, step: int, successor) -> SequifierInputs: + """Build h_(t+1) from the true successor and action a_t.""" + start = max(0, step + 2 - self.config.context_length) + observations = [ + episode.observation(position) for position in range(start, step + 1) + ] + [successor] + previous = [ + ( + len(self.config.action_values) + if episode.starts_episode + else action_index(episode.previous_action, self.config) + ) + if position == 0 + else action_index(episode.actions[position - 1], self.config) + for position in range(start, step + 2) + ] + return self.inputs(observations, previous, None, episode.static_inputs) + def inputs( self, observations: Sequence[Mapping], diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 99059055..4cd3287a 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -148,6 +148,61 @@ desired_return: 10.0 """ +iql_train_config_string = """project_root: . +model_name: offline-iql +device: cpu +seed: 1010 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128} + positional_encoding_scope: global + interfaces: + actor: + input_columns: [OBSERVATION_COLUMN] + target_columns: [action] + policy: + observation_columns: [OBSERVATION_COLUMN] + action_values: [0.0, 1.0] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +dataset_training: + demonstrations: + model_interface: actor + parts: + recorded: + paths: [PLEASE_FILL.csv] + columns: + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN} +global_training: + training_objective: policy + context_length: 16 + batch_size: 64 + learning_rate: 0.005 + validation_fraction: 0.2 + resume: {policy: if_exists} +training_plan: + phases: + - name: offline_improvement + algorithm: iql + policy: actor + sources: [{source: demonstrations.recorded}] + updates: 10000 + batch_size: 64 +""" + gitignore_string = """models/ logs/ checkpoints/ @@ -185,3 +240,6 @@ def make(args): with open(f"{project_name}/configs/infer-policy.yaml", "w") as f: f.write(policy_infer_config_string) + + with open(f"{project_name}/configs/train-iql.yaml", "w") as f: + f.write(iql_train_config_string) diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py new file mode 100644 index 00000000..d17f5bd1 --- /dev/null +++ b/src/sequifier/rl/iql.py @@ -0,0 +1,526 @@ +"""Fixed-dataset categorical Implicit Q-Learning.""" + +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +import torch + +from sequifier.artifacts.run_checkpoint import checkpoint_path +from sequifier.io.iql_dataset import IQLBatch, IQLTransitionDataset +from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode +from sequifier.io.tensors import collate_tensor_records +from sequifier.model.factory import build_transformer_network +from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class +from sequifier.rl.checkpoint import load_policy, save_policy +from sequifier.rl.models import make_iql_models +from sequifier.rl.policy import CategoricalPolicy, policy_execution_config +from sequifier.runtime.random_state import RandomState, RandomStateManager + + +@dataclass(frozen=True) +class IQLTrainingResult: + policy: CategoricalPolicy + metrics: tuple[dict[str, float], ...] + updates: int + training_episode_ids: tuple[str, ...] + validation_episode_ids: tuple[str, ...] + + +def _select(inputs, mask): + return type(inputs)( + {name: value[mask] for name, value in inputs.inputs.items()}, + {name: value[mask] for name, value in inputs.metadata.items()}, + ) + + +def _optimizer(module, spec, learning_rate): + if "lr" in spec.arguments: + raise ValueError("IQL optimizer learning rate belongs in the phase field") + return get_optimizer_class(spec.name)( + [parameter for parameter in module.parameters() if parameter.requires_grad], + lr=learning_rate, + **spec.arguments, + ) + + +def _step(loss, optimizer, parameters, clip): + if not torch.isfinite(loss): + raise ValueError("non-finite IQL loss") + optimizer.zero_grad(set_to_none=True) + loss.backward() + norm = torch.nn.utils.clip_grad_norm_(parameters, clip, error_if_nonfinite=True) + optimizer.step() + if any(not torch.isfinite(parameter).all() for parameter in parameters): + raise ValueError("IQL optimizer produced non-finite parameters") + optimizer.zero_grad(set_to_none=True) + return float(norm.detach()) + + +def _polyak(target, source, rate): + with torch.no_grad(): + for target_parameter, parameter in zip( + target.parameters(), source.parameters() + ): + target_parameter.lerp_(parameter, rate) + for target_buffer, buffer in zip(target.buffers(), source.buffers()): + target_buffer.copy_(buffer) + + +def _checkpoint_payload( + config, + data, + fitted, + statistics, + actor, + value, + q1, + q2, + targets, + optimizers, + schedulers, + sampler, + metrics, + update, + best_loss, + best_actor, + random, +): + return { + "artifact_type": "sequifier_iql_run", + "format_version": 1, + "config": _resume_contract(config), + "data_contract": [part.data_contract for part in data], + "fitted_policy": fitted.model_dump(mode="json"), + "statistics": statistics, + "update": update, + "best_loss": best_loss, + "best_actor": best_actor, + "metrics": metrics, + "networks": { + name: module.state_dict() + for name, module in ( + ("actor", actor.network), + ("value", value), + ("q1", q1), + ("q2", q2), + ("target_q1", targets[0]), + ("target_q2", targets[1]), + ) + }, + "optimizers": { + name: optimizer.state_dict() for name, optimizer in optimizers.items() + }, + "schedulers": { + name: scheduler.state_dict() for name, scheduler in schedulers.items() + }, + "sampler": sampler.get_state(), + "random": random.capture_local().state_dict(), + } + + +def _resume_contract(config): + values = config.model_dump(mode="json") + values["global_training"].pop("resume", None) + return values + + +def _save_checkpoint(path, payload): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + try: + torch.save(payload, temporary) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def _evaluate(actor, value, q1, q2, dataset, batch_size, gamma, device): + totals = {"action_nll": 0.0, "bellman_residual": 0.0, "value": 0.0, "q_min": 0.0} + with torch.no_grad(): + for start in range(0, len(dataset), batch_size): + batch = cast( + IQLBatch, + collate_tensor_records( + [ + dataset[index] + for index in range(start, min(start + batch_size, len(dataset))) + ] + ), + ).to(device) + action = batch.action + v = value(batch.current) + q = torch.minimum(q1(batch.current, action), q2(batch.current, action)) + nll = -actor.evaluate_actions(batch.current, action) + successor_value = torch.zeros_like(v) + if batch.bootstrap.any(): + successor_value[batch.bootstrap] = value( + _select(batch.successor, batch.bootstrap) + ) + totals["action_nll"] += float(nll.sum()) + totals["bellman_residual"] += float( + (q - batch.reward - gamma * successor_value).abs().sum() + ) + totals["value"] += float(v.sum()) + totals["q_min"] += float(q.sum()) + return {key: value / len(dataset) for key, value in totals.items()} + + +def run_iql_training(config) -> IQLTrainingResult: + phase = config.training_plan.phases[0] + spec = config.policy + if spec.return_conditioned: + raise ValueError("IQL requires an unconditioned policy") + if config.global_training.distributed or config.global_training.data_parallelism: + raise ValueError("IQL currently requires local execution") + if config.global_training.layer_autocast: + raise ValueError("IQL mixed precision is not available in this slice") + if config.global_training.torch_compile != "none": + raise ValueError("IQL requires torch_compile: none") + if config.global_training.accumulation_steps not in (None, 1): + raise ValueError("IQL does not support gradient accumulation") + if config.global_training.num_workers: + raise ValueError( + "IQL samples transitions in the main process; num_workers must be zero" + ) + for role in ("actor", "value", "critic"): + optimizer = getattr(phase, f"{role}_optimizer") + if optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"}: + raise ValueError( + f"IQL {role} optimizer must be Adam, AdamW, SGD, or RMSprop" + ) + if "lr" in optimizer.arguments: + raise ValueError("IQL optimizer learning rates belong in phase fields") + scheduler = getattr(phase, f"{role}_scheduler") + if scheduler.name == "ReduceLROnPlateau": + raise ValueError("IQL schedulers advance on optimizer updates, not metrics") + checkpoint = checkpoint_path(config) + resume = config.global_training.resume + if resume is not None and resume.policy == "required" and not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + will_resume = ( + resume is not None and resume.policy != "never" and checkpoint.is_file() + ) + if ( + config.initialize_from is not None + and not will_resume + and not Path(config.initialize_from).is_file() + ): + raise FileNotFoundError(config.initialize_from) + source_names = phase.trajectory_sources + (phase.validation_sources or []) + from sequifier.trajectory.importers import source_paths + + sources = {name: source_paths(config.sources[name]) for name in source_names} + paths = [path for values in sources.values() for path in values] + protected = set(paths) + for name in source_names: + source = config.sources[name] + protected.update( + Path(value).resolve() + for value in ( + source.manifest, + source.metadata_config_path, + source.static_data_path, + ) + if value is not None + ) + if config.initialize_from is not None: + protected.add(Path(config.initialize_from).resolve()) + outputs = {Path(config.model_path), Path(config.report_path), checkpoint} + if len(outputs) != 3 or outputs.intersection(protected): + raise ValueError("IQL outputs must be distinct from trajectory inputs") + if len(paths) != len(set(paths)): + raise ValueError("IQL trajectory sources must not overlap") + + def corpus(names): + return PolicySequenceDataset( + [config.sources[name] for name in names], + spec, + episode_cache_size=config.global_training.episode_cache_size, + ) + + training = corpus(phase.trajectory_sources) + if phase.validation_sources is None: + training, validation = split_by_episode( + training, + config.global_training.validation_fraction, + config.seed, + group_identical=True, + ) + else: + validation = corpus(phase.validation_sources) + if not len(training) or not len(validation): + raise ValueError("IQL needs nonempty train and validation partitions") + for attribute in ("episode_ids", "group_ids", "demonstration_keys"): + if (set(getattr(training, attribute)) & set(getattr(validation, attribute))) - { + None + }: + raise ValueError(f"IQL {attribute} overlaps training and validation") + fitted = training.fit_categorical_config() + for part in (training, validation): + part.use_config(fitted) + data = (IQLTransitionDataset(training), IQLTransitionDataset(validation)) + statistics = training.normalization_statistics() + execution = policy_execution_config( + fitted, + config.model, + config.device, + global_training=config.global_training, + statistics=statistics, + project_root=config.project_root, + model_name=config.model_name, + seed=config.seed, + ) + device = torch.device(config.device) + torch.manual_seed(config.seed) + actor = CategoricalPolicy.from_network( + build_transformer_network(execution, device=device).network, + execution, + fitted.interface_name, + ) + value, q1, q2, targets = make_iql_models( + execution, fitted.interface_name, len(fitted.action_values) + ) + if config.initialize_from is not None and not will_resume: + source = load_policy(config.initialize_from, device=config.device) + if ( + source.config != fitted + or source.interface.selected_columns_statistics != statistics + ): + raise ValueError("IQL warm-start policy schema or normalization differs") + actor.network.load_state_dict(source.network.state_dict(), strict=True) + modules = { + "actor": actor.network, + "value": value, + "critic": torch.nn.ModuleList([q1, q2]), + } + trainable_ids = [ + id(parameter) + for module in modules.values() + for parameter in module.parameters() + if parameter.requires_grad + ] + if len(trainable_ids) != len(set(trainable_ids)): + raise ValueError("IQL optimizer parameter assignments overlap") + optimizers = { + role: _optimizer( + module, + getattr(phase, f"{role}_optimizer"), + getattr(phase, f"{role}_learning_rate"), + ) + for role, module in modules.items() + } + schedulers = { + role: get_scheduler_class(getattr(phase, f"{role}_scheduler").name)( + optimizer, **getattr(phase, f"{role}_scheduler").arguments + ) + for role, optimizer in optimizers.items() + } + for module in (actor, value, q1, q2, *targets): + module.eval() + sampler = torch.Generator(device="cpu").manual_seed(config.seed) + random = RandomStateManager(device) + metrics = [] + update_start = 0 + best_loss = math.inf + best_actor = None + if resume is not None and resume.policy != "never": + if checkpoint.exists(): + payload = torch.load(checkpoint, map_location="cpu", weights_only=False) + if not isinstance(payload, Mapping): + raise ValueError("IQL checkpoint must be a mapping") + if ( + payload.get("artifact_type") != "sequifier_iql_run" + or payload.get("format_version") != 1 + ): + raise ValueError("IQL resume requires an IQL learner checkpoint") + if ( + payload["config"] != _resume_contract(config) + or payload["data_contract"] != [part.data_contract for part in data] + or payload["fitted_policy"] != fitted.model_dump(mode="json") + or payload["statistics"] != statistics + ): + raise ValueError("IQL resume config, data, or codec differs") + for name, module in ( + ("actor", actor.network), + ("value", value), + ("q1", q1), + ("q2", q2), + ("target_q1", targets[0]), + ("target_q2", targets[1]), + ): + module.load_state_dict(payload["networks"][name], strict=True) + for name in optimizers: + optimizers[name].load_state_dict(payload["optimizers"][name]) + schedulers[name].load_state_dict(payload["schedulers"][name]) + sampler.set_state(payload["sampler"]) + random.restore(RandomState.from_state_dict(payload["random"])) + metrics = list(payload["metrics"]) + update_start = int(payload["update"]) + if not 0 <= update_start <= phase.updates: + raise ValueError( + "IQL checkpoint update is outside the configured budget" + ) + best_loss = float(payload["best_loss"]) + best_actor = payload["best_actor"] + if not math.isfinite(best_loss) and best_actor is not None: + raise ValueError("IQL checkpoint has an invalid best actor score") + + for update in range(update_start + 1, phase.updates + 1): + indices = torch.randint(len(data[0]), (phase.batch_size,), generator=sampler) + batch = cast( + IQLBatch, + collate_tensor_records([data[0][int(index)] for index in indices]), + ).to(device) + with torch.no_grad(): + qbar = torch.minimum( + targets[0](batch.current, batch.action), + targets[1](batch.current, batch.action), + ) + old_v = value(batch.current) + advantage = qbar - old_v + weight = torch.exp( + (phase.inverse_temperature * advantage).clamp( + max=math.log(phase.max_weight) + ) + ) + successor_value = torch.zeros_like(old_v) + if batch.bootstrap.any(): + successor_value[batch.bootstrap] = value( + _select(batch.successor, batch.bootstrap) + ) + target = batch.reward + phase.gamma * successor_value + delta = qbar - value(batch.current) + v_loss = ( + torch.where(delta < 0, 1 - phase.expectile, phase.expectile) + * delta.square() + ) + v_loss = v_loss.mean() + v_norm = _step( + v_loss, optimizers["value"], list(value.parameters()), phase.gradient_clip + ) + schedulers["value"].step() + q1_value, q2_value = ( + q1(batch.current, batch.action), + q2(batch.current, batch.action), + ) + q_loss = ((q1_value - target).square() + (q2_value - target).square()).mean() + q_norm = _step( + q_loss, + optimizers["critic"], + list(q1.parameters()) + list(q2.parameters()), + phase.gradient_clip, + ) + schedulers["critic"].step() + _polyak(targets[0], q1, phase.target_update_rate) + _polyak(targets[1], q2, phase.target_update_rate) + actor_loss = -( + weight * actor.evaluate_actions(batch.current, batch.action) + ).mean() + actor_norm = _step( + actor_loss, + optimizers["actor"], + list(actor.parameters()), + phase.gradient_clip, + ) + schedulers["actor"].step() + if update % phase.evaluation_interval == 0 or update == phase.updates: + held_out = _evaluate( + actor, value, q1, q2, data[1], phase.batch_size, phase.gamma, device + ) + record = { + "update": update, + "value_loss": float(v_loss.detach()), + "critic_loss": float(q_loss.detach()), + "actor_loss": float(actor_loss.detach()), + "advantage_mean": float(advantage.mean()), + "weight_mean": float(weight.mean()), + "weight_cap_fraction": float( + (weight >= phase.max_weight).float().mean() + ), + "advantage_p10": float(torch.quantile(advantage, 0.1)), + "advantage_p50": float(torch.quantile(advantage, 0.5)), + "advantage_p90": float(torch.quantile(advantage, 0.9)), + "weight_p10": float(torch.quantile(weight, 0.1)), + "weight_p50": float(torch.quantile(weight, 0.5)), + "weight_p90": float(torch.quantile(weight, 0.9)), + "value_min": float(old_v.min()), + "value_max": float(old_v.max()), + "q_min": float(qbar.min()), + "q_max": float(qbar.max()), + "actor_optimizer_step": update, + "value_optimizer_step": update, + "critic_optimizer_step": update, + "value_grad_norm": v_norm, + "critic_grad_norm": q_norm, + "actor_grad_norm": actor_norm, + **{f"validation_{name}": result for name, result in held_out.items()}, + } + metrics.append(record) + if held_out["action_nll"] < best_loss: + best_loss = held_out["action_nll"] + best_actor = { + name: tensor.detach().cpu().clone() + for name, tensor in actor.network.state_dict().items() + } + if update % phase.checkpoint_interval == 0 or update == phase.updates: + _save_checkpoint( + checkpoint, + _checkpoint_payload( + config, + data, + fitted, + statistics, + actor, + value, + q1, + q2, + targets, + optimizers, + schedulers, + sampler, + metrics, + update, + best_loss, + best_actor, + random, + ), + ) + if best_actor is None: + raise ValueError("IQL did not produce a finite held-out actor likelihood") + actor.network.load_state_dict(best_actor, strict=True) + report = { + "format": "sequifier.iql-training-report", + "version": 1, + "algorithm": "iql", + "config": config.model_dump(mode="json"), + "training_episode_ids": training.episode_ids, + "validation_episode_ids": validation.episode_ids, + "sources": { + name: [str(path) for path in values] for name, values in sources.items() + }, + "initialize_from": config.initialize_from, + "training_decisions": len(data[0]), + "validation_decisions": len(data[1]), + "best_validation_action_nll": best_loss, + "updates": phase.updates, + "metrics": metrics, + "model_path": config.model_path, + "checkpoint_path": str(checkpoint), + } + save_policy(actor, config.model_path, metadata=report) + destination = Path(config.report_path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + return IQLTrainingResult( + actor, + tuple(metrics), + phase.updates, + tuple(training.episode_ids), + tuple(validation.episode_ids), + ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 95ee5231..9de6cc9c 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -9,13 +9,18 @@ from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.logging_paths import dataset_artifact_prefix from sequifier.rl.checkpoint import save_policy +from sequifier.rl.iql import IQLTrainingResult, run_iql_training from sequifier.rl.training import PolicyTrainingResult, train_policy from sequifier.trajectory.importers import source_paths -def run_policy_training(config: PolicyTrainConfig) -> PolicyTrainingResult: +def run_policy_training( + config: PolicyTrainConfig, +) -> PolicyTrainingResult | IQLTrainingResult: """Prepare episode partitions, train, and publish the selected policy and report.""" phase = config.training_plan.phases[0] + if phase.algorithm == "iql": + return run_iql_training(config) spec = config.policy all_paths: dict[str, list[Path]] = {} for name in phase.trajectory_sources + (phase.validation_sources or []): diff --git a/src/sequifier/rl/models.py b/src/sequifier/rl/models.py new file mode 100644 index 00000000..3a4662f3 --- /dev/null +++ b/src/sequifier/rl/models.py @@ -0,0 +1,72 @@ +"""Independent history value and state-action critics for offline IQL.""" + +from copy import deepcopy + +import torch +from torch import nn + +from sequifier.io.batch import SequifierInputs +from sequifier.io.policy_window import normalize_policy_features +from sequifier.model.factory import build_transformer_network + + +class HistoryCritic(nn.Module): + def __init__(self, execution_config, interface_name: str, action_count: int | None): + super().__init__() + self.network = build_transformer_network( + execution_config, device=torch.device(execution_config.device) + ).network + self.interface_name = interface_name + self.interface = execution_config.interface(interface_name) + for parameter in self.network.resolve_interface( + interface_name + ).decoder.parameters(): + parameter.requires_grad_(False) + width = self.network.dim_model + self.action_embedding = ( + nn.Embedding(action_count, width) if action_count else None + ) + self.head = nn.Sequential( + nn.Linear(width * (2 if action_count else 1), width), + nn.ReLU(), + nn.Linear(width, 1), + ) + self.to(execution_config.device) + + def forward(self, history: SequifierInputs, action: torch.Tensor | None = None): + if (action is None) != (self.action_embedding is None): + raise ValueError("Q requires a current action and V forbids one") + features = normalize_policy_features( + history.inputs, history.metadata, self.interface + ) + representation = self.network.encode( + features, history.metadata, interface_name=self.interface_name + ) + state = self.network.backbone.final_norm(representation[:, -1]) + if self.action_embedding is not None: + if action is None: + raise ValueError("Q requires a current action") + if ( + action.dtype != torch.long + or action.ndim != 1 + or action.shape[0] != state.shape[0] + ): + raise ValueError("critic actions must be int64 [batch]") + state = torch.cat((state, self.action_embedding(action)), dim=-1) + return self.head(state).squeeze(-1).float() + + +def make_iql_models(execution_config, interface_name: str, action_count: int): + value = HistoryCritic(execution_config, interface_name, None) + q1 = HistoryCritic(execution_config, interface_name, action_count) + q2 = HistoryCritic(execution_config, interface_name, action_count) + targets = ( + deepcopy(q1).requires_grad_(False), + deepcopy(q2).requires_grad_(False), + ) + modules = (value, q1, q2, *targets) + if len( + {id(parameter) for module in modules for parameter in module.parameters()} + ) != sum(1 for module in modules for _ in module.parameters()): + raise ValueError("IQL networks share parameters") + return value, q1, q2, targets diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 0160829b..f2801759 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -180,6 +180,14 @@ def forward(self, inputs: SequifierInputs) -> Tensor: def distribution(self, inputs: SequifierInputs) -> CategoricalActionDistribution: return CategoricalActionDistribution(self(inputs)) + def act(self, inputs: SequifierInputs, *, deterministic: bool, generator): + distribution = CategoricalActionDistribution(self(inputs)[:, -1]) + action = distribution.select(deterministic, generator) + return action, distribution.log_prob(action) + + def evaluate_actions(self, inputs: SequifierInputs, actions: Tensor) -> Tensor: + return CategoricalActionDistribution(self(inputs)[:, -1]).log_prob(actions) + def validate_inputs(self, inputs: SequifierInputs) -> None: expected = set(self.interface.input_columns) if set(inputs.inputs) != expected: diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index bdc8c760..d08dfb72 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -38,6 +38,8 @@ def train_policy( integrations: IntegrationManager | None = None, ) -> PolicyTrainingResult: phase = run_config.training_plan.phases[0] + if phase.algorithm != "policy_imitation": + raise ValueError("train_policy requires a policy_imitation phase") if not len(training) or not len(validation): raise ValueError("policy training and validation partitions must be nonempty") if training.config != validation.config or training.config != run_config.policy: diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index ce56c10b..dfc7c0c5 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -12,7 +12,7 @@ ) from sequifier.config.sequence_import_config import SequenceImportConfig from sequifier.config.train_config import PolicyTrainConfig -from sequifier.config.training_plan import PolicyImitationPhaseConfig +from sequifier.config.training_plan import IQLPhaseConfig, PolicyImitationPhaseConfig from sequifier.io.batch import SequifierBatch, SequifierInputs, collate_sequence_batches from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.io.policy_window import PolicyWindowView, action_index @@ -22,6 +22,7 @@ CategoricalActionDistribution, masked_action_nll_loss, ) +from sequifier.rl.iql import IQLTrainingResult, run_iql_training from sequifier.rl.learner import run_policy_training from sequifier.rl.policy import CategoricalPolicy from sequifier.rl.runtime import PolicyExecutor, collect_policy_rollout @@ -45,7 +46,10 @@ "PolicyInferenceConfig", "PolicyTrainConfig", "PolicyImitationPhaseConfig", + "IQLPhaseConfig", "run_policy_training", + "run_iql_training", + "IQLTrainingResult", "CategoricalPolicyConfig", "ReturnDefinition", "CategoricalActionDistribution", diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 10f4fd8c..496fb01c 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -179,7 +179,7 @@ def run_training( from sequifier.rl.learner import run_policy_training configure_logger(config.project_root, config.model_name, 0) - configure_determinism(config.seed) + configure_determinism(config.seed, config.global_training.enforce_determinism) run_policy_training(config) return if not isinstance(config, TrainModel): From 080cad4aface1aa64840bbc7bf0012149ac9e01c Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 11:28:45 +0100 Subject: [PATCH 14/71] Add PPO --- README.md | 2 +- documentation/configs/train.md | 60 ++- .../example_inputs/configs/infer-ppo-toy.yaml | 9 + .../configs/rollout-ppo-toy.yaml | 20 + .../example_inputs/configs/train-ppo-toy.yaml | 57 ++ .../example_inputs/ppo-observations.csv | 3 + .../example_inputs/ppo_toy_control.py | 30 ++ ...nforcement-learning-implementation-plan.md | 4 + .../plans/scoped-ppo-implementation-plan.md | 271 ++++++++++ documentation/policy-learning.md | 187 ++++++- release-notes/unreleased-policy-training.md | 28 +- src/sequifier/config/infer_config.py | 4 +- src/sequifier/config/train_config.py | 182 ++++++- src/sequifier/config/training_plan.py | 168 +++++- src/sequifier/consolidated-docs.md | 249 ++++++++- src/sequifier/make.py | 63 +++ src/sequifier/rl/control.py | 105 ++++ src/sequifier/rl/distributions.py | 3 +- src/sequifier/rl/environment.py | 188 +++++++ src/sequifier/rl/history.py | 192 +++++++ src/sequifier/rl/learner.py | 14 +- src/sequifier/rl/ppo.py | 499 ++++++++++++++++++ src/sequifier/rl/ppo_buffer.py | 298 +++++++++++ src/sequifier/rl/ppo_checkpoint.py | 158 ++++++ src/sequifier/rl/ppo_collector.py | 234 ++++++++ src/sequifier/rl/ppo_evaluation.py | 177 +++++++ src/sequifier/rl/ppo_update.py | 211 ++++++++ src/sequifier/rl/runtime.py | 36 +- src/sequifier/rl_api.py | 37 +- src/sequifier/train.py | 16 +- 30 files changed, 3447 insertions(+), 58 deletions(-) create mode 100644 documentation/example_inputs/configs/infer-ppo-toy.yaml create mode 100644 documentation/example_inputs/configs/rollout-ppo-toy.yaml create mode 100644 documentation/example_inputs/configs/train-ppo-toy.yaml create mode 100644 documentation/example_inputs/ppo-observations.csv create mode 100644 documentation/example_inputs/ppo_toy_control.py create mode 100644 documentation/plans/scoped-ppo-implementation-plan.md create mode 100644 src/sequifier/rl/control.py create mode 100644 src/sequifier/rl/environment.py create mode 100644 src/sequifier/rl/history.py create mode 100644 src/sequifier/rl/ppo.py create mode 100644 src/sequifier/rl/ppo_buffer.py create mode 100644 src/sequifier/rl/ppo_checkpoint.py create mode 100644 src/sequifier/rl/ppo_collector.py create mode 100644 src/sequifier/rl/ppo_evaluation.py create mode 100644 src/sequifier/rl/ppo_update.py diff --git a/README.md b/README.md index 3be2ce79..07438f64 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ The processing and visualization commands have dedicated documentation pages: - [preprocess documentation](./documentation/configs/preprocess.md) - [train documentation](./documentation/configs/train.md) -- [policy training](./documentation/policy-learning.md) +- [policy training](./documentation/policy-learning.md), including BC, offline IQL and [online categorical PPO](./documentation/policy-learning.md#online-categorical-ppo) - [infer documentation](./documentation/configs/infer.md) - [hyperparameter-search documentation](./documentation/configs/hyperparameter-search.md) - [visualize-training documentation](./documentation/commands/visualize-training.md) diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 776b6afb..5271ff93 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -1,7 +1,7 @@ # Train Command Guide For trajectory-backed categorical BC, return-conditioned imitation, and offline -IQL through the same commands, see [policy learning](../policy-learning.md). +IQL and online PPO through the same commands, see [policy learning](../policy-learning.md). `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated @@ -782,11 +782,65 @@ endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories al use the same episode representation. Models support CPU/CUDA/MPS, backbone transfer via `initialize_from`, and shared-engine checkpoint resume via `global_training.resume`. IQL additionally requires complete transition data -and an unconditioned categorical policy. Mixed phases, PPO and image/depth policies remain -planned. The former compact registries and configuration aliases have been +and an unconditioned categorical policy. PPO uses the separate online schema below. +Mixed phases and image/depth policies remain planned. The former compact registries and configuration aliases have been consolidated into the shared model/dataset authoring surface. See the [complete table example](../example_inputs/configs/train-policy-from-table.yaml), [return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml), [IQL example](../example_inputs/configs/train-iql-from-table.yaml), and [policy learning guide](../policy-learning.md). + +## Online PPO configuration + +Online PPO uses `PPOTrainConfig`, selected automatically by one `algorithm: ppo` +phase. It requires no datasets, sources, validation loaders, `epochs` or +`global_training`. See the [runnable YAML](../example_inputs/configs/train-ppo-toy.yaml) +and [adapter/resume guide](../policy-learning.md#online-categorical-ppo). + +Root fields are `project_root` (default `.`), `model_name`, `model`, +`context_length` (defaults to the backbone maximum), `device` (default `cpu`), +`seed` (default 1010, in `[0, 2**32)`), `enforce_determinism` (default true), +optional `initialize_from`, and optional `model_path`, `report_path`, +`checkpoint_path`. Paths resolve against `project_root`; outputs must be +distinct. Exactly one scalar policy interface is allowed and +`return_conditioned` must be false. Schema/class counts, action order, +normalization and architecture must be explicit and frozen. + +The phase accepts these fields; extra fields are rejected: + +| Fields | Default / contract | +| --- | --- | +| `algorithm`, `policy`, `name` | `ppo`, selected interface, `online_ppo` | +| `environment` | Required `factory: module:factory`, `config: {}`, `protocol_version: 2`, `units: {}` | +| `normalization` | Required `source: identity` or `statistics`; the latter requires exact real-column `statistics: {column: {mean: ..., std: ...}}` | +| `total_env_steps`, `rollout_length` | Required positive executed-action budgets | +| `allow_partial_rollout` | true; false requires a divisible total budget | +| `optimization_epochs`, `minibatch_size`, `accumulation_steps` | 4, 64, 1; all positive | +| `gamma`, `gae_lambda` | 0.99, 0.95; both in `[0, 1]` | +| `clip_epsilon`, `value_clip` | 0.2 in `(0, 1)`; null or positive value clip | +| `value_coefficient`, `entropy_coefficient` | 0.5 positive; 0.01 nonnegative | +| `target_kl`, `normalize_advantages` | null or positive KL limit; true | +| `reward_scale` | 1.0; fixed positive finite multiplier, checkpointed | +| `actor_learning_rate`, `value_learning_rate` | 0.0003 each, positive finite | +| `actor_optimizer`, `value_optimizer` | `{name: Adam}`; Adam/AdamW/SGD/RMSprop with constructor arguments; phase owns `lr` and parameters | +| `actor_scheduler`, `value_scheduler` | `{name: ConstantLR, factor: 1.0, total_iters: 1}`; scheduler constructor arguments, stepped after each role's optimizer step | +| `actor_gradient_clip`, `value_gradient_clip` | 1.0 each; positive finite norm limit | +| `evaluation_interval`, `checkpoint_interval` | 1 each, measured in complete iterations; final iteration always runs both | +| `evaluation.environment` | Required separate adapter factory/config, same frozen units as training | +| `evaluation.seeds` | Required nonempty list of integers in `[0, 2**63)` | +| `evaluation.max_episode_steps`, `evaluation.timeout_seconds` | Required positive per-episode action cap and whole-evaluation deadline | +| `evaluation.selection` | `deterministic` or explicitly `stochastic` | +| `evaluation.metric`, `evaluation.direction` | `mean_completed_return`, `max`; alternatives `mean_completed_length` / `success_rate`, direction `min` / `max` | +| `resume` | `{policy: never, mode: reset_environments}`; policy also `if_exists`/`required`, mode also `exact` | +| `seed_policy` | `derived_per_reset`; separate training/evaluation streams | +| `max_buffer_bytes` | 536870912; positive conservative host-buffer budget, estimated before collection and enforced per insertion | +| `archive_directory`, `archive_scores` | null (disabled), false; separate versioned collection records | + +`OneCycleLR.total_steps` must equal the maximum authored optimizer-step budget +(including accumulation and a final partial rollout); KL stopping may use fewer +steps. Plateau/compound schedulers, distributed modes, layer autocast and ONNX are not +PPO config fields. Optimizer and scheduler constructor errors are raised before +collection. A final shorter rollout executes exactly the remaining actions; +terminal actions count toward the budget. Checkpoint/export paths and resume +modes are algorithm-owned and do not change the supervised or offline schemas. diff --git a/documentation/example_inputs/configs/infer-ppo-toy.yaml b/documentation/example_inputs/configs/infer-ppo-toy.yaml new file mode 100644 index 00000000..8e5f2968 --- /dev/null +++ b/documentation/example_inputs/configs/infer-ppo-toy.yaml @@ -0,0 +1,9 @@ +project_root: . +model_path: models/online-ppo/online-ppo.pt +data_path: documentation/example_inputs/ppo-observations.csv +model_type: generative +read_format: csv +write_format: csv +device: cpu +deterministic: true +output_path: outputs/ppo-actions.csv diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml new file mode 100644 index 00000000..b101613a --- /dev/null +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -0,0 +1,20 @@ +project_root: . +model_path: models/online-ppo/online-ppo.pt +data_path: documentation/example_inputs/ppo-observations.csv +model_type: generative +read_format: csv +write_format: csv +device: cpu +deterministic: true +autoregressive: true +generation_steps: 8 +rollout: + mode: policy + adapter: + factory: sequifier.rl.control:ControlRolloutAdapter + config: {horizon: 8} + bindings: + - {input: cue, source: environment, key: cue} + trajectory_path: outputs/ppo-rollout.json + seed_index: 0 + max_episode_steps: 8 diff --git a/documentation/example_inputs/configs/train-ppo-toy.yaml b/documentation/example_inputs/configs/train-ppo-toy.yaml new file mode 100644 index 00000000..7dd574f2 --- /dev/null +++ b/documentation/example_inputs/configs/train-ppo-toy.yaml @@ -0,0 +1,57 @@ +project_root: . +model_name: online-ppo +device: cpu +seed: 1010 +context_length: 4 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 1 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 64} + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: [cue] + target_columns: [action] + policy: + observation_columns: [cue] + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +training_plan: + phases: + - name: online_control + algorithm: ppo + policy: actor + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + normalization: {source: identity} + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + actor_learning_rate: 0.0003 + value_learning_rate: 0.0003 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 + resume: {policy: if_exists, mode: exact} diff --git a/documentation/example_inputs/ppo-observations.csv b/documentation/example_inputs/ppo-observations.csv new file mode 100644 index 00000000..28a1a3cf --- /dev/null +++ b/documentation/example_inputs/ppo-observations.csv @@ -0,0 +1,3 @@ +cue +-1.0 +1.0 diff --git a/documentation/example_inputs/ppo_toy_control.py b/documentation/example_inputs/ppo_toy_control.py new file mode 100644 index 00000000..f18a29f0 --- /dev/null +++ b/documentation/example_inputs/ppo_toy_control.py @@ -0,0 +1,30 @@ +"""Python PPO workflow. Run from the repository root after installing Sequifier. + + python documentation/example_inputs/ppo_toy_control.py + +The bundled deterministic adapter is implemented in sequifier.rl.control. +Its random-policy expected return is 0; optimal eight-action return is 8. +This example trains a policy; it is not a test or a validation benchmark. +""" + +from typing import cast + +from sequifier.config.train_config import PPOTrainConfig, load_train_config +from sequifier.rl_api import evaluate_policy, load_policy, run_ppo_training + + +def main(): + loaded_config = load_train_config( + "documentation/example_inputs/configs/train-ppo-toy.yaml", {}, False + ) + if not isinstance(loaded_config, PPOTrainConfig): + raise TypeError("the example requires an online PPO config") + config = cast(PPOTrainConfig, loaded_config) + result = run_ppo_training(config) + actor = load_policy(config.model_path) + print("Best evaluated return:", result.best_metric) + print(evaluate_policy(actor, config.training_plan.phases[0].evaluation)) + + +if __name__ == "__main__": + main() diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md index ce750b4f..b05f82f4 100644 --- a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md +++ b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md @@ -56,6 +56,10 @@ learner checkpoint, and ordinary actor-only model artifacts. See the broader Stage 1/2 contracts below. No tests were written or run for this follow-up, as requested. +Scoped PPO follow-up: the [local categorical PPO implementation plan](scoped-ppo-implementation-plan.md) +defines the next implementation slice at the current BC/return-conditioned +product boundary. It does not replace the full Stage 3 requirements below. + Deliver this as one coordinated update, built and verified in the three dependency-ordered stages below. Completion means all three stages work together; stage boundaries are engineering checkpoints, not separate proposed releases. This document specifies new behavior and proposed API names, not functionality already present. ## 1. Outcome and scope diff --git a/documentation/plans/scoped-ppo-implementation-plan.md b/documentation/plans/scoped-ppo-implementation-plan.md new file mode 100644 index 00000000..5e524f9a --- /dev/null +++ b/documentation/plans/scoped-ppo-implementation-plan.md @@ -0,0 +1,271 @@ +# Scoped PPO implementation plan for the current policy-learning path + +Status: proposed implementation plan, 2026-11-09. This is a follow-up to the +[broader rollout and reinforcement-learning plan](rollout-and-reinforcement-learning-implementation-plan.md), +especially sections 3, 4, 5.1, 5.4, and 6. The target is **online PPO at the +current behavior-cloning and return-conditioned imitation product boundary**: +one scalar-observation, single-categorical-action policy can be trained through +`sequifier train`, evaluated in a local environment, checkpointed, exported as +the ordinary actor model, loaded by `sequifier infer`, and executed through the +same rollout machinery. This does not claim completion of full Stage 3. + +## 1. Baseline and scope + +The diff since `2d01d407cd2134c8197b1a18f0034d01486f18a9` adds opt-in +single-environment scalar forecast rollouts; validated version-1 JSON episode +records; common columnar episode imports from CSV/Parquet, prepared scalar data, +and JSON recordings; causal policy windows; one categorical distributional +actor; BC and return-conditioned imitation; ordinary actor artifacts and +format-v2 imitation run checkpoints; and policy inference/rollouts. The latest +commit adds one fixed-dataset IQL phase with V and twin Q/target Q networks, +transition sampling, weighted actor extraction, an IQL learner checkpoint, and +actor-only export. The IQL phase is single-process and single-categorical; it +does not add an online environment loop or PPO infrastructure. + +The current `collect_rollout` resets one seed-initialized episode, stops at an +episode end or budget, and always closes the adapter. `PolicyExecutor` builds +causal windows and samples actions, but its current execution path returns +domain action labels and records log-probabilities as side metadata. It does not +retain value estimates or a reusable live environment. `PolicyTrainConfig` +requires `dataset_training` and one offline phase; `load_train_config` rejects +PPO. The existing `HistoryCritic` can serve as a separate history-only V, while +the IQL Q networks and offline transition sampler are not PPO components. + +**Included:** one persistent in-process adapter with one environment slot; +scalar real or categorical observation/static fields already supported by the +policy codec; one fixed categorical action vocabulary; fixed-length causal +context; single-process CPU and supported local torch devices; synchronous +collect/update iterations; local deterministic rollout evaluation; explicit +resume behavior; Python and CLI entry points; actor-only portable artifacts. +An unconditioned BC or IQL actor may initialize PPO after strict compatibility +checks. PPO from scratch uses an explicit, frozen observation codec and +normalization declaration, including a deliberate identity-normalization +choice; it never fits statistics from its changing on-policy data. + +**Deferred:** vector/Gym/remote adapters, media/depth, continuous or joint +actions, DDP/FSDP, reward providers and delayed rewards, mixed phases or +same-run phase transfer, generalized rollout monitors/search, ONNX policy +execution, and sharded trajectory storage. Return-conditioned actors are +rejected for this PPO slice; dropping the desired-return input would silently +change the actor contract. Existing BC/return-conditioned/IQL behavior remains +available through their current routes. + +## 2. Contracts to settle first + +1. **Decision timing.** At decision `t`, actor and V read observations through + `o_t` and actions only through `a_(t-1)`. Execute `a_t`, then record its + reward, genuine `o_(t+1)`, `terminated`, and `truncated`. The successor V + window includes `o_(t+1)` and `a_t` as its previous action. Never use an + autoreset observation as that successor. Continue to permit both boundary + flags, with termination taking precedence for bootstrap. +2. **Environment lifecycle.** Add an opt-in online local adapter protocol (or + versioned extension to `EnvironmentAdapter`) whose `reset(seed)` returns the + initial observation and static values, and whose `step(action)` returns the + true successor, scalar finite reward, and boundary flags. One instance lives + from training start through all collect/update iterations and closes in + `finally`. Keep the existing seed-initialized, version-1 inference adapter + callable unchanged; provide a narrow compatibility wrapper where possible. + Reset only after a recorded boundary, before the next decision. A local + adapter may optionally declare and implement exact state snapshots; merely + having `state_dict` methods is insufficient. +3. **History and identity.** Move causal history maintenance into a per-slot + state object shared by `PolicyExecutor`, collection, and evaluation. It + stores the current observation, previous action/start token, static values, + attention mask, episode ID, step ID, and reset count. A collection horizon + cuts a live episode without resetting it. Derive seeds from run seed, + evaluation/training stream, and reset count; do not mix evaluation resets + into training RNG state. Assert that a policy version is constant within a + collected buffer; a live episode may span versions across buffers. +4. **Frozen representation.** Pin the observation/action schema, action-value + order, normalization statistics, context length, and deterministic input + transforms before the first reset. Use domain values at the adapter edge and + the existing codec for model tensors. Reject schema/shape/unit drift and + non-finite values before insertion. Disable dropout and stochastic + augmentation for collection and PPO scoring while retaining gradients in + scoring. Make `evaluate_actions` and `act` use FP32 probability math. +5. **Rewards and boundaries.** This slice optimizes the adapter's one scalar + reward with a fixed, checkpointed scale (default 1). Reject provisional or + delayed rewards. A true terminal has no bootstrap; a time-limit truncation + bootstraps only from its genuine final state; a collection cut is a fragment + and bootstraps without a reset. Reject a requested bootstrap when the true + successor is unavailable. + +**Gate A:** hand-authored resets, first/interior/final decisions, truncations, +cuts, and episode restarts prove alignment. Replacing current action, reward, +or future observations cannot change step-`t` actor/V outputs. Existing +seed-based rollout behavior and version-1 JSON loading still work. + +## 3. Dependency-ordered implementation packages + +### B. Persistent collection and on-policy data + +Build a `PPOCollector` over the shared history/adapter contract. At each +iteration freeze an actor/value snapshot and collect exactly `rollout_length` +decisions, except a declared final shorter iteration when `total_env_steps` is +not divisible by the horizon. The budget counts executed actions, including +terminal actions. A completed episode resets immediately for the next decision; +a live episode and its history continue into the next iteration. Collection +runs under `no_grad` and records one policy version for every decision. + +Add a sealed, bounded `PPORolloutBuffer` with decision references to a compact +observation/action history ring rather than a copied context window per step. +Each entry holds domain/encoded action, old joint log-probability, old V, +reward, true successor/final observation reference, termination/truncation, +bootstrap eligibility, episode/step identity, and policy version. Retain the +pre-buffer context needed to reconstruct the earliest decision window. Store +old V of each true successor or calculate it from the frozen collection value +network before the buffer is sealed. Memory use is bounded by the authored +horizon and context; estimate required memory before collection. Random +minibatch access reconstructs complete causal windows and preserves their +within-window time order. The buffer is immutable after target calculation and +cannot be reused for a later policy generation. + +Keep optional trajectory archival separate from the training buffer. Extend +the JSON episode/fragment record or introduce a small versioned collection +record so fragments retain preceding action/history, policy version, cutoff +reason, old log-probability/value when requested, and true final successor. +Write completed fragments atomically; preserve readers for existing version-1 +records. Archival is never silently fed back into an on-policy update. + +**Gate B:** collect across multiple short episodes and across a live episode +cut; verify exact executed-step budget, terminal reward, no lost/duplicated +actions, bounded storage, reproducible sampled actions, and reconstructed +windows equal collection windows before optimization. A buffer from an older +policy version is rejected. + +### C. Value model, GAE, and PPO math + +Reuse `CategoricalPolicy` and `CategoricalActionDistribution` for the actor. +Instantiate an independent history-only `HistoryCritic` for V; do not create +IQL's Q/target-Q networks. Validate disjoint actor/V trainable parameter IDs +and identical frozen observation codecs. Provide a single decision API that +returns domain action, encoded action, old log-probability, entropy, and V +from the same causal input. Re-scoring that exact action before the first +optimizer step must reproduce old log-probability and V within tolerance. + +Compute GAE in FP32 over the sealed buffer. With `b_t` meaning bootstrap is +allowed and `c_t` meaning the next stored decision is in the same episode and +same collection segment: + +```text +delta_t = reward_t + gamma * b_t * V_old(h_next) - V_old(h_t) +A_t = delta_t + gamma * lambda * c_t * A_(t+1) +target_t = stop_gradient(A_t + V_old(h_t)) +``` + +Set `(b,c)` to `(0,0)` at true termination, `(1,0)` at a time-limit truncation +with valid final observation, `(1,0)` at a live collection cut, and `(1,1)` +only for an ordinary transition followed by the next decision of this episode +within the buffer. Normalize advantages once over valid decisions for an +iteration when configured, never per minibatch. Keep old log-probabilities, +old values, advantages, and targets fixed for all optimization epochs. + +Implement clipped policy loss, independent value loss, and current-policy +categorical entropy. Value clipping around `V_old` is optional and off by +default. Calculate ratio from joint `logp_new - logp_old`; check it and all +losses for non-finite values. Implement optional target-KL early stopping and +gradient clipping. Use independent actor/V optimizers and schedulers with +explicit per-optimizer-step clocks. Correctly weight a smaller last minibatch +and gradient accumulation by valid decision count. Never construct a fake +supervised target or pass PPO loss through supervised `LossService`. + +**Gate C:** hand-computed positive/negative advantage clipping, value clipping, +entropy, KL, and GAE match reference values. Gradient ownership is disjoint. +The identity ratio is one before any update, including after a checkpoint +round trip. A one-iteration run has finite losses and actually changes actor +and V weights. + +### D. Online phase, execution, and evaluation + +Add a discriminated `PPOPhaseConfig` and an online environment source to +`config/training_plan.py`/`config/train_config.py`. Route it through +`load_train_config`, `train.py`, `rl/learner.py`, and a `rl/ppo.py` algorithm +runtime. It must not require `dataset_training`, offline `sources`, `epochs`, +validation loaders, or fabricated `global_training` batch counts. Preserve +legacy supervised and imitation defaults. For this slice require exactly one +PPO phase; mixed-phase scheduling remains a separate parent-plan item. + +The typed PPO schema includes `total_env_steps`, `rollout_length`, +`optimization_epochs`, minibatch and accumulation sizes, gamma, GAE lambda, +policy clip epsilon, optional value clip, value/entropy coefficients, optional +target KL, advantage normalization, actor/V optimizer and scheduler plans, +learning rates, gradient limits, evaluation/checkpoint intervals, adapter +factory/config, separate evaluation adapter factory/config, reset/seed policy, +normalization source, and resume mode. Validate ranges, field ownership, +unsupported distributions/devices/distribution modes, budget/remainder rules, +paths, and parameter ownership before a long collection starts. An optional +`initialize_from` actor must match schema, vocab, normalization, model +architecture, and context length exactly. Do not treat this as exact resume. + +Run evaluation on a separate local adapter instance with fixed seeds and a +frozen actor snapshot. Default to deterministic mode actions; allow explicitly +named stochastic evaluation. Report mean episode return, spread, lengths, +completed/censored counts, and success only when the adapter defines it. +Select the exported best actor by a declared evaluation metric/direction, +defaulting to mean completed-episode return; never select by PPO loss. If no +episode completes, report that the selection metric is unavailable rather than +treating censored return as complete. The evaluation budget and timeout must be +explicit so a nonterminating task cannot hang training. Expose the same +evaluator to Python callers and to the PPO CLI workflow; this does not yet +require generalized supervised/rollout monitor or +hyperparameter-search integration. + +**Gate D:** a clean PPO CLI run can start from a reset-only local environment, +evaluate without changing training RNG/history, choose a best actor by return, +write a report, and load that actor through the existing policy inference and +single-environment rollout APIs. Existing BC, return-conditioned, and IQL +configs still parse and dispatch as before. + +### E. Checkpoint, resume, and operational validation + +Write an algorithm-owned versioned PPO checkpoint at complete iteration +boundaries. Include actor/V specs and weights; both optimizer/scheduler states; +global environment steps, iteration and policy version; best actor/metric; +codec/statistics and config fingerprints; actor initialization provenance; +training/evaluation seed streams and all torch RNG states; and the live +history/episode counters. Checkpoint writes are atomic. Do not put V or +optimizers in the portable `sequifier_model` actor artifact. Keep reading the +existing supervised format-v2 and IQL checkpoints through their own routes. + +Support two declared resume modes. `exact` requires an adapter capability +declaration and successfully restored simulator/RNG state plus history; an +unsupported adapter fails preflight. `reset_environments` restores learner +state but resets the adapter and causal history, drops incomplete-episode +metrics, and marks the continuation as non-trajectory-equivalent. For a crash +inside collection or optimization, restart from the last committed iteration; +do not pretend the partial buffer was resumed. Validate incompatible config, +action codec, model, reward scale, or adapter identity before restore. + +Add a structured training report with counts, return/evaluation curves, +policy/value loss, entropy, clip fraction, approximate KL, explained variance, +gradient norms, ratio tails, invalid/censored episode counts, and separate +collection/update/evaluation timing. Add a minimal deterministic categorical +control adapter and PPO config to `documentation/example_inputs`, a `sequifier +make` template, Python API example, policy guide, config docs, consolidated +docs, and release notes. Document the exact scoped limitations and the two +resume semantics. + +**Gate E / done for this slice:** the fixed-seed control task improves over +its random policy over multiple seeds; a short deterministic CPU job stays in +CI. A clean train → evaluate → checkpoint → reload → infer → rollout workflow +passes. Interrupted versus uninterrupted exact-snapshot runs match within +device-appropriate tolerance; non-snapshot adapters reject exact resume and +continue in reset mode. Run the existing repository and external test suite +where available, and test the old supervised/BC/return-conditioned/IQL paths. +Record seeds, task rules, hardware, context length, model size, throughput, +and peak memory. Do not mark the broad parent plan complete from this slice. + +## 4. Implementation order and review boundaries + +| Package | Depends on | Reviewable result | +|---|---|---| +| A. Online contracts and causal state | Existing policy/rollout path | Reset/step/boundary fixtures, shared history state, frozen codec | +| B. Collector and buffer | A | Persistent single-slot collection, exact windows, versioned fragments | +| C. Value, GAE, PPO update | B | Independent V, numerical formulas, identity ratio, optimizer transactions | +| D. Config and evaluation | A–C | One online PPO phase, CLI/Python parity, return-based best actor | +| E. Checkpoints and validation | A–D | Resume modes, artifact/export, learning and regression evidence | + +The parent plan's work packages B/D/E/F/I/J remain broader than these scoped +packages. In particular, passing the gates above adds a usable local +categorical PPO path; it does not establish vector/remote/multimodal or +distributed PPO acceptance. diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index aa7f5411..273d025c 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -156,7 +156,7 @@ values are decoded using the original metadata, then fitted for the selected training partition. Missing reward/boundary information remains unknown. For a metadata-referenced prepared split, use `metadata_config_path` and `split_index` instead of `paths`. Scalar prepared histories are currently supported; -policy image/depth ingestion and PPO remain planned; IQL supports the scalar +policy image/depth ingestion remains planned; IQL and PPO support the scalar transition-capable sources described below. A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy @@ -444,8 +444,8 @@ also accept policy model and run artifacts. The previous standalone numeric-policy artifacts (including versions 1 and 2) use a different architecture and must be retrained. This workflow currently -supports one offline categorical actor phase. Mixed supervised/policy plans, -PPO, distributed policy learning, custom run integrations, multimodal +supports one categorical actor phase, including the online PPO route below. Mixed supervised/policy plans, +distributed policy learning, custom run integrations, multimodal trajectory storage, and ONNX policy execution through the CLI remain outside this workflow. PyTorch policy execution supports CPU, CUDA, and MPS. @@ -511,3 +511,184 @@ joint actions, and mixed phase plans are rejected. Python callers can use `run_iql_training(config)` from `sequifier.rl_api` with a validated `PolicyTrainConfig`. The return value contains the selected `CategoricalPolicy`, recorded update metrics, and training/validation episode IDs. + +## Online categorical PPO + +PPO runs one persistent local environment with one categorical action. Start +with the [complete control config](example_inputs/configs/train-ppo-toy.yaml): + +```sh +sequifier train --config-path documentation/example_inputs/configs/train-ppo-toy.yaml +``` + +`sequifier make` also writes `configs/train-ppo.yaml`. The PPO root schema is +`PPOTrainConfig`: it has `model`, `context_length`, `seed`, `device`, and exactly +one `training_plan.phases` entry with `algorithm: ppo`. It has no +`dataset_training`, offline `sources`, `epochs`, or `global_training`. Phase +fields own both optimizers and their update clocks. See the +[PPO configuration reference](configs/train.md#online-ppo-configuration) and +[Python workflow](example_inputs/ppo_toy_control.py). + +The bundled `sequifier.rl.control:CategoricalControlAdapter` alternates a scalar +cue between -1 and +1. Choosing left for -1 and right for +1 earns +1; the other +action earns -1. Each episode executes eight actions. A random policy's expected +return is zero; the optimum is eight. The example config uses CPU, seed 1010, +context length 4, a 32-dimensional one-layer transformer, 256 decisions per +collection, and 8,192 total actions. The training report records hardware, +parameter counts, actual throughput and peak process memory. Learning quality +and multiple-seed improvement have not been measured for this implementation; +no training or test jobs were run for this increment. No CI job was added, +because the implementation request explicitly excluded writing or running tests. + +### Online adapter contract + +A factory receives `environment.config` as keyword arguments. It must return a +fresh adapter with `protocol_version = 2`, implementing: + +```python +from sequifier.rl.environment import OnlineResetResult, OnlineStepResult + +class Environment: + protocol_version = 2 + supports_exact_snapshot = False + + def reset(self, seed: int) -> OnlineResetResult: + # Return initial domain values and episode-static inputs. + return OnlineResetResult({"observation": 0.0}) + + def step(self, action: dict) -> OnlineStepResult: + # Execute exactly once. Return the genuine successor, immediate finite + # scalar reward, and independent termination/truncation flags. + return OnlineStepResult({"observation": 1.0}, reward=1.0) + + def close(self): + pass +``` + +The actor reads observations through the current decision and previous actions +only. After executing an action, its genuine successor and that action enter +the next causal window. An environment must never substitute an autoreset +observation for a final successor. A true terminal may return `observation=None` +and never bootstraps; truncation and collection cuts require a genuine successor +and bootstrap V from it. If both flags are true, termination wins. A horizon +cut preserves the live environment and its bounded causal history for the next +policy version. Only recorded episode boundaries trigger resets before the next +decision. Rewards must be final and immediate; delayed/provisional rewards and +reward providers are outside this contract. + +Observations and static fields must exactly match the frozen scalar schema. +Real values must fit finite FP32, categorical values must belong to declared +vocabularies, and units returned at every reset/step must equal the authored +`environment.units` mapping (empty for unitless values). Declare units for each +field whose units matter. Stochastic augmentation is unavailable. Dropout stays +disabled during collection and optimization, while scoring retains gradients. +The existing version-1 seeded inference adapter API is unchanged. +`SeededAdapterBridge` can wrap a v1 environment with same-named environment +observation bindings when an application supplies an initial-state factory. + +### Frozen representation and optimization + +The policy's observation/static schema, action order, class counts, label maps, +model architecture and context are fixed before the first reset. Author +`normalization: {source: identity}` deliberately, or use +`source: statistics` with a `statistics` map containing exactly the real fields' +`mean` and positive `std`. PPO never estimates statistics from changing online +data. `initialize_from` accepts an ordinary unconditioned BC/IQL actor only if +its policy schema, action ordering, normalization, model and context match +exactly. It initializes actor weights, not the learner state. A +return-conditioned actor is rejected. + +A separate history-only V owns independent parameters; PPO creates no Q or +target-Q networks. Each buffer stores a single policy generation, old action +log-probabilities, old values and true successor values. Compact frames retain +pre-buffer context; minibatches reconstruct ordered, complete windows. +FP32 GAE breaks its recurrence at episode boundaries and horizon cuts, while +bootstrapping at nonterminal boundaries. Advantage normalization occurs once per +iteration. Old scores, advantages and targets stay fixed for all epochs. + +The objective uses clipped probability ratios, independent squared value loss, +and current-policy categorical entropy. Optional value clipping is centered on +old V; it defaults off. Probability math is FP32. Gradient accumulation and the +last partial minibatch are weighted by their number of actual decisions. Actor +and V schedulers advance independently after their optimizer steps. Optional +KL limits stop further optimizer groups. Non-finite inputs, scores, losses, +gradients and parameters fail the run instead of admitting invalid decisions. + +### Evaluation, artifacts and resuming + +Evaluation uses a fresh adapter, frozen actor snapshot, fixed evaluation seeds, +and separate action/reset seed streams. It restores Python, NumPy and torch RNG +state afterward. Default evaluation selects mode actions; use +`selection: stochastic` explicitly for sampled actions. `max_episode_steps` +and `timeout_seconds` are mandatory. Evaluation runs on the main thread of a +local POSIX process, with an interval-timer deadline across creation, resets +and steps. Adapter methods must be interruptible and `close()` must return +promptly. A factory must return a different object from the live training slot. + +Reports include completed returns and their spread, episode lengths, completed +and censored counts, and success rates only when the adapter supplies success. +An environment truncation or evaluation step cap is censored; it does not count +as a completed episode. Best-actor selection defaults to maximizing +`mean_completed_return`; selection can instead use `mean_completed_length` or +`success_rate` with an explicit direction. No completed episodes means the +return metric is unavailable. If no evaluation supplies the selection metric, +the latest actor is exported with `latest_actor_metric_unavailable` recorded; +it is not described as a best actor. Initial evaluation also participates in +best-actor selection. + +The default outputs are `models//.pt`, +`outputs/training//report.json`, and `checkpoints//ppo.pt`. +The portable model contains actor weights only and loads with `load_policy`, +`sequifier infer`, and `collect_policy_rollout`. The example includes ordinary +[inference](example_inputs/configs/infer-ppo-toy.yaml) and seeded +[rollout](example_inputs/configs/rollout-ppo-toy.yaml) configs: + +```sh +sequifier infer --config-path documentation/example_inputs/configs/infer-ppo-toy.yaml +sequifier infer --config-path documentation/example_inputs/configs/rollout-ppo-toy.yaml +``` + The learner checkpoint contains +actor/V weights and specs, both optimizers/schedulers and step clocks, the +frozen codec, config fingerprint, best actor, seed/RNG state, live history and +episode counters. Checkpoints commit atomically only at complete iteration +boundaries. A crash during collection or optimization restarts from the last +committed iteration; partial buffers are not resumed. + +Set the phase's `resume.policy` to `never`, `if_exists`, or `required` and choose +one of these declared `resume.mode` values: + +- `exact` requires `supports_exact_snapshot = True` and working + `snapshot_state()` / `restore_state(state)` methods. The snapshot must include + simulator and simulator RNG state using dictionaries, lists, tuples, finite + scalars, numeric NumPy arrays and torch tensors. Restore is checked against a fresh snapshot. + Merely exposing `state_dict` does not declare this capability. Exact mode + rejects unsupported adapters before collection. It preserves the live + history and episode counters; equivalence still depends on deterministic + adapter and device operations. +- `reset_environments` restores learner, action sampler and optimizer state, + then resets the environment and causal history. It discards incomplete + episode metrics and records a non-trajectory-equivalent continuation. Reset + counts continue forward, so the next reset uses a new derived seed. + +Resume validates the full run contract, including budget, model, codec, reward +scale, adapter identity/config and normalization. Resume policy and mode may +change; other changes require a new run. Evaluation seeds never affect training +reset counts. Optional `archive_directory` writes atomic, version-1 +`sequifier.ppo-collection` records, with pre-fragment context, identities, +policy generation, cutoff reason, raw/scaled rewards and true successors. `archive_scores: true` +also retains old log-probability, entropy and values. The independent +`load_collection_record` reader does not change existing v1 JSON episode +readers. Archives are never fed back into online updates. + +The report records return/evaluation curves, policy/value losses, entropy, clip +fraction, approximate KL, explained variance, gradient norms, ratio tails, +optimizer clocks, counts and separate collection/update/evaluation timing. +Invalid data fails immediately; successful reports therefore have zero invalid +episodes. A nonterminated final training episode is explicitly censored. + +This scope is one in-process adapter slot and one fixed categorical action on +CPU or a supported local CUDA/MPS device. Vector/Gym/remote adapters, +continuous/joint actions, media/depth, distributed learning, reward providers, +mixed phases, same-run phase transfer, generalized monitors/search and ONNX +policy execution remain deferred. This does not complete the broader rollout +and reinforcement-learning plan. diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index 96e64e1f..dad71d42 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -4,7 +4,7 @@ Offline IQL now accepts one fixed-dataset `iql` phase for an unconditioned, single-categorical actor. It adds strict transition/successor validation, independent value and twin-Q networks, target-Q updates, weighted actor extraction, a complete learner checkpoint, and actor-only model export. -Continuous/joint actions, PPO, distributed IQL, and mixed plans remain future +Continuous/joint actions, distributed IQL, and mixed plans remain future work. No tests were written or run for this increment, as requested. For imitation, `sequifier train` accepts one trajectory-backed `policy_imitation` phase; @@ -26,8 +26,8 @@ and context-sized window construction. Behavior cloning retains observed state/action pairs from truncated and cutoff episodes without return labels. Grouped episode splitting and training-only normalization remain in place. -`sequifier make` includes policy train, IQL train, and infer templates. Mixed algorithm -phases, PPO, distributed policy learning, multimodal trajectories, +`sequifier make` includes policy train, IQL train, PPO train, and infer templates. Mixed algorithm +phases, distributed policy learning, multimodal trajectories, and CLI ONNX policy execution remain future work. Policy sequence representation follow-up: @@ -59,3 +59,25 @@ Architecture consolidation: - Sequence import writes columnar episodes directly, retaining optional transitions. - Earlier private policy run checkpoints require a new run initialized from their model weights. Portable models and execution weights remain readable. + + +Scoped online PPO: + +- A single `ppo` phase trains an unconditioned categorical actor against one + persistent local version-2 adapter, without datasets or supervised budgets. +- Shared bounded causal histories, compact sealed buffers, true-successor + bootstrap, FP32 GAE and ratios, clipped policy/value objectives, entropy, + sample-weighted gradient accumulation and independent actor/V optimizers. +- Frozen codecs and explicit normalization; strict BC/IQL actor warm starts. +- Separate seeded, bounded rollout evaluation selects actors by completed return + or another declared environment metric, with censored metrics kept explicit. +- Atomic version-1 PPO learner checkpoints support declared exact simulator/RNG + snapshots or non-trajectory-equivalent environment resets. Actor export stays + an ordinary portable `sequifier_model` and includes no learner networks. +- Structured reports, optional independent collection archives, a deterministic + categorical control adapter, CLI template and Python workflow are included. +- No tests were written or run. No training/benchmark runs or learning claims + accompany this implementation; only static validation was performed. +- Vector/remote/Gym adapters, continuous/joint actions, multimodal/distributed + PPO, mixed phases, reward providers, generalized monitors/search and ONNX + policy execution remain deferred. The parent RL plan remains incomplete. diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 1c878017..10235351 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -440,9 +440,9 @@ def load_inferer_config( from sequifier.config.train_config import load_train_config training = load_train_config(training_config_path, {}, False) - from sequifier.config.train_config import PolicyTrainConfig + from sequifier.config.train_config import PolicyTrainConfig, PPOTrainConfig - if isinstance(training, PolicyTrainConfig): + if isinstance(training, (PolicyTrainConfig, PPOTrainConfig)): raise ValueError( "policy inference requires model_path pointing to its portable artifact" ) diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 5e08c5a6..3739ddf2 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -10,6 +10,7 @@ import copy import inspect +import json import math import os import warnings @@ -68,7 +69,7 @@ PolicyDatasetConfig, resolve_sequence_source, ) -from sequifier.config.training_plan import PolicyTrainingPlanModel +from sequifier.config.training_plan import PolicyTrainingPlanModel, PPOTrainingPlanModel from sequifier.config.training_plan import ( SupervisedTrainingPlanModel as TrainingPlanModel, ) @@ -899,6 +900,170 @@ def resolve(self): return self +class PPOTrainConfig(BaseModel): + """Online learning has no dataset loaders or supervised optimization fields.""" + + model_config = ConfigDict(extra="forbid") + project_root: str = "." + model_name: NonemptyStr = Field(pattern=r"^[^/\\]+$") + device: str = "cpu" + seed: int = Field(default=1010, ge=0, lt=2**32) + enforce_determinism: bool = True + context_length: int = Field(gt=0) + initialize_from: str | None = None + model: ModelSpecModel + training_plan: PPOTrainingPlanModel + model_path: NonemptyStr + report_path: NonemptyStr + checkpoint_path: NonemptyStr + + @model_validator(mode="before") + @classmethod + def defaults(cls, values): + values = dict(values) + model = ModelSpecModel.model_validate(values.get("model")) + values["model"] = model + values.setdefault( + "context_length", model.backbone.architecture.max_context_length + ) + root = Path(values.get("project_root", ".")).resolve() + values["project_root"] = str(root) + name = values.get("model_name", "") + for key, default in ( + ("model_path", f"models/{name}/{name}.pt"), + ("report_path", f"outputs/training/{name}/report.json"), + ("checkpoint_path", f"checkpoints/{name}/ppo.pt"), + ): + if values.get(key) == "": + raise ValueError("PPO output paths must be nonempty") + values[key] = str((root / (values.get(key) or default)).resolve()) + if values.get("initialize_from") is not None: + values["initialize_from"] = str( + (root / values["initialize_from"]).resolve() + ) + return values + + @property + def policy(self) -> CategoricalPolicyConfig: + name = self.training_plan.phases[0].policy + schema = self.model.interfaces[name].policy + if schema is None: + raise ValueError("PPO requires a categorical policy schema") + return CategoricalPolicyConfig( + **schema.model_dump(), + interface_name=name, + context_length=self.context_length, + ) + + @property + def normalization_statistics(self): + declaration = self.training_plan.phases[0].normalization + real = ( + set(self.policy.observation_columns + self.policy.static_columns) + - self.policy.categorical_n_classes.keys() + ) + if declaration.source == "identity": + return {column: {"mean": 0.0, "std": 1.0} for column in sorted(real)} + if set(declaration.statistics) != real: + raise ValueError("PPO statistics must cover exactly the real policy fields") + return declaration.statistics + + @model_validator(mode="after") + def resolve(self): + if self.model_name in {".", ".."} or len(self.training_plan.phases) != 1: + raise ValueError("PPO requires a model name and exactly one online phase") + phase = self.training_plan.phases[0] + if set(self.model.interfaces) != {phase.policy}: + raise ValueError("PPO requires exactly its selected actor interface") + policy = self.policy + interface = self.model.interfaces[phase.policy] + if policy.return_conditioned: + raise ValueError( + "PPO cannot drop the desired-return contract; use an unconditioned actor" + ) + if set(policy.categorical_columns) - policy.categorical_n_classes.keys(): + raise ValueError( + "PPO categorical fields require explicit frozen class counts" + ) + if ( + interface.input_columns + != policy.observation_columns + policy.static_columns + or interface.target_columns != [policy.action_column] + ): + raise ValueError( + "PPO interface columns must match its policy schema in order" + ) + if interface.decoder.support != 1 or interface.decoder.prediction_length != 1: + raise ValueError("PPO requires a one-step scalar categorical decoder") + if any( + ( + interface.categorical_hashing, + interface.static_conditioning, + interface.preconditioning_image_encoder, + interface.feature_layout, + ) + ): + raise ValueError("PPO supports only scalar observation and static fields") + if self.model.backbone.repository is not None: + raise ValueError("PPO actor initialization uses initialize_from") + if self.context_length > self.model.backbone.architecture.max_context_length: + raise ValueError("PPO context exceeds the model context limit") + device = torch.device(self.device) + if device.type not in {"cpu", "cuda", "mps"}: + raise ValueError("PPO requires a supported local torch device") + if not self.normalization_statistics and any( + column not in policy.categorical_n_classes + for column in policy.observation_columns + policy.static_columns + ): + raise ValueError("real PPO fields require frozen normalization") + json.dumps(self.model_dump(mode="json"), allow_nan=False) + outputs = [self.model_path, self.report_path, self.checkpoint_path] + if len(set(outputs)) != len(outputs) or self.initialize_from in outputs: + raise ValueError( + "PPO outputs must be distinct from each other and actor initialization" + ) + for output in outputs: + destination = Path(output) + if destination.is_dir() or any( + parent.exists() and not parent.is_dir() + for parent in destination.parents + ): + raise ValueError("PPO output paths require writable file destinations") + if any( + destination in Path(other).parents + for other in outputs + if other != output + ): + raise ValueError("PPO output files cannot contain another output path") + for environment in (phase.environment, phase.evaluation.environment): + if set(environment.units) - set( + policy.observation_columns + policy.static_columns + ): + raise ValueError("environment units must refer to policy fields") + if phase.environment.units != phase.evaluation.environment.units: + raise ValueError("training and evaluation units must match") + if phase.archive_directory is not None: + phase.archive_directory = str( + (Path(self.project_root) / phase.archive_directory).resolve() + ) + archive = Path(phase.archive_directory) + if (archive.exists() and not archive.is_dir()) or any( + parent.exists() and not parent.is_dir() for parent in archive.parents + ): + raise ValueError( + "PPO archive_directory requires a directory destination" + ) + if any( + archive == Path(path) or archive in Path(path).parents + for path in outputs + + ([self.initialize_from] if self.initialize_from else []) + ): + raise ValueError( + "PPO archive directory must be separate from artifacts" + ) + return self + + class DatasetPartSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -1458,8 +1623,8 @@ def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: @dataclass(frozen=True) class LoadedTrainConfig: - config: SequifierConfig | PolicyTrainConfig - resolved: ResolvedSequifierConfig | PolicyTrainConfig + config: SequifierConfig | PolicyTrainConfig | PPOTrainConfig + resolved: ResolvedSequifierConfig | PolicyTrainConfig | PPOTrainConfig metadata: dict[str, DatasetMetadata] @@ -2340,16 +2505,17 @@ def load_train_config_with_source( if isinstance(phase, dict) } if algorithms - {"supervised"}: - if len(phases) != 1 or not algorithms <= {"policy_imitation", "iql"}: + if len(phases) != 1 or not algorithms <= {"policy_imitation", "iql", "ppo"}: raise ValueError( - "Policy training supports one policy_imitation or iql phase; " - "mixed plans and PPO are not yet implemented." + "Policy training supports exactly one policy_imitation, iql, or ppo phase." ) if sensitive or metadata_inline_values or skip_metadata: raise ValueError( "trajectory training does not accept supervised metadata overrides" ) - authored = PolicyTrainConfig.model_validate(raw) + authored = ( + PPOTrainConfig if algorithms == {"ppo"} else PolicyTrainConfig + ).model_validate(raw) return LoadedTrainConfig(config=authored, resolved=authored, metadata={}) config = try_catch_excess_keys(config_path, SequifierConfig, raw) @@ -2414,7 +2580,7 @@ def load_train_config_with_source( @beartype def load_train_config( config_path: str, args_config: dict[str, Any], skip_metadata: bool -) -> ResolvedSequifierConfig | PolicyTrainConfig: +) -> ResolvedSequifierConfig | PolicyTrainConfig | PPOTrainConfig: return load_train_config_with_source( config_path, args_config, skip_metadata ).resolved diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py index e859e158..34cab7a5 100644 --- a/src/sequifier/config/training_plan.py +++ b/src/sequifier/config/training_plan.py @@ -1,5 +1,6 @@ """Shared authored phase contracts; algorithm-specific budgets remain explicit.""" +import math from typing import Annotated, Generic, Literal, Optional, TypeVar from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -151,9 +152,168 @@ def trajectory_sources(self) -> list[str]: return [value.source for value in self.sources] +class OnlineEnvironmentConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + factory: str + config: dict = Field(default_factory=dict) + protocol_version: Literal[2] = 2 + # Empty means unitless; adapters must report this exact declaration. + units: dict[str, str] = Field(default_factory=dict) + + @field_validator("factory") + @classmethod + def validate_factory(cls, value): + module, separator, attribute = value.partition(":") + if not separator or not module or not attribute.isidentifier(): + raise ValueError("environment.factory must use module:factory") + return value + + +class PPONormalizationConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + source: Literal["identity", "statistics"] + statistics: dict[str, dict[str, float]] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_statistics(self): + if self.source == "identity" and self.statistics: + raise ValueError("identity normalization cannot include statistics") + for column, values in self.statistics.items(): + if ( + set(values) != {"mean", "std"} + or not all(math.isfinite(value) for value in values.values()) + or values["std"] <= 0 + ): + raise ValueError( + f"{column}: statistics require finite mean and positive std" + ) + return self + + +class PPOEvaluationConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + environment: OnlineEnvironmentConfig + seeds: list[int] = Field(min_length=1) + max_episode_steps: int = Field(gt=0) + timeout_seconds: float = Field(gt=0, allow_inf_nan=False) + selection: Literal["deterministic", "stochastic"] = "deterministic" + metric: Literal[ + "mean_completed_return", "success_rate", "mean_completed_length" + ] = "mean_completed_return" + direction: Literal["min", "max"] = "max" + + @field_validator("seeds") + @classmethod + def validate_seeds(cls, values): + if any(value < 0 or value >= 2**63 for value in values): + raise ValueError("evaluation seeds must be in [0, 2**63)") + return values + + +class PPOResumeConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + policy: Literal["never", "if_exists", "required"] = "never" + mode: Literal["exact", "reset_environments"] = "reset_environments" + + +class PPOPhaseConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + algorithm: Literal["ppo"] + name: str = Field(default="online_ppo", min_length=1) + policy: str = Field(min_length=1) + environment: OnlineEnvironmentConfig + evaluation: PPOEvaluationConfig + normalization: PPONormalizationConfig + resume: PPOResumeConfig = Field(default_factory=PPOResumeConfig) + seed_policy: Literal["derived_per_reset"] = "derived_per_reset" + total_env_steps: int = Field(gt=0) + rollout_length: int = Field(gt=0) + allow_partial_rollout: bool = True + optimization_epochs: int = Field(default=4, gt=0) + minibatch_size: int = Field(default=64, gt=0) + accumulation_steps: int = Field(default=1, gt=0) + gamma: float = Field(default=0.99, ge=0, le=1, allow_inf_nan=False) + gae_lambda: float = Field(default=0.95, ge=0, le=1, allow_inf_nan=False) + clip_epsilon: float = Field(default=0.2, gt=0, lt=1, allow_inf_nan=False) + value_clip: float | None = Field(default=None, gt=0, allow_inf_nan=False) + value_coefficient: float = Field(default=0.5, gt=0, allow_inf_nan=False) + entropy_coefficient: float = Field(default=0.01, ge=0, allow_inf_nan=False) + target_kl: float | None = Field(default=None, gt=0, allow_inf_nan=False) + normalize_advantages: bool = True + reward_scale: float = Field(default=1.0, gt=0, allow_inf_nan=False) + actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + actor_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + value_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + actor_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + value_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + actor_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) + value_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) + evaluation_interval: int = Field(default=1, gt=0) + checkpoint_interval: int = Field(default=1, gt=0) + archive_directory: str | None = Field(default=None, min_length=1) + archive_scores: bool = False + max_buffer_bytes: int = Field(default=536870912, gt=0) + + @model_validator(mode="after") + def validate_plans(self): + if ( + not self.allow_partial_rollout + and self.total_env_steps % self.rollout_length + ): + raise ValueError( + "rollout_length must divide total_env_steps unless allow_partial_rollout" + ) + for optimizer in (self.actor_optimizer, self.value_optimizer): + if optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"}: + raise ValueError("PPO optimizers must be Adam, AdamW, SGD, or RMSprop") + if "lr" in optimizer.arguments or "params" in optimizer.arguments: + raise ValueError( + "PPO learning rates and parameter ownership belong to the phase" + ) + full, remainder = divmod(self.total_env_steps, self.rollout_length) + group_size = self.minibatch_size * self.accumulation_steps + max_steps = self.optimization_epochs * ( + full * ((self.rollout_length + group_size - 1) // group_size) + + ((remainder + group_size - 1) // group_size if remainder else 0) + ) + for scheduler in (self.actor_scheduler, self.value_scheduler): + if ( + scheduler.name == "OneCycleLR" + and scheduler.arguments.get("total_steps") != max_steps + ): + raise ValueError( + "PPO OneCycleLR.total_steps must equal the maximum authored optimizer-step budget" + ) + if ( + scheduler.name + in {"ReduceLROnPlateau", "ChainedScheduler", "SequentialLR"} + or "optimizer" in scheduler.arguments + ): + raise ValueError( + "PPO schedulers must advance independently on optimizer steps" + ) + return self + + Phase = TypeVar( "Phase", - bound=TrainingPhaseSpecModel | PolicyImitationPhaseConfig | IQLPhaseConfig, + bound=TrainingPhaseSpecModel + | PolicyImitationPhaseConfig + | IQLPhaseConfig + | PPOPhaseConfig, ) @@ -197,7 +357,11 @@ def validate_unique_names(self): SupervisedTrainingPlanModel = TrainingPlanModel[TrainingPhaseSpecModel] PolicyTrainingPlanModel = TrainingPlanModel[PolicyImitationPhaseConfig | IQLPhaseConfig] +PPOTrainingPlanModel = TrainingPlanModel[PPOPhaseConfig] AlgorithmPhase = Annotated[ - TrainingPhaseSpecModel | PolicyImitationPhaseConfig | IQLPhaseConfig, + TrainingPhaseSpecModel + | PolicyImitationPhaseConfig + | IQLPhaseConfig + | PPOPhaseConfig, Field(discriminator="algorithm"), ] diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 3faa4d5a..4ccd3563 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -116,7 +116,7 @@ The processing and visualization commands have dedicated documentation pages: - [preprocess documentation](./documentation/configs/preprocess.md) - [train documentation](./documentation/configs/train.md) -- [policy training](./documentation/policy-learning.md) +- [policy training](./documentation/policy-learning.md), including BC, offline IQL and [online categorical PPO](./documentation/policy-learning.md#online-categorical-ppo) - [infer documentation](./documentation/configs/infer.md) - [hyperparameter-search documentation](./documentation/configs/hyperparameter-search.md) - [visualize-training documentation](./documentation/commands/visualize-training.md) @@ -952,7 +952,7 @@ See [policy learning](../policy-learning.md#optional-committed-import) and # Train Command Guide For trajectory-backed categorical BC, return-conditioned imitation, and offline -IQL through the same commands, see [policy learning](../policy-learning.md). +IQL and online PPO through the same commands, see [policy learning](../policy-learning.md). `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated @@ -1733,8 +1733,8 @@ endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories al use the same episode representation. Models support CPU/CUDA/MPS, backbone transfer via `initialize_from`, and shared-engine checkpoint resume via `global_training.resume`. IQL additionally requires complete transition data -and an unconditioned categorical policy. Mixed phases, PPO and image/depth policies remain -planned. The former compact registries and configuration aliases have been +and an unconditioned categorical policy. PPO uses the separate online schema below. +Mixed phases and image/depth policies remain planned. The former compact registries and configuration aliases have been consolidated into the shared model/dataset authoring surface. See the [complete table example](../example_inputs/configs/train-policy-from-table.yaml), @@ -1742,6 +1742,60 @@ See the [complete table example](../example_inputs/configs/train-policy-from-tab [IQL example](../example_inputs/configs/train-iql-from-table.yaml), and [policy learning guide](../policy-learning.md). +## Online PPO configuration + +Online PPO uses `PPOTrainConfig`, selected automatically by one `algorithm: ppo` +phase. It requires no datasets, sources, validation loaders, `epochs` or +`global_training`. See the [runnable YAML](../example_inputs/configs/train-ppo-toy.yaml) +and [adapter/resume guide](../policy-learning.md#online-categorical-ppo). + +Root fields are `project_root` (default `.`), `model_name`, `model`, +`context_length` (defaults to the backbone maximum), `device` (default `cpu`), +`seed` (default 1010, in `[0, 2**32)`), `enforce_determinism` (default true), +optional `initialize_from`, and optional `model_path`, `report_path`, +`checkpoint_path`. Paths resolve against `project_root`; outputs must be +distinct. Exactly one scalar policy interface is allowed and +`return_conditioned` must be false. Schema/class counts, action order, +normalization and architecture must be explicit and frozen. + +The phase accepts these fields; extra fields are rejected: + +| Fields | Default / contract | +| --- | --- | +| `algorithm`, `policy`, `name` | `ppo`, selected interface, `online_ppo` | +| `environment` | Required `factory: module:factory`, `config: {}`, `protocol_version: 2`, `units: {}` | +| `normalization` | Required `source: identity` or `statistics`; the latter requires exact real-column `statistics: {column: {mean: ..., std: ...}}` | +| `total_env_steps`, `rollout_length` | Required positive executed-action budgets | +| `allow_partial_rollout` | true; false requires a divisible total budget | +| `optimization_epochs`, `minibatch_size`, `accumulation_steps` | 4, 64, 1; all positive | +| `gamma`, `gae_lambda` | 0.99, 0.95; both in `[0, 1]` | +| `clip_epsilon`, `value_clip` | 0.2 in `(0, 1)`; null or positive value clip | +| `value_coefficient`, `entropy_coefficient` | 0.5 positive; 0.01 nonnegative | +| `target_kl`, `normalize_advantages` | null or positive KL limit; true | +| `reward_scale` | 1.0; fixed positive finite multiplier, checkpointed | +| `actor_learning_rate`, `value_learning_rate` | 0.0003 each, positive finite | +| `actor_optimizer`, `value_optimizer` | `{name: Adam}`; Adam/AdamW/SGD/RMSprop with constructor arguments; phase owns `lr` and parameters | +| `actor_scheduler`, `value_scheduler` | `{name: ConstantLR, factor: 1.0, total_iters: 1}`; scheduler constructor arguments, stepped after each role's optimizer step | +| `actor_gradient_clip`, `value_gradient_clip` | 1.0 each; positive finite norm limit | +| `evaluation_interval`, `checkpoint_interval` | 1 each, measured in complete iterations; final iteration always runs both | +| `evaluation.environment` | Required separate adapter factory/config, same frozen units as training | +| `evaluation.seeds` | Required nonempty list of integers in `[0, 2**63)` | +| `evaluation.max_episode_steps`, `evaluation.timeout_seconds` | Required positive per-episode action cap and whole-evaluation deadline | +| `evaluation.selection` | `deterministic` or explicitly `stochastic` | +| `evaluation.metric`, `evaluation.direction` | `mean_completed_return`, `max`; alternatives `mean_completed_length` / `success_rate`, direction `min` / `max` | +| `resume` | `{policy: never, mode: reset_environments}`; policy also `if_exists`/`required`, mode also `exact` | +| `seed_policy` | `derived_per_reset`; separate training/evaluation streams | +| `max_buffer_bytes` | 536870912; positive conservative host-buffer budget, estimated before collection and enforced per insertion | +| `archive_directory`, `archive_scores` | null (disabled), false; separate versioned collection records | + +`OneCycleLR.total_steps` must equal the maximum authored optimizer-step budget +(including accumulation and a final partial rollout); KL stopping may use fewer +steps. Plateau/compound schedulers, distributed modes, layer autocast and ONNX are not +PPO config fields. Optimizer and scheduler constructor errors are raised before +collection. A final shorter rollout executes exactly the remaining actions; +terminal actions count toward the budget. Checkpoint/export paths and resume +modes are algorithm-owned and do not change the supervised or offline schemas. + # Infer Command Guide @@ -2497,7 +2551,7 @@ values are decoded using the original metadata, then fitted for the selected training partition. Missing reward/boundary information remains unknown. For a metadata-referenced prepared split, use `metadata_config_path` and `split_index` instead of `paths`. Scalar prepared histories are currently supported; -policy image/depth ingestion and PPO remain planned; IQL supports the scalar +policy image/depth ingestion remains planned; IQL and PPO support the scalar transition-capable sources described below. A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy @@ -2785,8 +2839,8 @@ also accept policy model and run artifacts. The previous standalone numeric-policy artifacts (including versions 1 and 2) use a different architecture and must be retrained. This workflow currently -supports one offline categorical actor phase. Mixed supervised/policy plans, -PPO, distributed policy learning, custom run integrations, multimodal +supports one categorical actor phase, including the online PPO route below. Mixed supervised/policy plans, +distributed policy learning, custom run integrations, multimodal trajectory storage, and ONNX policy execution through the CLI remain outside this workflow. PyTorch policy execution supports CPU, CUDA, and MPS. @@ -2852,3 +2906,184 @@ joint actions, and mixed phase plans are rejected. Python callers can use `run_iql_training(config)` from `sequifier.rl_api` with a validated `PolicyTrainConfig`. The return value contains the selected `CategoricalPolicy`, recorded update metrics, and training/validation episode IDs. + +## Online categorical PPO + +PPO runs one persistent local environment with one categorical action. Start +with the [complete control config](example_inputs/configs/train-ppo-toy.yaml): + +```sh +sequifier train --config-path documentation/example_inputs/configs/train-ppo-toy.yaml +``` + +`sequifier make` also writes `configs/train-ppo.yaml`. The PPO root schema is +`PPOTrainConfig`: it has `model`, `context_length`, `seed`, `device`, and exactly +one `training_plan.phases` entry with `algorithm: ppo`. It has no +`dataset_training`, offline `sources`, `epochs`, or `global_training`. Phase +fields own both optimizers and their update clocks. See the +[PPO configuration reference](configs/train.md#online-ppo-configuration) and +[Python workflow](example_inputs/ppo_toy_control.py). + +The bundled `sequifier.rl.control:CategoricalControlAdapter` alternates a scalar +cue between -1 and +1. Choosing left for -1 and right for +1 earns +1; the other +action earns -1. Each episode executes eight actions. A random policy's expected +return is zero; the optimum is eight. The example config uses CPU, seed 1010, +context length 4, a 32-dimensional one-layer transformer, 256 decisions per +collection, and 8,192 total actions. The training report records hardware, +parameter counts, actual throughput and peak process memory. Learning quality +and multiple-seed improvement have not been measured for this implementation; +no training or test jobs were run for this increment. No CI job was added, +because the implementation request explicitly excluded writing or running tests. + +### Online adapter contract + +A factory receives `environment.config` as keyword arguments. It must return a +fresh adapter with `protocol_version = 2`, implementing: + +```python +from sequifier.rl.environment import OnlineResetResult, OnlineStepResult + +class Environment: + protocol_version = 2 + supports_exact_snapshot = False + + def reset(self, seed: int) -> OnlineResetResult: + # Return initial domain values and episode-static inputs. + return OnlineResetResult({"observation": 0.0}) + + def step(self, action: dict) -> OnlineStepResult: + # Execute exactly once. Return the genuine successor, immediate finite + # scalar reward, and independent termination/truncation flags. + return OnlineStepResult({"observation": 1.0}, reward=1.0) + + def close(self): + pass +``` + +The actor reads observations through the current decision and previous actions +only. After executing an action, its genuine successor and that action enter +the next causal window. An environment must never substitute an autoreset +observation for a final successor. A true terminal may return `observation=None` +and never bootstraps; truncation and collection cuts require a genuine successor +and bootstrap V from it. If both flags are true, termination wins. A horizon +cut preserves the live environment and its bounded causal history for the next +policy version. Only recorded episode boundaries trigger resets before the next +decision. Rewards must be final and immediate; delayed/provisional rewards and +reward providers are outside this contract. + +Observations and static fields must exactly match the frozen scalar schema. +Real values must fit finite FP32, categorical values must belong to declared +vocabularies, and units returned at every reset/step must equal the authored +`environment.units` mapping (empty for unitless values). Declare units for each +field whose units matter. Stochastic augmentation is unavailable. Dropout stays +disabled during collection and optimization, while scoring retains gradients. +The existing version-1 seeded inference adapter API is unchanged. +`SeededAdapterBridge` can wrap a v1 environment with same-named environment +observation bindings when an application supplies an initial-state factory. + +### Frozen representation and optimization + +The policy's observation/static schema, action order, class counts, label maps, +model architecture and context are fixed before the first reset. Author +`normalization: {source: identity}` deliberately, or use +`source: statistics` with a `statistics` map containing exactly the real fields' +`mean` and positive `std`. PPO never estimates statistics from changing online +data. `initialize_from` accepts an ordinary unconditioned BC/IQL actor only if +its policy schema, action ordering, normalization, model and context match +exactly. It initializes actor weights, not the learner state. A +return-conditioned actor is rejected. + +A separate history-only V owns independent parameters; PPO creates no Q or +target-Q networks. Each buffer stores a single policy generation, old action +log-probabilities, old values and true successor values. Compact frames retain +pre-buffer context; minibatches reconstruct ordered, complete windows. +FP32 GAE breaks its recurrence at episode boundaries and horizon cuts, while +bootstrapping at nonterminal boundaries. Advantage normalization occurs once per +iteration. Old scores, advantages and targets stay fixed for all epochs. + +The objective uses clipped probability ratios, independent squared value loss, +and current-policy categorical entropy. Optional value clipping is centered on +old V; it defaults off. Probability math is FP32. Gradient accumulation and the +last partial minibatch are weighted by their number of actual decisions. Actor +and V schedulers advance independently after their optimizer steps. Optional +KL limits stop further optimizer groups. Non-finite inputs, scores, losses, +gradients and parameters fail the run instead of admitting invalid decisions. + +### Evaluation, artifacts and resuming + +Evaluation uses a fresh adapter, frozen actor snapshot, fixed evaluation seeds, +and separate action/reset seed streams. It restores Python, NumPy and torch RNG +state afterward. Default evaluation selects mode actions; use +`selection: stochastic` explicitly for sampled actions. `max_episode_steps` +and `timeout_seconds` are mandatory. Evaluation runs on the main thread of a +local POSIX process, with an interval-timer deadline across creation, resets +and steps. Adapter methods must be interruptible and `close()` must return +promptly. A factory must return a different object from the live training slot. + +Reports include completed returns and their spread, episode lengths, completed +and censored counts, and success rates only when the adapter supplies success. +An environment truncation or evaluation step cap is censored; it does not count +as a completed episode. Best-actor selection defaults to maximizing +`mean_completed_return`; selection can instead use `mean_completed_length` or +`success_rate` with an explicit direction. No completed episodes means the +return metric is unavailable. If no evaluation supplies the selection metric, +the latest actor is exported with `latest_actor_metric_unavailable` recorded; +it is not described as a best actor. Initial evaluation also participates in +best-actor selection. + +The default outputs are `models//.pt`, +`outputs/training//report.json`, and `checkpoints//ppo.pt`. +The portable model contains actor weights only and loads with `load_policy`, +`sequifier infer`, and `collect_policy_rollout`. The example includes ordinary +[inference](example_inputs/configs/infer-ppo-toy.yaml) and seeded +[rollout](example_inputs/configs/rollout-ppo-toy.yaml) configs: + +```sh +sequifier infer --config-path documentation/example_inputs/configs/infer-ppo-toy.yaml +sequifier infer --config-path documentation/example_inputs/configs/rollout-ppo-toy.yaml +``` + The learner checkpoint contains +actor/V weights and specs, both optimizers/schedulers and step clocks, the +frozen codec, config fingerprint, best actor, seed/RNG state, live history and +episode counters. Checkpoints commit atomically only at complete iteration +boundaries. A crash during collection or optimization restarts from the last +committed iteration; partial buffers are not resumed. + +Set the phase's `resume.policy` to `never`, `if_exists`, or `required` and choose +one of these declared `resume.mode` values: + +- `exact` requires `supports_exact_snapshot = True` and working + `snapshot_state()` / `restore_state(state)` methods. The snapshot must include + simulator and simulator RNG state using dictionaries, lists, tuples, finite + scalars, numeric NumPy arrays and torch tensors. Restore is checked against a fresh snapshot. + Merely exposing `state_dict` does not declare this capability. Exact mode + rejects unsupported adapters before collection. It preserves the live + history and episode counters; equivalence still depends on deterministic + adapter and device operations. +- `reset_environments` restores learner, action sampler and optimizer state, + then resets the environment and causal history. It discards incomplete + episode metrics and records a non-trajectory-equivalent continuation. Reset + counts continue forward, so the next reset uses a new derived seed. + +Resume validates the full run contract, including budget, model, codec, reward +scale, adapter identity/config and normalization. Resume policy and mode may +change; other changes require a new run. Evaluation seeds never affect training +reset counts. Optional `archive_directory` writes atomic, version-1 +`sequifier.ppo-collection` records, with pre-fragment context, identities, +policy generation, cutoff reason, raw/scaled rewards and true successors. `archive_scores: true` +also retains old log-probability, entropy and values. The independent +`load_collection_record` reader does not change existing v1 JSON episode +readers. Archives are never fed back into online updates. + +The report records return/evaluation curves, policy/value losses, entropy, clip +fraction, approximate KL, explained variance, gradient norms, ratio tails, +optimizer clocks, counts and separate collection/update/evaluation timing. +Invalid data fails immediately; successful reports therefore have zero invalid +episodes. A nonterminated final training episode is explicitly censored. + +This scope is one in-process adapter slot and one fixed categorical action on +CPU or a supported local CUDA/MPS device. Vector/Gym/remote adapters, +continuous/joint actions, media/depth, distributed learning, reward providers, +mixed phases, same-run phase transfer, generalized monitors/search and ONNX +policy execution remain deferred. This does not complete the broader rollout +and reinforcement-learning plan. diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 4cd3287a..6b4e611e 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -203,6 +203,66 @@ batch_size: 64 """ +ppo_train_config_string = """project_root: . +model_name: online-ppo +device: cpu +seed: 1010 +context_length: 4 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 1 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 64} + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: [cue] + target_columns: [action] + policy: + observation_columns: [cue] + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +training_plan: + phases: + - name: online_control + algorithm: ppo + policy: actor + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + normalization: {source: identity} + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + actor_learning_rate: 0.0003 + value_learning_rate: 0.0003 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 + resume: {policy: if_exists, mode: exact} +""" + + gitignore_string = """models/ logs/ checkpoints/ @@ -243,3 +303,6 @@ def make(args): with open(f"{project_name}/configs/train-iql.yaml", "w") as f: f.write(iql_train_config_string) + + with open(f"{project_name}/configs/train-ppo.yaml", "w") as f: + f.write(ppo_train_config_string) diff --git a/src/sequifier/rl/control.py b/src/sequifier/rl/control.py new file mode 100644 index 00000000..b75f6d83 --- /dev/null +++ b/src/sequifier/rl/control.py @@ -0,0 +1,105 @@ +"""Minimal deterministic categorical control environment for local PPO examples. + +The observation alternates between -1 and +1. Matching its sign earns +1; +choosing the other action earns -1. An episode lasts eight actions by default. +A random policy has expected return zero and the optimal return is the horizon. +""" + +import random + +from sequifier.rl.environment import OnlineResetResult, OnlineStepResult +from sequifier.rollout.contracts import StepResult + + +class CategoricalControlAdapter: + protocol_version = 2 + supports_exact_snapshot = True + + def __init__(self, horizon=8): + if not isinstance(horizon, int) or isinstance(horizon, bool) or horizon <= 0: + raise ValueError("control horizon must be a positive integer") + self.horizon = horizon + self.step_id = 0 + self.cue = 1.0 + self.correct = 0 + self.ended = True + + def reset(self, seed): + self.cue = random.Random(seed).choice((-1.0, 1.0)) + self.step_id, self.correct = 0, 0 + self.ended = False + return OnlineResetResult({"cue": self.cue}) + + def step(self, action): + if ( + self.ended + or set(action) != {"action"} + or action["action"] not in {"left", "right"} + ): + raise ValueError( + "control requires a live episode and one left/right action" + ) + correct = action["action"] == ("right" if self.cue > 0 else "left") + self.correct += int(correct) + self.step_id += 1 + self.cue = -self.cue + self.ended = self.step_id == self.horizon + return OnlineStepResult( + {"cue": self.cue}, + 1.0 if correct else -1.0, + terminated=self.ended, + success=self.correct == self.horizon if self.ended else None, + ) + + def snapshot_state(self): + return { + "horizon": self.horizon, + "step_id": self.step_id, + "cue": self.cue, + "correct": self.correct, + "ended": self.ended, + } + + def restore_state(self, state): + if ( + state["horizon"] != self.horizon + or not 0 <= state["step_id"] <= self.horizon + or state["cue"] not in {-1.0, 1.0} + or not 0 <= state["correct"] <= state["step_id"] + or not isinstance(state["ended"], bool) + ): + raise ValueError("incompatible control snapshot") + self.step_id, self.cue = state["step_id"], state["cue"] + self.correct, self.ended = state["correct"], state["ended"] + + def close(self): + pass + + +class ControlRolloutAdapter: + """Version-1 bridge for the existing sequifier infer rollout machinery.""" + + protocol_version = 1 + + def __init__(self, horizon=8): + self.environment = CategoricalControlAdapter(horizon) + + def reset(self, request): + self.environment.reset(request.seed) + cue = request.seed_history["cue"][-1] + if cue not in {-1.0, 1.0}: + raise ValueError("control seed cue must be -1 or +1") + self.environment.cue = float(cue) + + def step(self, request): + if request.observation != {"cue": self.environment.cue}: + raise ValueError("control observation and environment disagree") + result = self.environment.step(request.predictions) + if result.observation is None: + raise ValueError("control must return the true successor") + return StepResult( + result.observation, result.reward, result.terminated, result.truncated + ) + + def close(self): + self.environment.close() diff --git a/src/sequifier/rl/distributions.py b/src/sequifier/rl/distributions.py index cebeef2b..283903c4 100644 --- a/src/sequifier/rl/distributions.py +++ b/src/sequifier/rl/distributions.py @@ -12,12 +12,13 @@ def action_generator(device: torch.device, seed: int) -> torch.Generator: class CategoricalActionDistribution: def __init__(self, logits: Tensor): + logits = logits.float() if logits.ndim < 2 or logits.shape[-1] < 2 or not torch.isfinite(logits).all(): raise ValueError( "categorical policy logits must be finite with at least two actions" ) self.logits = logits - self.distribution = torch.distributions.Categorical(logits=logits) + self.distribution = torch.distributions.Categorical(logits=self.logits) def mode(self) -> Tensor: return self.logits.argmax(dim=-1) diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py new file mode 100644 index 00000000..e11c455a --- /dev/null +++ b/src/sequifier/rl/environment.py @@ -0,0 +1,188 @@ +"""Version-2 online local adapter contract, independent of seeded v1 rollouts.""" + +from dataclasses import dataclass, field +from importlib import import_module +from typing import Mapping, Protocol + +from sequifier.io.values import DomainValue, numeric_scalar +from sequifier.rollout.contracts import ResetRequest, StepRequest + + +@dataclass(frozen=True) +class OnlineResetResult: + observation: Mapping[str, DomainValue] + static_inputs: Mapping[str, DomainValue] = field(default_factory=dict) + units: Mapping[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class OnlineStepResult: + # None is permitted only at a true terminal, where no bootstrap is requested. + observation: Mapping[str, DomainValue] | None + reward: float + terminated: bool = False + truncated: bool = False + units: Mapping[str, str] = field(default_factory=dict) + success: bool | None = None + + +class OnlineEnvironmentAdapter(Protocol): + protocol_version: int + supports_exact_snapshot: bool + + def reset(self, seed: int) -> OnlineResetResult: ... + def step(self, action: Mapping[str, DomainValue]) -> OnlineStepResult: ... + def close(self) -> None: ... + def snapshot_state(self) -> dict: ... + def restore_state(self, state: dict) -> None: ... + + +def load_online_adapter(spec): + module, attribute = spec.factory.split(":") + adapter = getattr(import_module(module), attribute)(**spec.config) + try: + if ( + not isinstance(getattr(adapter, "protocol_version", None), int) + or adapter.protocol_version != 2 + ): + raise ValueError( + "online adapters must explicitly declare protocol_version = 2" + ) + for method in ("reset", "step", "close"): + if not callable(getattr(adapter, method, None)): + raise ValueError(f"online adapter requires {method}") + return adapter + except BaseException: + close = getattr(adapter, "close", None) + if callable(close): + close() + raise + + +def require_exact_snapshot(adapter): + if getattr(adapter, "supports_exact_snapshot", None) is not True: + raise ValueError( + "exact resume requires an explicit supports_exact_snapshot capability" + ) + if not callable(getattr(adapter, "snapshot_state", None)) or not callable( + getattr(adapter, "restore_state", None) + ): + raise ValueError( + "exact snapshot adapters require snapshot_state and restore_state" + ) + + +def validate_reset(result, units): + if not isinstance(result, OnlineResetResult) or dict(result.units) != units: + raise ValueError( + "online reset requires an initial observation with the frozen units" + ) + return result + + +def validate_step(result, units): + if not isinstance(result, OnlineStepResult): + raise ValueError( + "online step must return OnlineStepResult with an immediate scalar reward" + ) + if not isinstance(result.terminated, bool) or not isinstance( + result.truncated, bool + ): + raise ValueError("online boundary flags must be booleans") + if dict(result.units) != units: + raise ValueError("online observation units changed") + numeric_scalar(result.reward, "online reward") + if result.observation is None and not result.terminated: + raise ValueError( + "bootstrap requires the genuine successor, never an autoreset observation" + ) + if result.success is not None and not isinstance(result.success, bool): + raise ValueError("adapter success must be boolean when defined") + return result + + +class SeededAdapterBridge: + """Expose a v1 adapter online when an explicit reset-state factory is available. + + The seed factory takes a seed and returns OnlineResetResult. All observations + must be produced by environment bindings with identical names. Snapshot + support is deliberately not inferred from the wrapped object. + """ + + protocol_version = 2 + supports_exact_snapshot = False + + def __init__( + self, adapter, seed_factory, action_column, context_length=1, units=None + ): + from sequifier.rollout.factories import validate_adapter + + validate_adapter(adapter) + if ( + not isinstance(context_length, int) + or isinstance(context_length, bool) + or context_length < 1 + ): + raise ValueError("bridge context_length must be positive") + self.adapter = adapter + self.seed_factory = seed_factory + self.action_column = action_column + self.context_length = context_length + self.units = units or {} + self.resets = 0 + + def reset(self, seed): + result = validate_reset(self.seed_factory(seed), self.units) + self.episode_id = f"bridge-{self.resets}" + self.resets += 1 + self.step_id = 0 + self.observation = dict(result.observation) + self.static_inputs = dict(result.static_inputs) + self.adapter.reset( + ResetRequest( + self.episode_id, + { + key: (value,) * self.context_length + for key, value in self.observation.items() + }, + (False,) * (self.context_length - 1) + (True,), + self.static_inputs, + seed, + ) + ) + return result + + def step(self, action): + if set(action) != {self.action_column}: + raise ValueError("bridge requires its one declared action column") + result = self.adapter.step( + StepRequest( + self.episode_id, + self.step_id, + self.observation, + dict(action), + self.static_inputs, + ) + ) + self.step_id += 1 + self.observation = dict(result.external_inputs) + return OnlineStepResult( + self.observation, + result.reward, + result.terminated, + result.truncated, + self.units, + ) + + def close(self): + self.adapter.close() + + +def adapter_identity(adapter): + return { + "module": type(adapter).__module__, + "class": type(adapter).__qualname__, + "protocol_version": adapter.protocol_version, + "supports_exact_snapshot": getattr(adapter, "supports_exact_snapshot", False) + is True, + } diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py new file mode 100644 index 00000000..d9f3ae76 --- /dev/null +++ b/src/sequifier/rl/history.py @@ -0,0 +1,192 @@ +"""Bounded causal state shared by online collection and policy execution.""" + +from collections import deque +from dataclasses import dataclass + +from sequifier.io.policy_window import PolicyWindowView, encode_feature +from sequifier.io.values import domain_fields + + +@dataclass(frozen=True) +class PolicyFrame: + observation: tuple + previous_action: int + static_inputs: tuple + episode_id: str + step_id: int + desired_return: float | None = None + + def state_dict(self): + return { + "observation": dict(self.observation), + "previous_action": self.previous_action, + "static_inputs": dict(self.static_inputs), + "episode_id": self.episode_id, + "step_id": self.step_id, + "desired_return": self.desired_return, + } + + @classmethod + def from_state_dict(cls, values): + return cls( + tuple(values["observation"].items()), + values["previous_action"], + tuple(values["static_inputs"].items()), + values["episode_id"], + values["step_id"], + values["desired_return"], + ) + + +def history_inputs(spec, frames): + frames = list(frames) + return PolicyWindowView(spec).inputs( + [dict(frame.observation) for frame in frames], + [frame.previous_action for frame in frames], + [frame.desired_return for frame in frames] if spec.return_conditioned else None, + dict(frames[-1].static_inputs), + ) + + +class PolicyHistory: + def __init__(self, spec, *, strict_vocabulary=False): + self.spec = spec + self.strict_vocabulary = strict_vocabulary + self.frames = deque(maxlen=spec.context_length) + self.reset_count = 0 + + @property + def attention_valid_mask(self): + return (False,) * (self.spec.context_length - len(self.frames)) + (True,) * len( + self.frames + ) + + @property + def current(self): + if not self.frames: + raise ValueError("policy history has not been reset") + return self.frames[-1] + + def _frame( + self, observation, previous_action, static, episode_id, step_id, desired_return + ): + if ( + not isinstance(episode_id, str) + or not episode_id + or not isinstance(step_id, int) + or isinstance(step_id, bool) + or step_id < 0 + ): + raise ValueError( + "policy frame requires an episode ID and nonnegative integer step" + ) + action_count = len(self.spec.action_values) + if ( + not isinstance(previous_action, int) + or isinstance(previous_action, bool) + or not ( + previous_action == action_count + if step_id == 0 + else 0 <= previous_action < action_count + ) + ): + raise ValueError("policy start token is allowed only on the first decision") + observation = domain_fields( + observation, set(self.spec.observation_columns), "policy observation" + ) + static = domain_fields( + static, set(self.spec.static_columns), "policy static inputs" + ) + for column, value in {**observation, **static}.items(): + mapping = self.spec.id_maps.get(column) + if ( + self.strict_vocabulary + and mapping is not None + and value not in mapping + and str(value) not in mapping + ): + raise ValueError( + f"{column}: online input is outside its frozen vocabulary" + ) + encode_feature(value, column, self.spec) + frame = PolicyFrame( + tuple(observation.items()), + previous_action, + tuple(static.items()), + episode_id, + step_id, + desired_return, + ) + # Reject values outside FP32 before making them part of live history. + history_inputs(self.spec, [frame]) + return frame + + def reset(self, observation, static_inputs, episode_id, desired_return=None): + frame = self._frame( + observation, + len(self.spec.action_values), + static_inputs, + episode_id, + 0, + desired_return, + ) + self.frames.clear() + self.frames.append(frame) + self.reset_count += 1 + + def advance(self, action, successor, desired_return=None): + current = self.current + if not 0 <= action < len(self.spec.action_values): + raise ValueError("history action is outside the actor vocabulary") + frame = self._frame( + successor, + action, + dict(current.static_inputs), + current.episode_id, + current.step_id + 1, + desired_return, + ) + self.frames.append(frame) + + def inputs(self): + return history_inputs(self.spec, self.frames) + + def state_dict(self): + return { + "frames": [frame.state_dict() for frame in self.frames], + "reset_count": self.reset_count, + "attention_valid_mask": self.attention_valid_mask, + } + + def load_state_dict(self, values): + frames = values["frames"] + if not 0 < len(frames) <= self.spec.context_length: + raise ValueError("invalid checkpoint history length") + restored = [] + for values_frame in frames: + frame = PolicyFrame.from_state_dict(values_frame) + restored.append( + self._frame( + dict(frame.observation), + frame.previous_action, + dict(frame.static_inputs), + frame.episode_id, + frame.step_id, + frame.desired_return, + ) + ) + if any( + right.episode_id != left.episode_id + or right.step_id != left.step_id + 1 + or right.static_inputs != left.static_inputs + for left, right in zip(restored, restored[1:]) + ): + raise ValueError("checkpoint causal history is discontinuous") + self.frames.clear() + self.frames.extend(restored) + self.reset_count = int(values["reset_count"]) + if ( + self.reset_count < 1 + or tuple(values["attention_valid_mask"]) != self.attention_valid_mask + ): + raise ValueError("checkpoint history mask or reset count is invalid") diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 9de6cc9c..889bfbfc 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -3,9 +3,10 @@ import json import re from pathlib import Path +from typing import TYPE_CHECKING from sequifier.artifacts.run_checkpoint import checkpoint_path -from sequifier.config.train_config import PolicyTrainConfig +from sequifier.config.train_config import PolicyTrainConfig, PPOTrainConfig from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.logging_paths import dataset_artifact_prefix from sequifier.rl.checkpoint import save_policy @@ -13,11 +14,18 @@ from sequifier.rl.training import PolicyTrainingResult, train_policy from sequifier.trajectory.importers import source_paths +if TYPE_CHECKING: + from sequifier.rl.ppo import PPOTrainingResult + def run_policy_training( - config: PolicyTrainConfig, -) -> PolicyTrainingResult | IQLTrainingResult: + config: PolicyTrainConfig | PPOTrainConfig, +) -> "PolicyTrainingResult | IQLTrainingResult | PPOTrainingResult": """Prepare episode partitions, train, and publish the selected policy and report.""" + if isinstance(config, PPOTrainConfig): + from sequifier.rl.ppo import run_ppo_training + + return run_ppo_training(config) phase = config.training_plan.phases[0] if phase.algorithm == "iql": return run_iql_training(config) diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py new file mode 100644 index 00000000..a5cd6401 --- /dev/null +++ b/src/sequifier/rl/ppo.py @@ -0,0 +1,499 @@ +"""Synchronous single-slot categorical PPO, with portable actor-only export.""" + +import hashlib +import platform +import statistics +import sys +import time +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path + +import torch +from loguru import logger + +from sequifier.model.factory import build_transformer_network +from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class +from sequifier.rl.checkpoint import load_policy, save_policy +from sequifier.rl.environment import ( + adapter_identity, + load_online_adapter, + require_exact_snapshot, +) +from sequifier.rl.models import HistoryCritic +from sequifier.rl.policy import CategoricalPolicy, policy_execution_config +from sequifier.rl.ppo_buffer import write_json_atomic +from sequifier.rl.ppo_checkpoint import ( + capture_rng_state, + clone_weights, + contract_fingerprint, + load_ppo_checkpoint, + restore_rng_state, + resume_contract, + save_ppo_checkpoint, + snapshots_equal, +) +from sequifier.rl.ppo_collector import PPOCollector, stream_seed +from sequifier.rl.ppo_evaluation import evaluate_policy, require_evaluation_timer +from sequifier.rl.ppo_update import update_ppo + + +@dataclass(frozen=True) +class PPOTrainingResult: + policy: CategoricalPolicy + metrics: tuple[dict, ...] + evaluations: tuple[dict, ...] + env_steps: int + iterations: int + best_metric: float | None + report: dict + + +def _make_optimizer(module, spec, learning_rate): + return get_optimizer_class(spec.name)( + [parameter for parameter in module.parameters() if parameter.requires_grad], + lr=learning_rate, + **spec.arguments, + ) + + +def _initial_actor(config, execution, will_resume): + actor = CategoricalPolicy.from_network( + build_transformer_network( + execution, device=torch.device(config.device) + ).network, + execution, + config.policy.interface_name, + ) + provenance = None + if config.initialize_from is not None and not will_resume: + source = load_policy(config.initialize_from, device=config.device) + if ( + source.config != config.policy + or source.interface.selected_columns_statistics + != config.normalization_statistics + or source.execution_config.model != execution.model + ): + raise ValueError( + "PPO initialize_from must exactly match policy, normalization, model architecture and context" + ) + actor.network.load_state_dict(source.network.state_dict(), strict=True) + provenance = { + "path": config.initialize_from, + "sha256": hashlib.sha256( + Path(config.initialize_from).read_bytes() + ).hexdigest(), + } + return actor.eval(), provenance + + +def run_ppo_training(config) -> PPOTrainingResult: + """Train one authored PPO phase; commits occur only after complete updates.""" + from sequifier.config.train_config import PPOTrainConfig + from sequifier.helpers import configure_determinism + + if not isinstance(config, PPOTrainConfig): + raise TypeError("run_ppo_training requires PPOTrainConfig") + require_evaluation_timer() + import resource + + configure_determinism(config.seed, config.enforce_determinism) + phase = config.training_plan.phases[0] + device = torch.device(config.device) + if device.type == "cuda" and not torch.cuda.is_available(): + raise ValueError("requested CUDA device is unavailable") + if device.type == "mps" and not torch.backends.mps.is_available(): + raise ValueError("requested MPS device is unavailable") + checkpoint = Path(config.checkpoint_path) + if phase.resume.policy == "required" and not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + will_resume = phase.resume.policy != "never" and checkpoint.is_file() + payload = load_ppo_checkpoint(checkpoint, config) if will_resume else None + if ( + config.initialize_from is not None + and not will_resume + and not Path(config.initialize_from).is_file() + ): + raise FileNotFoundError(config.initialize_from) + execution = policy_execution_config( + config.policy, + config.model, + config.device, + statistics=config.normalization_statistics, + project_root=config.project_root, + model_name=config.model_name, + seed=config.seed, + ) + actor, provenance = _initial_actor(config, execution, will_resume) + actor.ppo_policy_version = 0 + actor.set_normalization_statistics(config.normalization_statistics) + value = HistoryCritic(execution, config.policy.interface_name, None).eval() + modules = {"actor": actor, "value": value} + if any( + not torch.isfinite(tensor).all() + for module in modules.values() + for tensor in module.state_dict().values() + ): + raise ValueError("PPO actor and V initialization must be finite") + ids = [ + id(parameter) + for module in modules.values() + for parameter in module.parameters() + if parameter.requires_grad + ] + if len(ids) != len(set(ids)): + raise ValueError("PPO actor and V must own disjoint trainable parameters") + if ( + value.interface.selected_columns_statistics + != actor.interface.selected_columns_statistics + ): + raise ValueError("PPO actor and V normalization differs") + plans = { + "actor": ( + actor, + phase.actor_optimizer, + phase.actor_learning_rate, + phase.actor_scheduler, + ), + "value": ( + value, + phase.value_optimizer, + phase.value_learning_rate, + phase.value_scheduler, + ), + } + optimizers = { + role: _make_optimizer(module, optimizer_spec, learning_rate) + for role, (module, optimizer_spec, learning_rate, _) in plans.items() + } + schedulers = { + role: get_scheduler_class(scheduler_spec.name)( + optimizers[role], **scheduler_spec.arguments + ) + for role, (_, _, _, scheduler_spec) in plans.items() + } + sampler = torch.Generator(device="cpu").manual_seed( + stream_seed(config.seed, "minibatches", 0) + ) + clocks = {"actor": 0, "value": 0} + metrics, evaluations, continuations = [], [], [] + env_steps, iteration, policy_version = 0, 0, 0 + best_actor, best_metric, best_iteration = None, None, None + trajectory_equivalent = True + adapter = load_online_adapter(phase.environment) + try: + if phase.resume.mode == "exact": + require_exact_snapshot(adapter) + collector = PPOCollector(actor, value, adapter, phase, config.seed) + if payload is not None: + if payload["adapter_identity"] != adapter_identity(adapter): + raise ValueError( + "PPO adapter implementation identity or snapshot capability differs" + ) + actor.network.load_state_dict(payload["actor"], strict=True) + value.load_state_dict(payload["value"], strict=True) + for role in optimizers: + optimizers[role].load_state_dict(payload["optimizers"][role]) + schedulers[role].load_state_dict(payload["schedulers"][role]) + sampler.set_state(payload["sampler"]) + clocks = dict(payload["optimizer_steps"]) + env_steps, iteration, policy_version = ( + payload["env_steps"], + payload["iteration"], + payload["policy_version"], + ) + best_actor, best_metric, best_iteration = ( + payload["best_actor"], + payload["best_metric"], + payload["best_iteration"], + ) + metrics, evaluations = ( + list(payload["metrics"]), + list(payload["evaluations"]), + ) + continuations = list(payload["continuations"]) + provenance = payload["initialization"] + trajectory_equivalent = payload["trajectory_equivalent"] + exact = phase.resume.mode == "exact" + if exact: + if payload["environment_state"] is None: + raise ValueError( + "checkpoint has no declared exact environment snapshot" + ) + adapter.restore_state(deepcopy(payload["environment_state"])) + if not snapshots_equal( + adapter.snapshot_state(), payload["environment_state"] + ): + raise ValueError( + "adapter did not restore its simulator and RNG snapshot exactly" + ) + else: + trajectory_equivalent = False + collector.load_state_dict(payload["collector"], exact=exact) + continuations.append( + { + "iteration": iteration, + "mode": phase.resume.mode, + "trajectory_equivalent": trajectory_equivalent, + "dropped_incomplete_episode": not exact + and not payload["collector"]["ended"], + } + ) + actor.ppo_policy_version = policy_version + restore_rng_state(device, payload["random"]) + + def evaluate(): + nonlocal best_actor, best_metric, best_iteration + result = evaluate_policy(actor, phase.evaluation, forbidden_adapter=adapter) + result.update( + { + "env_steps": env_steps, + "iteration": iteration, + "policy_version": policy_version, + } + ) + evaluations.append(result) + score = result["selection_value"] + better = score is not None and ( + best_metric is None + or ( + score > best_metric + if phase.evaluation.direction == "max" + else score < best_metric + ) + ) + if better: + best_metric, best_iteration = score, iteration + best_actor = clone_weights(actor.network) + return result["seconds"] + + if not evaluations: + evaluate() + while env_steps < phase.total_env_steps: + horizon = min(phase.rollout_length, phase.total_env_steps - env_steps) + started = time.monotonic() + buffer, episodes = collector.collect(horizon, policy_version) + collection_seconds = time.monotonic() - started + if phase.archive_directory is not None: + buffer.archive( + Path(phase.archive_directory) + / f"collection-{iteration + 1:08d}.json", + include_scores=phase.archive_scores, + ) + started = time.monotonic() + record = update_ppo( + actor, + value, + buffer, + phase, + optimizers, + schedulers, + sampler, + policy_version, + clocks, + ) + update_seconds = time.monotonic() - started + env_steps += horizon + iteration += 1 + policy_version += 1 + evaluation_seconds = 0.0 + if ( + iteration % phase.evaluation_interval == 0 + or env_steps == phase.total_env_steps + ): + evaluation_seconds = evaluate() + completed_returns = [ + episode["return"] for episode in episodes if episode["terminated"] + ] + completed_lengths = [ + episode["length"] for episode in episodes if episode["terminated"] + ] + record.update( + { + "iteration": iteration, + "policy_version": policy_version, + "collected_policy_version": buffer.policy_version, + "env_steps": env_steps, + "collected_decisions": horizon, + "collection_seconds": collection_seconds, + "update_seconds": update_seconds, + "evaluation_seconds": evaluation_seconds, + "collection_steps_per_second": horizon + / max(collection_seconds, 1e-9), + "finished_episodes": episodes, + "training_mean_completed_return": statistics.fmean( + completed_returns + ) + if completed_returns + else None, + "training_return_std": statistics.pstdev(completed_returns) + if completed_returns + else None, + "training_mean_completed_length": statistics.fmean( + completed_lengths + ) + if completed_lengths + else None, + "completed_episode_count": sum( + episode["terminated"] for episode in episodes + ), + "censored_episode_count": sum( + episode["truncated"] and not episode["terminated"] + for episode in episodes + ), + "invalid_episode_count": 0, + "live_episode_length": 0 + if collector.ended + else collector.episode_length, + "buffer_estimated_bytes": buffer.estimated_bytes, + "actor_learning_rate": optimizers["actor"].param_groups[0]["lr"], + "value_learning_rate": optimizers["value"].param_groups[0]["lr"], + } + ) + metrics.append(record) + logger.info( + "PPO iteration {}: {}/{} actions, policy loss {:.5f}, V loss {:.5f}", + iteration, + env_steps, + phase.total_env_steps, + record["policy_loss"], + record["value_loss"], + ) + if ( + iteration % phase.checkpoint_interval == 0 + or env_steps == phase.total_env_steps + ): + environment_state = None + if getattr(adapter, "supports_exact_snapshot", None) is True: + require_exact_snapshot(adapter) + environment_state = adapter.snapshot_state() + if not isinstance(environment_state, dict) or not snapshots_equal( + environment_state, environment_state + ): + raise ValueError( + "adapter exact snapshot must be a finite, portable state dictionary" + ) + save_ppo_checkpoint( + checkpoint, + { + "artifact_type": "sequifier_ppo_run", + "format_version": 1, + "config": resume_contract(config), + "config_fingerprint": contract_fingerprint(config), + "policy": config.policy.model_dump(mode="json"), + "statistics": config.normalization_statistics, + "actor_spec": config.model.model_dump(mode="json"), + "value_spec": { + "type": "HistoryCritic", + "action_count": None, + "model": config.model.model_dump(mode="json"), + }, + "actor": clone_weights(actor.network), + "value": clone_weights(value), + "optimizers": { + role: optimizer.state_dict() + for role, optimizer in optimizers.items() + }, + "schedulers": { + role: scheduler.state_dict() + for role, scheduler in schedulers.items() + }, + "optimizer_steps": clocks, + "sampler": sampler.get_state(), + "random": capture_rng_state(device), + "seed_streams": { + "version": 1, + "run_seed": config.seed, + "evaluation_seeds": phase.evaluation.seeds, + }, + "collector": collector.state_dict(), + "environment_state": environment_state, + "adapter_identity": adapter_identity(adapter), + "env_steps": env_steps, + "iteration": iteration, + "policy_version": policy_version, + "best_actor": best_actor, + "best_metric": best_metric, + "best_iteration": best_iteration, + "initialization": provenance, + "trajectory_equivalent": trajectory_equivalent, + "continuations": continuations, + "metrics": metrics, + "evaluations": evaluations, + }, + ) + finally: + adapter.close() + if best_actor is not None: + actor.network.load_state_dict(best_actor, strict=True) + elapsed = sum( + record["collection_seconds"] + record["update_seconds"] for record in metrics + ) + peak_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + report = { + "format": "sequifier.ppo-training-report", + "version": 1, + "algorithm": "ppo", + "config": config.model_dump(mode="json"), + "config_fingerprint": contract_fingerprint(config), + "env_steps": env_steps, + "iterations": iteration, + "policy_version": policy_version, + "optimizer_steps": clocks, + "metrics": metrics, + "evaluations": evaluations, + "monitor": { + "metric": phase.evaluation.metric, + "direction": phase.evaluation.direction, + "value": best_metric, + "available": best_metric is not None, + "best_iteration": best_iteration, + }, + "export_selection": "best_evaluated_actor" + if best_actor is not None + else "latest_actor_metric_unavailable", + "initialization": provenance, + "trajectory_equivalent": trajectory_equivalent, + "continuations": continuations, + "invalid_episode_count": 0, + "evaluation_censored_count": sum( + result["censored_count"] for result in evaluations + ), + "training_completed_count": sum( + record["completed_episode_count"] for record in metrics + ), + "training_censored_count": sum( + record["censored_episode_count"] for record in metrics + ), + "final_live_episode_censored": not collector.ended, + "hardware": { + "device": str(device), + "platform": platform.platform(), + "processor": platform.processor(), + "torch_version": str(torch.__version__), + "torch_threads": torch.get_num_threads(), + }, + "context_length": config.context_length, + "actor_parameters": sum(parameter.numel() for parameter in actor.parameters()), + "value_parameters": sum(parameter.numel() for parameter in value.parameters()), + "steps_per_second": env_steps / max(elapsed, 1e-9), + "peak_process_memory_bytes": peak_rss + if sys.platform == "darwin" + else peak_rss * 1024, + "peak_cuda_memory_bytes": torch.cuda.max_memory_allocated(device) + if device.type == "cuda" + else None, + "model_path": config.model_path, + "checkpoint_path": config.checkpoint_path, + } + save_policy(actor, config.model_path, metadata=report) + write_json_atomic(config.report_path, report) + return PPOTrainingResult( + actor, + tuple(metrics), + tuple(evaluations), + env_steps, + iteration, + best_metric, + report, + ) diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py new file mode 100644 index 00000000..594558c5 --- /dev/null +++ b/src/sequifier/rl/ppo_buffer.py @@ -0,0 +1,298 @@ +"""Compact, single-generation on-policy storage and fixed FP32 GAE targets.""" + +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import cast + +import torch + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.io.batch import SequifierInputs +from sequifier.io.tensors import collate_tensor_records +from sequifier.rl.history import history_inputs + + +@dataclass(frozen=True) +class PPODecision: + frame: int + successor: int | None + action: int + domain_action: str | float | int + old_log_probability: float + old_value: float + entropy: float + reward: float + raw_reward: float + successor_value: float + terminated: bool + truncated: bool + bootstrap: bool + episode_id: str + step_id: int + policy_version: int + + +@dataclass(frozen=True) +class PPORolloutBuffer: + """Sealed tuples retain each frame once, including the pre-buffer context. + + Access reconstructs complete ordered windows. No optimization tensor is + shared with the fixed old scores or targets held by this buffer. + """ + + spec: CategoricalPolicyConfig + policy_version: int + frames: tuple + decisions: tuple[PPODecision, ...] + advantages: tuple[float, ...] + targets: tuple[float, ...] + estimated_bytes: int + + def __len__(self): + return len(self.decisions) + + def require_version(self, version): + if version != self.policy_version or any( + step.policy_version != version for step in self.decisions + ): + raise ValueError("PPO cannot update from another policy generation") + + def window(self, reference): + frame = self.frames[reference] + start = reference + while ( + start > 0 + and reference - start + 1 < self.spec.context_length + and self.frames[start - 1].episode_id == frame.episode_id + ): + if self.frames[start - 1].step_id + 1 != self.frames[start].step_id: + raise ValueError("PPO frame history is discontinuous") + start -= 1 + return history_inputs(self.spec, self.frames[start : reference + 1]) + + def batch(self, indices, device): + indices = [int(index) for index in indices] + return ( + cast( + SequifierInputs, + collate_tensor_records( + [self.window(self.decisions[index].frame) for index in indices] + ), + ).to(device), + torch.tensor( + [self.decisions[index].action for index in indices], + dtype=torch.long, + device=device, + ), + torch.tensor( + [self.decisions[index].old_log_probability for index in indices], + dtype=torch.float32, + device=device, + ), + torch.tensor( + [self.decisions[index].old_value for index in indices], + dtype=torch.float32, + device=device, + ), + torch.tensor( + [self.advantages[index] for index in indices], + dtype=torch.float32, + device=device, + ), + torch.tensor( + [self.targets[index] for index in indices], + dtype=torch.float32, + device=device, + ), + ) + + def archive(self, path, *, include_scores): + """A separate versioned collection record, never an offline update source.""" + decisions = [] + for index, step in enumerate(self.decisions): + record = asdict(step) + boundary = step.terminated or step.truncated + cut = index == len(self) - 1 and not boundary + record["cutoff_reason"] = ( + "terminated" + if step.terminated + else "truncated" + if step.truncated + else "collection_cut" + if cut + else None + ) + if not include_scores: + for key in ( + "old_log_probability", + "old_value", + "entropy", + "successor_value", + ): + record.pop(key) + decisions.append(record) + payload = { + "format": "sequifier.ppo-collection", + "version": 1, + "policy_version": self.policy_version, + "policy": self.spec.model_dump(mode="json"), + "frames": [frame.state_dict() for frame in self.frames], + "decisions": decisions, + } + write_json_atomic(path, payload) + + +def write_json_atomic(path, payload): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + try: + temporary.write_text( + json.dumps(payload, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def load_collection_record(path): + """Read the independent archival format without changing v1 episode readers.""" + payload = json.loads(Path(path).read_text(encoding="utf-8")) + if ( + payload.get("format") != "sequifier.ppo-collection" + or payload.get("version") != 1 + ): + raise ValueError("expected version-1 PPO collection record") + from sequifier.config.policy_config import CategoricalPolicyConfig + from sequifier.rl.history import PolicyFrame + + spec = CategoricalPolicyConfig.model_validate(payload["policy"]) + frames = [PolicyFrame.from_state_dict(frame) for frame in payload["frames"]] + for frame in frames: + history_inputs(spec, [frame]) + for step in payload["decisions"]: + frame = frames[step["frame"]] + if ( + step["policy_version"] != payload["policy_version"] + or (step["episode_id"], step["step_id"]) + != (frame.episode_id, frame.step_id) + or not 0 <= step["action"] < len(spec.action_values) + or step["domain_action"] != spec.action_values[step["action"]] + or step["bootstrap"] != (not step["terminated"]) + ): + raise ValueError("collection record violates the decision contract") + if step["successor"] is None: + if step["bootstrap"]: + raise ValueError("collection record lacks a bootstrap successor") + else: + successor = frames[step["successor"]] + if ( + successor.episode_id != frame.episode_id + or successor.step_id != frame.step_id + 1 + or successor.previous_action != step["action"] + ): + raise ValueError("collection record successor is discontinuous") + if not torch.isfinite(torch.tensor(step["reward"], dtype=torch.float32)): + raise ValueError("collection record reward must be finite FP32") + return payload + + +def compute_gae(decisions, gamma, gae_lambda, normalize): + """Boundary-aware FP32 targets; normalization happens once per iteration.""" + advantages = torch.zeros(len(decisions), dtype=torch.float32) + following_advantage = torch.zeros((), dtype=torch.float32) + for index in range(len(decisions) - 1, -1, -1): + step = decisions[index] + continues = ( + index + 1 < len(decisions) + and not (step.terminated or step.truncated) + and decisions[index + 1].episode_id == step.episode_id + and decisions[index + 1].step_id == step.step_id + 1 + ) + delta = ( + torch.tensor(step.reward, dtype=torch.float32) + + gamma + * float(step.bootstrap) + * torch.tensor(step.successor_value, dtype=torch.float32) + - torch.tensor(step.old_value, dtype=torch.float32) + ) + following_advantage = ( + delta + gamma * gae_lambda * float(continues) * following_advantage + ) + advantages[index] = following_advantage + targets = advantages + torch.tensor( + [step.old_value for step in decisions], dtype=torch.float32 + ) + if normalize: + advantages = (advantages - advantages.mean()) / advantages.std( + unbiased=False + ).clamp_min(1e-8) + if not torch.isfinite(advantages).all() or not torch.isfinite(targets).all(): + raise ValueError("non-finite PPO advantage or target") + return tuple(advantages.tolist()), tuple(targets.tolist()) + + +class PPOBufferBuilder: + def __init__(self, spec, version, horizon, initial_frames, max_bytes): + self.spec = spec.model_copy(deep=True) + self.version = version + self.horizon = horizon + self.frames = list(initial_frames) + self.decisions = [] + self.max_bytes = max_bytes + self.bytes_per_frame = max( + [self.frame_bytes(frame) for frame in self.frames] + [1024] + ) + self.estimated_bytes = ( + 2 * horizon + spec.context_length + ) * self.bytes_per_frame + horizon * 1024 + if self.estimated_bytes > max_bytes: + raise ValueError( + f"PPO buffer estimate {self.estimated_bytes} exceeds max_buffer_bytes={max_bytes}" + ) + self.used_bytes = sum(self.frame_bytes(frame) for frame in self.frames) + self.sealed = False + + @staticmethod + def frame_bytes(frame): + # Conservative Python-object allowance plus the complete domain payload. + return 1024 + 4 * len( + json.dumps(frame.state_dict(), allow_nan=False).encode("utf-8") + ) + + def append_frame(self, frame): + if self.sealed: + raise ValueError("PPO buffer is sealed") + cost = self.frame_bytes(frame) + if self.used_bytes + cost + len(self.decisions) * 1024 > self.max_bytes: + raise ValueError("PPO history exceeded its declared memory budget") + if len(self.frames) >= 2 * self.horizon + self.spec.context_length: + raise ValueError("PPO history exceeded its bounded frame capacity") + self.frames.append(frame) + self.used_bytes += cost + return len(self.frames) - 1 + + def append(self, decision): + if self.sealed or len(self.decisions) >= self.horizon: + raise ValueError("PPO buffer is sealed or full") + if decision.policy_version != self.version: + raise ValueError("policy generation changed during collection") + self.decisions.append(decision) + if self.used_bytes + len(self.decisions) * 1024 > self.max_bytes: + raise ValueError("PPO decisions exceeded the memory budget") + + def seal(self, gamma, gae_lambda, normalize): + if self.sealed or len(self.decisions) != self.horizon: + raise ValueError("PPO must seal exactly one complete collection horizon") + self.sealed = True + advantages, targets = compute_gae(self.decisions, gamma, gae_lambda, normalize) + return PPORolloutBuffer( + self.spec, + self.version, + tuple(self.frames), + tuple(self.decisions), + advantages, + targets, + max(self.estimated_bytes, self.used_bytes + len(self.decisions) * 1024), + ) diff --git a/src/sequifier/rl/ppo_checkpoint.py b/src/sequifier/rl/ppo_checkpoint.py new file mode 100644 index 00000000..b95cee97 --- /dev/null +++ b/src/sequifier/rl/ppo_checkpoint.py @@ -0,0 +1,158 @@ +"""Atomic, algorithm-owned PPO iteration checkpoints.""" + +import hashlib +import json +import math +from collections.abc import Mapping +from pathlib import Path + +import numpy as np +import torch + +from sequifier.runtime.random_state import RandomState, RandomStateManager + + +def resume_contract(config): + values = config.model_dump(mode="json") + values["training_plan"]["phases"][0].pop("resume") + return values + + +def contract_fingerprint(config): + return hashlib.sha256( + json.dumps(resume_contract(config), sort_keys=True, allow_nan=False).encode() + ).hexdigest() + + +def clone_weights(module): + return { + name: tensor.detach().cpu().clone() + for name, tensor in module.state_dict().items() + } + + +def save_ppo_checkpoint(path, payload): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + try: + torch.save(payload, temporary) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def load_ppo_checkpoint(path, config): + payload = torch.load(path, map_location="cpu", weights_only=False) + if ( + not isinstance(payload, Mapping) + or payload.get("artifact_type") != "sequifier_ppo_run" + or payload.get("format_version") != 1 + ): + raise ValueError("PPO resume requires a version-1 PPO learner checkpoint") + if payload.get("config_fingerprint") != contract_fingerprint(config) or payload.get( + "config" + ) != resume_contract(config): + raise ValueError( + "PPO resume config, adapter identity, codec, model, or reward scale differs" + ) + if ( + payload.get("policy") != config.policy.model_dump(mode="json") + or payload.get("statistics") != config.normalization_statistics + ): + raise ValueError("PPO resume frozen representation differs") + phase = config.training_plan.phases[0] + steps = payload["env_steps"] + iteration = payload["iteration"] + expected_steps = min(iteration * phase.rollout_length, phase.total_env_steps) + if ( + not isinstance(steps, int) + or isinstance(steps, bool) + or not isinstance(iteration, int) + or isinstance(iteration, bool) + or iteration < 1 + or not 0 < steps <= phase.total_env_steps + or steps != expected_steps + or iteration != (steps + phase.rollout_length - 1) // phase.rollout_length + or payload["policy_version"] != iteration + or len(payload["metrics"]) != iteration + ): + raise ValueError("PPO checkpoint is not a complete iteration boundary") + if payload["best_metric"] is not None and not math.isfinite(payload["best_metric"]): + raise ValueError("PPO checkpoint best metric must be finite or unavailable") + if (payload["best_actor"] is None) != (payload["best_metric"] is None): + raise ValueError( + "PPO checkpoint best actor and metric must be recorded together" + ) + if set(payload["optimizer_steps"]) != {"actor", "value"} or any( + not isinstance(clock, int) or isinstance(clock, bool) or clock < iteration + for clock in payload["optimizer_steps"].values() + ): + raise ValueError("PPO checkpoint optimizer clocks are invalid") + for state in (payload["actor"], payload["value"], payload["best_actor"] or {}): + if any(not torch.isfinite(tensor).all() for tensor in state.values()): + raise ValueError("PPO checkpoint contains non-finite weights") + return payload + + +def snapshots_equal(left, right): + """Exact adapters use torch-serializable scalar, tensor and container state.""" + if type(left) is not type(right): + return False + if isinstance(left, torch.Tensor): + return ( + left.dtype == right.dtype + and bool(torch.isfinite(left).all()) + and torch.equal(left.cpu(), right.cpu()) + ) + if isinstance(left, np.ndarray): + finite = ( + np.isfinite(left).all() + if left.dtype.kind in "biufc" + else left.dtype.kind in "US" + ) + return ( + left.dtype == right.dtype and bool(finite) and np.array_equal(left, right) + ) + if isinstance(left, np.generic): + return snapshots_equal(left.item(), right.item()) + if isinstance(left, dict): + return left.keys() == right.keys() and all( + snapshots_equal(left[key], right[key]) for key in left + ) + if isinstance(left, (list, tuple)): + return len(left) == len(right) and all( + snapshots_equal(a, b) for a, b in zip(left, right) + ) + if isinstance(left, float): + return math.isfinite(left) and left == right + if isinstance(left, (str, int, bool, type(None))): + return left == right + raise ValueError( + "exact adapter snapshots must use scalar, tensor, NumPy array, dict, list, or tuple values" + ) + + +def capture_rng_state(device): + return { + "local": RandomStateManager(device).capture_local().state_dict(), + "cuda_all": torch.cuda.get_rng_state_all() + if torch.cuda.is_available() + else None, + "mps": torch.mps.get_rng_state() if torch.backends.mps.is_available() else None, + } + + +def restore_rng_state(device, state): + RandomStateManager(device).restore(RandomState.from_state_dict(state["local"])) + if state["cuda_all"] is not None: + if ( + not torch.cuda.is_available() + or len(state["cuda_all"]) != torch.cuda.device_count() + ): + raise ValueError("PPO checkpoint CUDA RNG topology differs") + torch.cuda.set_rng_state_all(state["cuda_all"]) + if state["mps"] is not None: + if not torch.backends.mps.is_available(): + raise ValueError("PPO checkpoint requires the saved MPS RNG") + torch.mps.set_rng_state(state["mps"]) diff --git a/src/sequifier/rl/ppo_collector.py b/src/sequifier/rl/ppo_collector.py new file mode 100644 index 00000000..8e552ce5 --- /dev/null +++ b/src/sequifier/rl/ppo_collector.py @@ -0,0 +1,234 @@ +"""One persistent environment slot; horizons cut history without resetting it.""" + +import hashlib +import math +from dataclasses import dataclass + +import torch + +from sequifier.rl.distributions import CategoricalActionDistribution, action_generator +from sequifier.rl.environment import validate_reset, validate_step +from sequifier.rl.history import PolicyHistory +from sequifier.rl.ppo_buffer import PPOBufferBuilder, PPODecision + + +def stream_seed(seed, stream, reset_count): + value = f"sequifier-ppo-v1:{seed}:{stream}:{reset_count}".encode() + return int.from_bytes(hashlib.blake2b(value, digest_size=8).digest(), "big") % ( + 2**63 + ) + + +@dataclass(frozen=True) +class PolicyDecision: + domain_action: dict + action: int + log_probability: float + entropy: float + value: float + + +def validate_online_inputs(actor, inputs): + inputs = inputs.to(actor.device) + actor.validate_inputs(inputs) + features, _ = actor.model_inputs(inputs) + if any( + tensor.is_floating_point() and not torch.isfinite(tensor).all() + for tensor in features.values() + ): + raise ValueError("frozen input transforms produced non-finite values") + return inputs + + +@torch.no_grad() +def decide(actor, value, inputs, generator, *, deterministic=False): + """Score action and history-only V from exactly the same causal input.""" + actor.eval() + if value is not None: + value.eval() + inputs = validate_online_inputs(actor, inputs) + distribution = CategoricalActionDistribution(actor(inputs)[:, -1]) + action = distribution.select(deterministic, generator) + estimate = ( + value(inputs) if value is not None else torch.zeros(1, device=actor.device) + ) + if not torch.isfinite(estimate).all(): + raise ValueError("non-finite PPO collection value") + index = int(action[0]) + return PolicyDecision( + {actor.config.action_column: actor.config.action_values[index]}, + index, + float(distribution.log_prob(action)[0]), + float(distribution.entropy()[0]), + float(estimate[0]), + ) + + +class PPOCollector: + def __init__(self, actor, value, adapter, phase, seed): + self.actor = actor + self.value = value + self.adapter = adapter + self.phase = phase + self.seed = seed + self.history = PolicyHistory(actor.config, strict_vocabulary=True) + self.generator = action_generator( + actor.device, stream_seed(seed, "training-actions", 0) + ) + self.ended = True + self.episode_return = 0.0 + self.episode_length = 0 + + @property + def reset_count(self): + return self.history.reset_count + + def reset(self): + result = validate_reset( + self.adapter.reset( + stream_seed(self.seed, "training-reset", self.reset_count) + ), + self.phase.environment.units, + ) + self.history.reset( + result.observation, result.static_inputs, f"train-{self.reset_count}" + ) + self.ended = False + self.episode_return = 0.0 + self.episode_length = 0 + validate_online_inputs(self.actor, self.history.inputs()) + + def collect(self, horizon, policy_version): + if self.actor.ppo_policy_version != policy_version: + raise ValueError("collection policy generation disagrees with the actor") + self.actor.eval() + self.value.eval() + # Preflight before the first reset where possible; exact domain size is + # checked again after reset and on each insertion. + minimum = (2 * horizon + self.actor.config.context_length) * ( + 1024 + 128 * len(self.actor.interface.input_columns) + ) + horizon * 1024 + if minimum > self.phase.max_buffer_bytes: + raise ValueError( + f"PPO buffer requires approximately {minimum} bytes before collection" + ) + if self.ended: + self.reset() + builder = PPOBufferBuilder( + self.actor.config, + policy_version, + horizon, + self.history.frames, + self.phase.max_buffer_bytes, + ) + current_reference = len(builder.frames) - 1 + completed = [] + with torch.no_grad(): + for _ in range(horizon): + if self.ended: + self.reset() + current_reference = builder.append_frame(self.history.current) + if current_reference is None: + raise ValueError( + "live PPO decision lacks its observation reference" + ) + current = self.history.current + decision = decide( + self.actor, self.value, self.history.inputs(), self.generator + ) + result = validate_step( + self.adapter.step(decision.domain_action), + self.phase.environment.units, + ) + successor_reference = None + successor_value = 0.0 + if result.observation is not None: + self.history.advance(decision.action, result.observation) + successor_inputs = validate_online_inputs( + self.actor, self.history.inputs() + ) + if not result.terminated: + successor_value = float(self.value(successor_inputs)[0]) + if not torch.isfinite( + torch.tensor(successor_value, dtype=torch.float32) + ): + raise ValueError("non-finite PPO successor value") + successor_reference = builder.append_frame(self.history.current) + reward = float(result.reward) * self.phase.reward_scale + if not torch.isfinite( + torch.tensor([reward, successor_value], dtype=torch.float32) + ).all(): + raise ValueError("reward or successor V exceeds finite FP32 range") + builder.append( + PPODecision( + current_reference, + successor_reference, + decision.action, + self.actor.config.action_values[decision.action], + decision.log_probability, + decision.value, + decision.entropy, + reward, + float(result.reward), + successor_value, + result.terminated, + result.truncated, + not result.terminated, + current.episode_id, + current.step_id, + policy_version, + ) + ) + current_reference = successor_reference + self.episode_return += float(result.reward) + if not math.isfinite(self.episode_return): + raise ValueError("non-finite training episode return") + self.episode_length += 1 + self.ended = result.terminated or result.truncated + if self.ended: + completed.append( + { + "episode_id": current.episode_id, + "return": self.episode_return, + "length": self.episode_length, + "terminated": result.terminated, + "truncated": result.truncated, + "success": result.success, + } + ) + return builder.seal( + self.phase.gamma, self.phase.gae_lambda, self.phase.normalize_advantages + ), completed + + def state_dict(self): + return { + "history": self.history.state_dict(), + "reset_count": self.reset_count, + "generator": self.generator.get_state(), + "ended": self.ended, + "episode_return": self.episode_return, + "episode_length": self.episode_length, + } + + def load_state_dict(self, state, *, exact): + reset_count = int(state["reset_count"]) + if reset_count < 1: + raise ValueError("PPO checkpoint has an invalid reset count") + self.generator.set_state(state["generator"]) + if exact: + self.history.load_state_dict(state["history"]) + if ( + self.reset_count != reset_count + or self.history.current.episode_id != f"train-{reset_count - 1}" + ): + raise ValueError( + "PPO checkpoint episode identity disagrees with reset count" + ) + self.ended = bool(state["ended"]) + self.episode_return = float(state["episode_return"]) + self.episode_length = int(state["episode_length"]) + else: + self.history.reset_count = reset_count + self.ended = True + self.episode_return = 0.0 + self.episode_length = 0 diff --git a/src/sequifier/rl/ppo_evaluation.py b/src/sequifier/rl/ppo_evaluation.py new file mode 100644 index 00000000..a831e6d2 --- /dev/null +++ b/src/sequifier/rl/ppo_evaluation.py @@ -0,0 +1,177 @@ +"""Bounded local policy evaluation with independent seeds, adapters and RNG.""" + +import math +import signal +import statistics +import threading +import time +from contextlib import contextmanager +from copy import deepcopy + +from sequifier.rl.distributions import action_generator +from sequifier.rl.environment import load_online_adapter, validate_reset, validate_step +from sequifier.rl.history import PolicyHistory +from sequifier.rl.ppo_checkpoint import capture_rng_state, restore_rng_state +from sequifier.rl.ppo_collector import decide, stream_seed, validate_online_inputs + + +class EvaluationTimeout(TimeoutError): + pass + + +def require_evaluation_timer(): + if threading.current_thread() is not threading.main_thread() or not hasattr( + signal, "setitimer" + ): + raise ValueError( + "bounded local PPO evaluation requires the main thread and a POSIX interval timer" + ) + + +@contextmanager +def _deadline(seconds): + require_evaluation_timer() + previous_handler = signal.getsignal(signal.SIGALRM) + previous_timer = signal.getitimer(signal.ITIMER_REAL) + started = time.monotonic() + + def expired(signum, frame): + raise EvaluationTimeout("PPO evaluation exceeded timeout_seconds") + + signal.signal(signal.SIGALRM, expired) + try: + signal.setitimer(signal.ITIMER_REAL, seconds) + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + if previous_timer[0] > 0: + signal.setitimer( + signal.ITIMER_REAL, + max(1e-6, previous_timer[0] - (time.monotonic() - started)), + previous_timer[1], + ) + + +def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): + """Evaluate a frozen snapshot; completed returns exclude censored episodes. + + The timeout covers adapter creation/reset/steps for the entire evaluation. + Adapters must release resources promptly from close(). + """ + require_evaluation_timer() + saved_random = capture_rng_state(actor.device) + started = time.monotonic() + adapter = None + episodes = [] + timed_out = False + attempted = 0 + episode_return = 0.0 + length = 0 + active = False + try: + snapshot = deepcopy(actor).eval() + with _deadline(evaluation.timeout_seconds): + adapter = load_online_adapter(evaluation.environment) + if adapter is forbidden_adapter: + adapter = None # The training owner must retain and close it. + raise ValueError( + "evaluation factory returned the live training adapter" + ) + for index, seed in enumerate(evaluation.seeds): + attempted += 1 + episode_return, length = 0.0, 0 + active = True + result = validate_reset( + adapter.reset(stream_seed(seed, "evaluation-reset", 0)), + evaluation.environment.units, + ) + history = PolicyHistory(snapshot.config, strict_vocabulary=True) + history.reset( + result.observation, result.static_inputs, f"evaluation-{index}" + ) + generator = action_generator( + snapshot.device, stream_seed(seed, "evaluation-actions", 0) + ) + terminated = False + success = None + for _ in range(evaluation.max_episode_steps): + decision = decide( + snapshot, + None, + history.inputs(), + generator, + deterministic=evaluation.selection == "deterministic", + ) + step = validate_step( + adapter.step(decision.domain_action), + evaluation.environment.units, + ) + length += 1 + episode_return += float(step.reward) + if not math.isfinite(episode_return): + raise ValueError("non-finite evaluation episode return") + if step.observation is not None: + history.advance(decision.action, step.observation) + validate_online_inputs(snapshot, history.inputs()) + terminated = step.terminated + if step.terminated or step.truncated: + success = step.success + break + episodes.append( + { + "seed": seed, + "return": episode_return, + "length": length, + "completed": terminated, + "censored": not terminated, + "success": success if terminated else None, + } + ) + active = False + except EvaluationTimeout: + timed_out = True + if active: + episodes.append( + { + "seed": evaluation.seeds[attempted - 1], + "return": episode_return, + "length": length, + "completed": False, + "censored": True, + "success": None, + } + ) + finally: + try: + if adapter is not None: + adapter.close() + finally: + restore_rng_state(actor.device, saved_random) + completed = [episode for episode in episodes if episode["completed"]] + returns = [episode["return"] for episode in completed] + lengths = [episode["length"] for episode in completed] + successes = [episode["success"] for episode in completed] + result = { + "mean_completed_return": statistics.fmean(returns) if returns else None, + "return_std": statistics.pstdev(returns) if returns else None, + "return_min": min(returns) if returns else None, + "return_max": max(returns) if returns else None, + "mean_completed_length": statistics.fmean(lengths) if lengths else None, + "mean_length": statistics.fmean(episode["length"] for episode in episodes) + if episodes + else None, + "completed_count": len(completed), + "censored_count": len(episodes) - len(completed), + "unattempted_count": len(evaluation.seeds) - attempted, + "timed_out": timed_out, + "episodes": episodes, + "seconds": time.monotonic() - started, + "selection": evaluation.selection, + } + if successes and all(success is not None for success in successes): + result["success_rate"] = statistics.fmean(successes) + result["selection_metric"] = evaluation.metric + result["selection_value"] = result.get(evaluation.metric) + result["selection_available"] = result["selection_value"] is not None + return result diff --git a/src/sequifier/rl/ppo_update.py b/src/sequifier/rl/ppo_update.py new file mode 100644 index 00000000..3670a91e --- /dev/null +++ b/src/sequifier/rl/ppo_update.py @@ -0,0 +1,211 @@ +"""PPO objectives and sample-weighted independent actor/value optimization.""" + +from typing import Any + +import torch + +from sequifier.rl.distributions import CategoricalActionDistribution + + +def ppo_objectives( + log_probability, + old_log_probability, + advantages, + prediction, + old_value, + target, + entropy, + phase, +): + log_ratio = log_probability.float() - old_log_probability.float() + ratio = log_ratio.exp() + clipped = ratio.clamp(1 - phase.clip_epsilon, 1 + phase.clip_epsilon) + policy_loss = -torch.minimum(ratio * advantages, clipped * advantages) + value_error = (prediction.float() - target).square() + if phase.value_clip is not None: + clipped_value = old_value + (prediction.float() - old_value).clamp( + -phase.value_clip, phase.value_clip + ) + value_error = torch.maximum(value_error, (clipped_value - target).square()) + value_loss = 0.5 * value_error + approximate_kl = ratio - 1 - log_ratio + tensors = (ratio, policy_loss, value_loss, entropy, approximate_kl) + if any(not torch.isfinite(tensor).all() for tensor in tensors): + raise ValueError("non-finite PPO probability ratio or objective") + return ( + policy_loss, + value_loss, + { + "policy_loss": policy_loss, + "value_loss": value_loss, + "entropy": entropy, + "approximate_kl": approximate_kl, + "clip_fraction": ((ratio - 1).abs() > phase.clip_epsilon).float(), + "ratio": ratio, + }, + ) + + +@torch.no_grad() +def validate_collection_scores(actor, value, buffer, batch_size): + """Check both identity ratio and V before the first optimizer transaction.""" + actor.eval() + value.eval() + for start in range(0, len(buffer), batch_size): + history, actions, old_logp, old_v, _, _ = buffer.batch( + range(start, min(start + batch_size, len(buffer))), actor.device + ) + if not torch.allclose( + actor.evaluate_actions(history, actions), old_logp, atol=1e-5, rtol=1e-4 + ): + raise ValueError("PPO reconstructed action scores differ from collection") + if not torch.allclose(value(history), old_v, atol=1e-5, rtol=1e-4): + raise ValueError("PPO reconstructed values differ from collection") + + +def update_ppo( + actor, value, buffer, phase, optimizers, schedulers, sampler, policy_version, clocks +): + buffer.require_version(policy_version) + if actor.ppo_policy_version != policy_version: + raise ValueError( + "PPO buffer has already been consumed or belongs to a stale actor" + ) + actor.eval() # Keep dropout off while retaining autograd. + value.eval() + validate_collection_scores(actor, value, buffer, phase.minibatch_size) + parameters = { + "actor": [p for p in actor.parameters() if p.requires_grad], + "value": [p for p in value.parameters() if p.requires_grad], + } + totals = { + key: 0.0 + for key in ( + "policy_loss", + "value_loss", + "entropy", + "approximate_kl", + "clip_fraction", + ) + } + ratio_chunks = [] + count = 0 + norms = {"actor": [], "value": []} + stopped = False + gradient_limits = { + "actor": phase.actor_gradient_clip, + "value": phase.value_gradient_clip, + } + group_size = phase.minibatch_size * phase.accumulation_steps + epochs = 0 + for epoch in range(phase.optimization_epochs): + order = torch.randperm(len(buffer), generator=sampler).tolist() + for start in range(0, len(order), group_size): + group = order[start : start + group_size] + minibatches = [ + group[offset : offset + phase.minibatch_size] + for offset in range(0, len(group), phase.minibatch_size) + ] + if phase.target_kl is not None: + kl_sum = 0.0 + with torch.no_grad(): + for indices in minibatches: + history, actions, old_logp, _, _, _ = buffer.batch( + indices, actor.device + ) + log_ratio = actor.evaluate_actions(history, actions) - old_logp + kl = log_ratio.exp() - 1 - log_ratio + if not torch.isfinite(kl).all(): + raise ValueError("non-finite PPO KL") + kl_sum += float(kl.sum()) + if kl_sum / len(group) > phase.target_kl: + stopped = True + break + for optimizer in optimizers.values(): + optimizer.zero_grad(set_to_none=True) + for indices in minibatches: + history, actions, old_logp, old_v, advantages, target = buffer.batch( + indices, actor.device + ) + distribution = CategoricalActionDistribution(actor(history)[:, -1]) + policy_loss, value_loss, metrics = ppo_objectives( + distribution.log_prob(actions), + old_logp, + advantages, + value(history), + old_v, + target, + distribution.entropy(), + phase, + ) + actor_loss = ( + policy_loss - phase.entropy_coefficient * metrics["entropy"] + ).sum() / len(group) + critic_loss = phase.value_coefficient * value_loss.sum() / len(group) + if not torch.isfinite(actor_loss) or not torch.isfinite(critic_loss): + raise ValueError("non-finite weighted PPO loss") + actor_loss.backward() + critic_loss.backward() + for key in totals: + totals[key] += float(metrics[key].detach().sum()) + ratio_chunks.append(metrics["ratio"].detach().cpu()) + count += len(indices) + for role in ("actor", "value"): + norm = torch.nn.utils.clip_grad_norm_( + parameters[role], + gradient_limits[role], + error_if_nonfinite=True, + ) + norms[role].append(float(norm.detach())) + for role in ("actor", "value"): + optimizers[role].step() + schedulers[role].step() + clocks[role] += 1 + optimizers[role].zero_grad(set_to_none=True) + if any( + not torch.isfinite(parameter).all() + for parameter in parameters[role] + ): + raise ValueError( + f"PPO {role} optimizer produced non-finite parameters" + ) + epochs = epoch + 1 + if stopped: + break + if not count: + raise ValueError( + "PPO KL stopped all updates before an identity-ratio transaction" + ) + ratios = torch.cat(ratio_chunks) + targets = torch.tensor(buffer.targets, dtype=torch.float32) + with torch.no_grad(): + predictions = [] + for start in range(0, len(buffer), phase.minibatch_size): + history, *_ = buffer.batch( + range(start, min(start + phase.minibatch_size, len(buffer))), + actor.device, + ) + predictions.append(value(history).cpu()) + residual_variance = (targets - torch.cat(predictions)).var(unbiased=False) + target_variance = targets.var(unbiased=False) + result: dict[str, Any] = {key: total / count for key, total in totals.items()} + result.update( + { + "actor_optimizer_step": clocks["actor"], + "value_optimizer_step": clocks["value"], + "actor_grad_norm": sum(norms["actor"]) / len(norms["actor"]), + "value_grad_norm": sum(norms["value"]) / len(norms["value"]), + "ratio_min": float(ratios.min()), + "ratio_max": float(ratios.max()), + "ratio_p01": float(torch.quantile(ratios, 0.01)), + "ratio_p99": float(torch.quantile(ratios, 0.99)), + "explained_variance": float(1 - residual_variance / target_variance) + if target_variance > 1e-12 + else None, + "optimization_epochs": epochs, + "scored_decisions": count, + "kl_stopped": stopped, + } + ) + actor.ppo_policy_version = policy_version + 1 + return result diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index 2666803b..c23ecaf1 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -6,6 +6,7 @@ from sequifier.io.policy_window import PolicyWindowView from sequifier.io.values import DomainValue, domain_fields from sequifier.rl.distributions import CategoricalActionDistribution, action_generator +from sequifier.rl.history import PolicyHistory from sequifier.rl.policy import CategoricalPolicy from sequifier.rollout.contracts import EnvironmentAdapter, ResetRequest from sequifier.rollout.coordinator import collect_rollout @@ -49,6 +50,13 @@ def reset(self, request: ResetRequest) -> None: if spec.return_conditioned else [] ) + self.history = PolicyHistory(spec) + self.history.reset( + {column: values[-1] for column, values in request.seed_history.items()}, + request.static_inputs, + request.episode_id, + self.return_trace[0] if spec.return_conditioned else None, + ) self.previous_action_trace = [] self.log_probabilities = [] self.generator = action_generator(self.policy.device, request.seed) @@ -67,24 +75,11 @@ def __call__(self, history: SeedHistory) -> dict[str, DomainValue]: observation = { column: values[-1] for column, values in history.observations.items() } - inputs = PolicyWindowView(spec).inputs( - [ - { - name: values[position] - for name, values in history.observations.items() - } - for position, valid in enumerate(history.attention_valid_mask) - if valid - ], - self.previous_action_trace[ - max(0, len(self.previous_action_trace) - spec.context_length + 1) : - ] - + [self.previous_action], - self.return_trace[-spec.context_length :] - if spec.return_conditioned - else None, - history.static_inputs, - ) + if observation != dict( + self.history.current.observation + ) or history.static_inputs != dict(self.history.current.static_inputs): + raise ValueError("policy causal history disagrees with rollout history") + inputs = self.history.inputs() with torch.no_grad(): distribution = CategoricalActionDistribution( self.policy(inputs.to(self.policy.device))[:, -1, :] @@ -117,6 +112,11 @@ def observe(self, transition: Transition) -> None: self.return_trace[-1], transition.reward, spec.returns.gamma ) ) + self.history.advance( + index, + transition.successor_observation, + self.return_trace[-1] if spec.return_conditioned else None, + ) self.previous_action = index self.pending = None self.ended = transition.terminated or transition.truncated diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index dfc7c0c5..01eb6297 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -1,4 +1,4 @@ -"""Local trajectory-backed categorical policies and return-conditioned learning.""" +"""Local categorical policy learning, online collection and execution.""" from sequifier.config.policy_config import ( CategoricalPolicyConfig, @@ -11,8 +11,16 @@ SequenceSourceConfig, ) from sequifier.config.sequence_import_config import SequenceImportConfig -from sequifier.config.train_config import PolicyTrainConfig -from sequifier.config.training_plan import IQLPhaseConfig, PolicyImitationPhaseConfig +from sequifier.config.train_config import PolicyTrainConfig, PPOTrainConfig +from sequifier.config.training_plan import ( + IQLPhaseConfig, + OnlineEnvironmentConfig, + PolicyImitationPhaseConfig, + PPOEvaluationConfig, + PPONormalizationConfig, + PPOPhaseConfig, + PPOResumeConfig, +) from sequifier.io.batch import SequifierBatch, SequifierInputs, collate_sequence_batches from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.io.policy_window import PolicyWindowView, action_index @@ -22,9 +30,18 @@ CategoricalActionDistribution, masked_action_nll_loss, ) +from sequifier.rl.environment import ( + OnlineEnvironmentAdapter, + OnlineResetResult, + OnlineStepResult, + SeededAdapterBridge, +) from sequifier.rl.iql import IQLTrainingResult, run_iql_training from sequifier.rl.learner import run_policy_training from sequifier.rl.policy import CategoricalPolicy +from sequifier.rl.ppo import PPOTrainingResult, run_ppo_training +from sequifier.rl.ppo_buffer import load_collection_record +from sequifier.rl.ppo_evaluation import evaluate_policy from sequifier.rl.runtime import PolicyExecutor, collect_policy_rollout from sequifier.rl.training import PolicyTrainingResult, train_policy from sequifier.trajectory.importers import import_sequences, source_episodes @@ -45,6 +62,20 @@ "import_sequences", "PolicyInferenceConfig", "PolicyTrainConfig", + "PPOTrainConfig", + "PPOPhaseConfig", + "OnlineEnvironmentConfig", + "PPOEvaluationConfig", + "PPONormalizationConfig", + "PPOResumeConfig", + "OnlineEnvironmentAdapter", + "OnlineResetResult", + "OnlineStepResult", + "SeededAdapterBridge", + "PPOTrainingResult", + "run_ppo_training", + "evaluate_policy", + "load_collection_record", "PolicyImitationPhaseConfig", "IQLPhaseConfig", "run_policy_training", diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 496fb01c..a2b9c2b8 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -19,8 +19,11 @@ ModelArtifact, load_execution_model, ) -from sequifier.config.train_config import PolicyTrainConfig # noqa: E402 from sequifier.config.train_config import load_train_config # noqa: E402 +from sequifier.config.train_config import ( # noqa: E402 + PolicyTrainConfig, + PPOTrainConfig, +) from sequifier.config.train_config import ( # noqa: E402 ResolvedSequifierConfig as TrainModel, ) @@ -163,7 +166,7 @@ def _mp_train_worker_wrapper( @beartype def run_training( - config: TrainModel | PolicyTrainConfig, + config: TrainModel | PolicyTrainConfig | PPOTrainConfig, *, integration_specs: tuple[IntegrationSpec, ...] = (), integration_instances: tuple[Any, ...] = (), @@ -171,7 +174,7 @@ def run_training( ) -> None: """Launch canonical service-composed training locally or across workers.""" - if isinstance(config, PolicyTrainConfig): + if isinstance(config, (PolicyTrainConfig, PPOTrainConfig)): if integration_specs or integration_instances or semantic_optimizer_grouping: raise ValueError( "policy training does not yet support custom integrations or parameter grouping" @@ -179,7 +182,12 @@ def run_training( from sequifier.rl.learner import run_policy_training configure_logger(config.project_root, config.model_name, 0) - configure_determinism(config.seed, config.global_training.enforce_determinism) + configure_determinism( + config.seed, + config.enforce_determinism + if isinstance(config, PPOTrainConfig) + else config.global_training.enforce_determinism, + ) run_policy_training(config) return if not isinstance(config, TrainModel): From c178e1857fbd96c544b762599e88c0dc7b846bbe Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 12:11:13 +0100 Subject: [PATCH 15/71] implement fixes --- .../example_inputs/configs/train-bc-toy.yaml | 2 +- .../example_inputs/configs/train-iql-toy.yaml | 2 +- documentation/policy-learning.md | 12 ++++++++---- src/sequifier/config/train_config.py | 4 +++- src/sequifier/config/training_plan.py | 4 ++-- src/sequifier/consolidated-docs.md | 12 ++++++++---- src/sequifier/make.py | 2 +- src/sequifier/rl/iql.py | 12 +++++++++--- 8 files changed, 33 insertions(+), 17 deletions(-) diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml index f6789d82..9d5b7219 100644 --- a/documentation/example_inputs/configs/train-bc-toy.yaml +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -23,7 +23,7 @@ model: ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 1, __policy_previous_action: 6} + feature_embedding_dims: {position: 2, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} dataset_training: demonstrations: diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml index 184d4eb7..1c22bfde 100644 --- a/documentation/example_inputs/configs/train-iql-toy.yaml +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -24,7 +24,7 @@ model: ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 1, __policy_previous_action: 6} + feature_embedding_dims: {position: 2, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} dataset_training: demonstrations: diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 273d025c..e26cbf5c 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -503,8 +503,10 @@ python documentation/example_inputs/iql_toy_control.py evaluate models/toy-bc/to complete update checkpoint. `policy: required` needs an explicit `checkpoint_path`. Exact resume validates the configuration, source content, split, fitted vocabulary, and normalization. A BC model passed to -`initialize_from` is a warm start, not a learner checkpoint. IQL currently -supports scalar histories, one categorical action, and local execution; a +`initialize_from` is a warm start, not a learner checkpoint; the actor's policy +schema, normalization, model architecture, and context must match. IQL requires +float32 layers without autocast and supports scalar histories, one categorical +action, and local execution; a return-conditioned actor, distributed training, mixed precision, continuous or joint actions, and mixed phase plans are rejected. @@ -592,8 +594,10 @@ The policy's observation/static schema, action order, class counts, label maps, model architecture and context are fixed before the first reset. Author `normalization: {source: identity}` deliberately, or use `source: statistics` with a `statistics` map containing exactly the real fields' -`mean` and positive `std`. PPO never estimates statistics from changing online -data. `initialize_from` accepts an ordinary unconditioned BC/IQL actor only if +`mean` and nonnegative `std`. Zero standard deviations from constant features +use the shared normalizer's epsilon, preserving BC/IQL warm-start statistics. +PPO never estimates statistics from changing online data. `initialize_from` +accepts an ordinary unconditioned BC/IQL actor only if its policy schema, action ordering, normalization, model and context match exactly. It initializes actor weights, not the learner state. A return-conditioned actor is rejected. diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 3739ddf2..0279490f 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -1054,7 +1054,9 @@ def resolve(self): "PPO archive_directory requires a directory destination" ) if any( - archive == Path(path) or archive in Path(path).parents + archive == Path(path) + or archive in Path(path).parents + or Path(path) in archive.parents for path in outputs + ([self.initialize_from] if self.initialize_from else []) ): diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py index 34cab7a5..0927dd45 100644 --- a/src/sequifier/config/training_plan.py +++ b/src/sequifier/config/training_plan.py @@ -182,10 +182,10 @@ def validate_statistics(self): if ( set(values) != {"mean", "std"} or not all(math.isfinite(value) for value in values.values()) - or values["std"] <= 0 + or values["std"] < 0 ): raise ValueError( - f"{column}: statistics require finite mean and positive std" + f"{column}: statistics require finite mean and nonnegative std" ) return self diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 4ccd3563..2eb000cf 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -2898,8 +2898,10 @@ python documentation/example_inputs/iql_toy_control.py evaluate models/toy-bc/to complete update checkpoint. `policy: required` needs an explicit `checkpoint_path`. Exact resume validates the configuration, source content, split, fitted vocabulary, and normalization. A BC model passed to -`initialize_from` is a warm start, not a learner checkpoint. IQL currently -supports scalar histories, one categorical action, and local execution; a +`initialize_from` is a warm start, not a learner checkpoint; the actor's policy +schema, normalization, model architecture, and context must match. IQL requires +float32 layers without autocast and supports scalar histories, one categorical +action, and local execution; a return-conditioned actor, distributed training, mixed precision, continuous or joint actions, and mixed phase plans are rejected. @@ -2987,8 +2989,10 @@ The policy's observation/static schema, action order, class counts, label maps, model architecture and context are fixed before the first reset. Author `normalization: {source: identity}` deliberately, or use `source: statistics` with a `statistics` map containing exactly the real fields' -`mean` and positive `std`. PPO never estimates statistics from changing online -data. `initialize_from` accepts an ordinary unconditioned BC/IQL actor only if +`mean` and nonnegative `std`. Zero standard deviations from constant features +use the shared normalizer's epsilon, preserving BC/IQL warm-start statistics. +PPO never estimates statistics from changing online data. `initialize_from` +accepts an ordinary unconditioned BC/IQL actor only if its policy schema, action ordering, normalization, model and context match exactly. It initializes actor weights, not the learner state. A return-conditioned actor is rejected. diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 6b4e611e..4b4a5307 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -172,7 +172,7 @@ ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6} + feature_embedding_dims: {OBSERVATION_COLUMN: 2, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} dataset_training: demonstrations: diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index d17f5bd1..575a5318 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -176,8 +176,11 @@ def run_iql_training(config) -> IQLTrainingResult: raise ValueError("IQL requires an unconditioned policy") if config.global_training.distributed or config.global_training.data_parallelism: raise ValueError("IQL currently requires local execution") - if config.global_training.layer_autocast: - raise ValueError("IQL mixed precision is not available in this slice") + if config.global_training.layer_autocast or any( + dtype != "float32" + for dtype in (config.global_training.layer_type_dtypes or {}).values() + ): + raise ValueError("IQL requires float32 layers without autocast") if config.global_training.torch_compile != "none": raise ValueError("IQL requires torch_compile: none") if config.global_training.accumulation_steps not in (None, 1): @@ -289,8 +292,11 @@ def corpus(names): if ( source.config != fitted or source.interface.selected_columns_statistics != statistics + or source.execution_config.model != execution.model ): - raise ValueError("IQL warm-start policy schema or normalization differs") + raise ValueError( + "IQL warm-start policy schema, normalization, or model architecture differs" + ) actor.network.load_state_dict(source.network.state_dict(), strict=True) modules = { "actor": actor.network, From 51583f39595781285687336856628b9adeae1b25 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 12:20:55 +0100 Subject: [PATCH 16/71] implement fixes --- documentation/policy-learning.md | 6 +++--- src/sequifier/consolidated-docs.md | 6 +++--- src/sequifier/rl/models.py | 2 +- src/sequifier/rl/ppo_buffer.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index e26cbf5c..87695d3c 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -218,9 +218,9 @@ Likewise, `dataset_training.` accepts `model_interface` and `parts`, with part containing a sequence source specification. Reference it with the ordinary `{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these parts do not need fabricated criteria. Model/interface references are validated. -The shared phase schema preserves supervised defaults; this learner still supports -one `policy_imitation` or `iql` phase, with BC selected on its policy. Mixed algorithms, -backbone repositories, image/depth policies and online learning remain planned. +The shared phase schema preserves supervised defaults; policy training supports +one `policy_imitation`, `iql`, or `ppo` phase, with BC selected on its policy. +Mixed algorithms, backbone repositories, and image/depth policies remain planned. Use `initialize_from` for backbone artifact transfer. See [train-policy-from-table.yaml](example_inputs/configs/train-policy-from-table.yaml) diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 2eb000cf..c5589372 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -2613,9 +2613,9 @@ Likewise, `dataset_training.` accepts `model_interface` and `parts`, with part containing a sequence source specification. Reference it with the ordinary `{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these parts do not need fabricated criteria. Model/interface references are validated. -The shared phase schema preserves supervised defaults; this learner still supports -one `policy_imitation` or `iql` phase, with BC selected on its policy. Mixed algorithms, -backbone repositories, image/depth policies and online learning remain planned. +The shared phase schema preserves supervised defaults; policy training supports +one `policy_imitation`, `iql`, or `ppo` phase, with BC selected on its policy. +Mixed algorithms, backbone repositories, and image/depth policies remain planned. Use `initialize_from` for backbone artifact transfer. See [train-policy-from-table.yaml](example_inputs/configs/train-policy-from-table.yaml) diff --git a/src/sequifier/rl/models.py b/src/sequifier/rl/models.py index 3a4662f3..8c4557a3 100644 --- a/src/sequifier/rl/models.py +++ b/src/sequifier/rl/models.py @@ -42,7 +42,7 @@ def forward(self, history: SequifierInputs, action: torch.Tensor | None = None): representation = self.network.encode( features, history.metadata, interface_name=self.interface_name ) - state = self.network.backbone.final_norm(representation[:, -1]) + state = representation[:, -1] if self.action_embedding is not None: if action is None: raise ValueError("Q requires a current action") diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py index 594558c5..67079962 100644 --- a/src/sequifier/rl/ppo_buffer.py +++ b/src/sequifier/rl/ppo_buffer.py @@ -224,7 +224,7 @@ def compute_gae(decisions, gamma, gae_lambda, normalize): targets = advantages + torch.tensor( [step.old_value for step in decisions], dtype=torch.float32 ) - if normalize: + if normalize and len(decisions) > 1: advantages = (advantages - advantages.mean()) / advantages.std( unbiased=False ).clamp_min(1e-8) From 26caeab4aea00d384d6639d494317a5ee8266d9b Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 16:00:38 +0100 Subject: [PATCH 17/71] GRPO and DPO --- README.md | 6 +- documentation/configs/infer.md | 8 +- documentation/configs/train.md | 102 +- .../example_inputs/configs/infer-policy.yaml | 2 +- .../example_inputs/configs/infer-ppo-toy.yaml | 2 +- .../configs/rollout-ppo-toy.yaml | 2 +- .../example_inputs/configs/train-bc-toy.yaml | 29 +- .../configs/train-dpo-control.yaml | 44 + .../configs/train-grpo-control.yaml | 50 + .../configs/train-iql-from-table.yaml | 81 +- .../example_inputs/configs/train-iql-toy.yaml | 50 +- .../configs/train-multiphase-control.yaml | 97 ++ .../configs/train-policy-from-table.yaml | 42 +- .../example_inputs/configs/train-ppo-toy.yaml | 59 +- .../configs/train-return-conditioned.yaml | 81 +- .../example_inputs/policy_control_data.py | 89 ++ .../example_inputs/ppo_toy_control.py | 35 +- ...dpo-grpo-multiphase-implementation-plan.md | 390 ++++++ documentation/policy-learning.md | 1060 ++++++--------- release-notes/unreleased-policy-training.md | 116 +- src/sequifier/artifacts/model_config.py | 2 - src/sequifier/artifacts/run_checkpoint.py | 5 - src/sequifier/config/infer_config.py | 4 +- src/sequifier/config/policy_config.py | 4 +- src/sequifier/config/policy_plan.py | 341 +++++ src/sequifier/config/policy_train_config.py | 6 +- src/sequifier/config/train_config.py | 460 +++---- src/sequifier/config/training_plan.py | 259 +--- src/sequifier/consolidated-docs.md | 1152 ++++++----------- src/sequifier/docs.py | 2 +- src/sequifier/io/policy_dataset.py | 31 + src/sequifier/io/policy_window.py | 4 +- src/sequifier/io/preference_dataset.py | 277 ++++ src/sequifier/make.py | 238 +++- src/sequifier/rl/checkpoint.py | 22 +- src/sequifier/rl/collection.py | 60 + src/sequifier/rl/control.py | 26 +- src/sequifier/rl/dpo.py | 149 +++ src/sequifier/rl/environment.py | 97 +- src/sequifier/rl/grpo.py | 193 +++ src/sequifier/rl/grpo_buffer.py | 112 ++ src/sequifier/rl/grpo_collector.py | 166 +++ src/sequifier/rl/grpo_update.py | 126 ++ src/sequifier/rl/history.py | 8 + src/sequifier/rl/iql.py | 567 ++------ src/sequifier/rl/learner.py | 604 +++++++-- src/sequifier/rl/offline_data.py | 63 + src/sequifier/rl/policy.py | 40 +- src/sequifier/rl/ppo.py | 553 ++------ src/sequifier/rl/ppo_buffer.py | 29 +- src/sequifier/rl/ppo_checkpoint.py | 158 --- src/sequifier/rl/ppo_collector.py | 57 +- src/sequifier/rl/ppo_evaluation.py | 14 +- src/sequifier/rl/preparation.py | 252 ++++ src/sequifier/rl/registry.py | 51 + src/sequifier/rl/run_state.py | 375 ++++++ src/sequifier/rl/snapshots.py | 44 + src/sequifier/rl/training.py | 241 ++-- src/sequifier/rl_api.py | 118 +- src/sequifier/runtime/builder.py | 10 +- src/sequifier/sequifier.py | 5 +- src/sequifier/train.py | 13 +- src/sequifier/training/checkpoint_service.py | 5 - src/sequifier/training/engine.py | 44 +- src/sequifier/training/loss.py | 6 - src/sequifier/training/runtime.py | 147 +-- src/sequifier/training/state.py | 4 - src/sequifier/trajectory/preferences.py | 52 + 68 files changed, 5475 insertions(+), 4066 deletions(-) create mode 100644 documentation/example_inputs/configs/train-dpo-control.yaml create mode 100644 documentation/example_inputs/configs/train-grpo-control.yaml create mode 100644 documentation/example_inputs/configs/train-multiphase-control.yaml create mode 100644 documentation/example_inputs/policy_control_data.py create mode 100644 documentation/plans/dpo-grpo-multiphase-implementation-plan.md create mode 100644 src/sequifier/config/policy_plan.py create mode 100644 src/sequifier/io/preference_dataset.py create mode 100644 src/sequifier/rl/collection.py create mode 100644 src/sequifier/rl/dpo.py create mode 100644 src/sequifier/rl/grpo.py create mode 100644 src/sequifier/rl/grpo_buffer.py create mode 100644 src/sequifier/rl/grpo_collector.py create mode 100644 src/sequifier/rl/grpo_update.py create mode 100644 src/sequifier/rl/offline_data.py delete mode 100644 src/sequifier/rl/ppo_checkpoint.py create mode 100644 src/sequifier/rl/preparation.py create mode 100644 src/sequifier/rl/registry.py create mode 100644 src/sequifier/rl/run_state.py create mode 100644 src/sequifier/rl/snapshots.py create mode 100644 src/sequifier/trajectory/preferences.py diff --git a/README.md b/README.md index 07438f64..7e4f141d 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ There are seven standalone commands within sequifier: `make`, `preprocess`, `tra | --- | --- | | `make` | Create a new sequifier project with config templates. | | `preprocess` | Convert input data into fixed-length subsequences. | -| `train` | Train a supervised model or a categorical policy from sequence tables or rollout recordings. | +| `train` | Train supervised models or ordered imitation, IQL, PPO, DPO and GRPO policy phases. | | `infer` | Generate predictions, probabilities, embeddings, or policy actions and rollouts. | | `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | | `visualize-training` | Generate interactive HTML plots from structured training metrics. | @@ -116,7 +116,7 @@ The processing and visualization commands have dedicated documentation pages: - [preprocess documentation](./documentation/configs/preprocess.md) - [train documentation](./documentation/configs/train.md) -- [policy training](./documentation/policy-learning.md), including BC, offline IQL and [online categorical PPO](./documentation/policy-learning.md#online-categorical-ppo) +- [policy training](./documentation/policy-learning.md), including BC, IQL, DPO, GRPO and [online categorical PPO](./documentation/policy-learning.md#online-categorical-ppo) - [infer documentation](./documentation/configs/infer.md) - [hyperparameter-search documentation](./documentation/configs/hyperparameter-search.md) - [visualize-training documentation](./documentation/commands/visualize-training.md) @@ -422,3 +422,5 @@ Please cite with: } ``` + +Policy runs support ordered imitation, IQL, DPO, GRPO and PPO phases; see the [policy learning guide](documentation/policy-learning.md). diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index a40f6ba3..2847d48c 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -267,10 +267,10 @@ ORT installation. Outputs stay indexed by outer item coordinates. ## Policy model interfaces -`sequifier infer` recognizes a policy interface in a portable PyTorch model or -run checkpoint. Use raw CSV/Parquet observation rows and optional recorded action -histories; alignment, vocabulary, and normalization come from the artifact. +`sequifier infer` recognizes a policy interface in a portable PyTorch actor. +Use the selected actor export rather than the policy run checkpoint. Use raw +CSV/Parquet observation rows and optional recorded action histories; alignment, vocabulary, and normalization come from the artifact. `model_type: generative` and `model_type: policy` are accepted. Environment execution uses the existing `rollout` section with `mode: policy`. See the -[policy guide](../policy-learning.md#predict-actions-with-sequifier-infer) and +[policy guide](../policy-learning.md#python-and-inference) and [inference example](../example_inputs/configs/infer-policy.yaml). diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 5271ff93..9b5592f7 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -768,79 +768,29 @@ healthy active update. Momentum and weight decay cannot update parameters whose gradients are absent. -## Policy sequence training - -`sequifier train` accepts one `policy_imitation` or `iql` phase. Put the observation/action -schema in `model.interfaces..policy`, a sequence source in each -`dataset_training..parts.`, and optimizer/context settings in -`global_training` with `training_objective: policy`. Reference sources with -`{source: dataset.part}`. The algorithm supplies action NLL. - -`return_conditioned: false` selects BC, which needs observations and action labels. -Return-conditioned training additionally requires rewards and a known return -endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories all -use the same episode representation. Models support CPU/CUDA/MPS, backbone -transfer via `initialize_from`, and shared-engine checkpoint resume via -`global_training.resume`. IQL additionally requires complete transition data -and an unconditioned categorical policy. PPO uses the separate online schema below. -Mixed phases and image/depth policies remain planned. The former compact registries and configuration aliases have been -consolidated into the shared model/dataset authoring surface. - -See the [complete table example](../example_inputs/configs/train-policy-from-table.yaml), -[return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml), -[IQL example](../example_inputs/configs/train-iql-from-table.yaml), -and [policy learning guide](../policy-learning.md). - -## Online PPO configuration - -Online PPO uses `PPOTrainConfig`, selected automatically by one `algorithm: ppo` -phase. It requires no datasets, sources, validation loaders, `epochs` or -`global_training`. See the [runnable YAML](../example_inputs/configs/train-ppo-toy.yaml) -and [adapter/resume guide](../policy-learning.md#online-categorical-ppo). - -Root fields are `project_root` (default `.`), `model_name`, `model`, -`context_length` (defaults to the backbone maximum), `device` (default `cpu`), -`seed` (default 1010, in `[0, 2**32)`), `enforce_determinism` (default true), -optional `initialize_from`, and optional `model_path`, `report_path`, -`checkpoint_path`. Paths resolve against `project_root`; outputs must be -distinct. Exactly one scalar policy interface is allowed and -`return_conditioned` must be false. Schema/class counts, action order, -normalization and architecture must be explicit and frozen. - -The phase accepts these fields; extra fields are rejected: - -| Fields | Default / contract | -| --- | --- | -| `algorithm`, `policy`, `name` | `ppo`, selected interface, `online_ppo` | -| `environment` | Required `factory: module:factory`, `config: {}`, `protocol_version: 2`, `units: {}` | -| `normalization` | Required `source: identity` or `statistics`; the latter requires exact real-column `statistics: {column: {mean: ..., std: ...}}` | -| `total_env_steps`, `rollout_length` | Required positive executed-action budgets | -| `allow_partial_rollout` | true; false requires a divisible total budget | -| `optimization_epochs`, `minibatch_size`, `accumulation_steps` | 4, 64, 1; all positive | -| `gamma`, `gae_lambda` | 0.99, 0.95; both in `[0, 1]` | -| `clip_epsilon`, `value_clip` | 0.2 in `(0, 1)`; null or positive value clip | -| `value_coefficient`, `entropy_coefficient` | 0.5 positive; 0.01 nonnegative | -| `target_kl`, `normalize_advantages` | null or positive KL limit; true | -| `reward_scale` | 1.0; fixed positive finite multiplier, checkpointed | -| `actor_learning_rate`, `value_learning_rate` | 0.0003 each, positive finite | -| `actor_optimizer`, `value_optimizer` | `{name: Adam}`; Adam/AdamW/SGD/RMSprop with constructor arguments; phase owns `lr` and parameters | -| `actor_scheduler`, `value_scheduler` | `{name: ConstantLR, factor: 1.0, total_iters: 1}`; scheduler constructor arguments, stepped after each role's optimizer step | -| `actor_gradient_clip`, `value_gradient_clip` | 1.0 each; positive finite norm limit | -| `evaluation_interval`, `checkpoint_interval` | 1 each, measured in complete iterations; final iteration always runs both | -| `evaluation.environment` | Required separate adapter factory/config, same frozen units as training | -| `evaluation.seeds` | Required nonempty list of integers in `[0, 2**63)` | -| `evaluation.max_episode_steps`, `evaluation.timeout_seconds` | Required positive per-episode action cap and whole-evaluation deadline | -| `evaluation.selection` | `deterministic` or explicitly `stochastic` | -| `evaluation.metric`, `evaluation.direction` | `mean_completed_return`, `max`; alternatives `mean_completed_length` / `success_rate`, direction `min` / `max` | -| `resume` | `{policy: never, mode: reset_environments}`; policy also `if_exists`/`required`, mode also `exact` | -| `seed_policy` | `derived_per_reset`; separate training/evaluation streams | -| `max_buffer_bytes` | 536870912; positive conservative host-buffer budget, estimated before collection and enforced per insertion | -| `archive_directory`, `archive_scores` | null (disabled), false; separate versioned collection records | - -`OneCycleLR.total_steps` must equal the maximum authored optimizer-step budget -(including accumulation and a final partial rollout); KL stopping may use fewer -steps. Plateau/compound schedulers, distributed modes, layer autocast and ONNX are not -PPO config fields. Optimizer and scheduler constructor errors are raised before -collection. A final shorter rollout executes exactly the remaining actions; -terminal actions count toward the budget. Checkpoint/export paths and resume -modes are algorithm-owned and do not change the supervised or offline schemas. +## Policy runs + +Policy training uses one `PolicyRunConfig` with top-level ordered `phases`, +`model`, `policy_interface`, `context_length`, optional named `sources` and +`preferences`, and run-wide output/resume settings. Supported algorithms are +`policy_imitation`, `iql`, `ppo`, `dpo`, and `grpo`. Epochs, offline updates, +PPO environment steps and GRPO groups are explicit per-phase budgets. + +Each phase selects `handoff: best | last` (default `best`). One frozen actor +representation and causal history contract spans the run; all optimizer/critic +state starts fresh at each handoff. Return conditioning supports imitation only. +Policy runs are local FP32 runs and cannot contain supervised forecasting phases. +The supervised `training_plan`/`dataset_training`/`global_training` schema above +remains independent. + +See the [policy learning guide](../policy-learning.md) for full fields, objective +formulas, environment capabilities, preference manifests, checkpoint boundaries, +selection monitors and exact/non-exact continuation. Examples: + +- [BC](../example_inputs/configs/train-bc-toy.yaml) +- [Return-conditioned imitation](../example_inputs/configs/train-return-conditioned.yaml) +- [IQL](../example_inputs/configs/train-iql-from-table.yaml) +- [PPO](../example_inputs/configs/train-ppo-toy.yaml) +- [DPO](../example_inputs/configs/train-dpo-control.yaml) +- [GRPO](../example_inputs/configs/train-grpo-control.yaml) +- [Imitation → IQL → DPO → GRPO → PPO](../example_inputs/configs/train-multiphase-control.yaml) diff --git a/documentation/example_inputs/configs/infer-policy.yaml b/documentation/example_inputs/configs/infer-policy.yaml index a785e027..ae708cb8 100644 --- a/documentation/example_inputs/configs/infer-policy.yaml +++ b/documentation/example_inputs/configs/infer-policy.yaml @@ -1,5 +1,5 @@ project_root: . -model_path: models/policy/policy.pt +model_path: models/policy/actor.pt # Raw observations/static inputs in original units; no forecast preprocessing required. data_path: data/policy-observations.csv model_type: generative diff --git a/documentation/example_inputs/configs/infer-ppo-toy.yaml b/documentation/example_inputs/configs/infer-ppo-toy.yaml index 8e5f2968..3ac5a8f8 100644 --- a/documentation/example_inputs/configs/infer-ppo-toy.yaml +++ b/documentation/example_inputs/configs/infer-ppo-toy.yaml @@ -1,5 +1,5 @@ project_root: . -model_path: models/online-ppo/online-ppo.pt +model_path: models/online-ppo/actor.pt data_path: documentation/example_inputs/ppo-observations.csv model_type: generative read_format: csv diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index b101613a..13fcfb92 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -1,5 +1,5 @@ project_root: . -model_path: models/online-ppo/online-ppo.pt +model_path: models/online-ppo/actor.pt data_path: documentation/example_inputs/ppo-observations.csv model_type: generative read_format: csv diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml index 9d5b7219..f154972c 100644 --- a/documentation/example_inputs/configs/train-bc-toy.yaml +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -25,21 +25,16 @@ model: output_dim: 8 feature_embedding_dims: {position: 2, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: {paths: [data/offline-transitions.csv]} -global_training: - training_objective: policy - context_length: 16 +policy_interface: actor +context_length: 16 +sources: + demonstrations.recorded: + paths: [data/offline-transitions.csv] +validation_fraction: 0.2 +phases: +- name: behavior_cloning + algorithm: policy_imitation + sources: [demonstrations.recorded] + epochs: 30 batch_size: 64 - learning_rate: 0.0003 - validation_fraction: 0.2 -training_plan: - phases: - - name: behavior_cloning - algorithm: policy_imitation - policy: actor - sources: [{source: demonstrations.recorded}] - epochs: 30 + actor_learning_rate: 0.0003 diff --git a/documentation/example_inputs/configs/train-dpo-control.yaml b/documentation/example_inputs/configs/train-dpo-control.yaml new file mode 100644 index 00000000..013a145a --- /dev/null +++ b/documentation/example_inputs/configs/train-dpo-control.yaml @@ -0,0 +1,44 @@ +project_root: . +model_name: dpo-control +device: cpu +seed: 1010 +context_length: 4 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 1 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 64} + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: [cue] + target_columns: [action] + policy: + observation_columns: [cue] + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +policy_interface: actor +normalization: {source: identity} +phases: +- {name: preferences, algorithm: dpo, preferences: rankings, updates: 100, beta: 0.1} +sources: + decisions: + paths: [data/control-decisions.csv] + columns: + episode_id: episodeId + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {cue: next_cue} + group: contextId +preferences: {rankings: data/control-preferences.json} diff --git a/documentation/example_inputs/configs/train-grpo-control.yaml b/documentation/example_inputs/configs/train-grpo-control.yaml new file mode 100644 index 00000000..81367eb2 --- /dev/null +++ b/documentation/example_inputs/configs/train-grpo-control.yaml @@ -0,0 +1,50 @@ +project_root: . +model_name: grpo-control +device: cpu +seed: 1010 +context_length: 4 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 1 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 64} + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: [cue] + target_columns: [action] + policy: + observation_columns: [cue] + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +policy_interface: actor +normalization: {source: identity} +phases: +- name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_update: 2 + group_size: 4 + max_steps: 8 + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 + metric: score_mean diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index 68c2524d..57074ab2 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -2,9 +2,6 @@ project_root: . model_name: offline-iql device: cpu seed: 1010 -# Optional compatible, unconditioned BC actor artifact: -# initialize_from: models/behavior-cloning/behavior-cloning.pt - model: backbone: architecture: @@ -26,51 +23,37 @@ model: ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - position: 1 - velocity: 1 - __policy_previous_action: 6 + feature_embedding_dims: {position: 1, velocity: 1, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} - -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: [data/offline-transitions.parquet] - columns: - sequence_id: sequenceId - item_position: itemPosition - action: action - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {position: next_position, velocity: next_velocity} - -global_training: - training_objective: policy - context_length: 16 +policy_interface: actor +context_length: 16 +sources: + demonstrations.recorded: + paths: [data/offline-transitions.parquet] + columns: + sequence_id: sequenceId + item_position: itemPosition + action: action + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {position: next_position, velocity: next_velocity} +validation_fraction: 0.2 +resume: {policy: if_exists} +phases: +- name: offline_improvement + algorithm: iql + sources: [demonstrations.recorded] + updates: 10000 batch_size: 64 - learning_rate: 0.005 - validation_fraction: 0.2 - resume: {policy: if_exists} - -training_plan: - phases: - - name: offline_improvement - algorithm: iql - policy: actor - sources: [{source: demonstrations.recorded}] - updates: 10000 - batch_size: 64 - gamma: 0.99 - expectile: 0.7 - inverse_temperature: 3.0 - max_weight: 100.0 - target_update_rate: 0.005 - actor_learning_rate: 0.0003 - value_learning_rate: 0.0003 - critic_learning_rate: 0.0003 - evaluation_interval: 100 - checkpoint_interval: 100 + gamma: 0.99 + expectile: 0.7 + inverse_temperature: 3.0 + max_weight: 100.0 + target_update_rate: 0.005 + actor_learning_rate: 0.0003 + value_learning_rate: 0.0003 + critic_learning_rate: 0.0003 + evaluation_interval: 100 + checkpoint_interval: 100 diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml index 1c22bfde..9c63cff9 100644 --- a/documentation/example_inputs/configs/train-iql-toy.yaml +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -2,7 +2,7 @@ project_root: . model_name: toy-iql device: cpu seed: 1010 -initialize_from: models/toy-bc/toy-bc.pt +initialize_from: models/toy-bc/actor.pt model: backbone: architecture: @@ -26,32 +26,24 @@ model: output_dim: 8 feature_embedding_dims: {position: 2, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: [data/offline-transitions.csv] - columns: - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {position: next_position} -global_training: - training_objective: policy - context_length: 16 +policy_interface: actor +context_length: 16 +sources: + demonstrations.recorded: + paths: [data/offline-transitions.csv] + columns: + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {position: next_position} +validation_fraction: 0.2 +resume: {policy: if_exists} +phases: +- name: offline_improvement + algorithm: iql + sources: [demonstrations.recorded] + updates: 10000 batch_size: 64 - learning_rate: 0.005 - validation_fraction: 0.2 - resume: {policy: if_exists} -training_plan: - phases: - - name: offline_improvement - algorithm: iql - policy: actor - sources: [{source: demonstrations.recorded}] - updates: 10000 - batch_size: 64 - evaluation_interval: 100 - checkpoint_interval: 100 + evaluation_interval: 100 + checkpoint_interval: 100 diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml new file mode 100644 index 00000000..ad46a13a --- /dev/null +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -0,0 +1,97 @@ +project_root: . +model_name: multiphase-control +device: cpu +seed: 1010 +context_length: 4 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 1 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 64} + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: [cue] + target_columns: [action] + policy: + observation_columns: [cue] + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +policy_interface: actor +normalization: {source: identity} +phases: +- name: imitation + algorithm: policy_imitation + sources: [decisions] + epochs: 4 +- name: offline_improvement + algorithm: iql + sources: [decisions] + updates: 100 +- {name: preferences, algorithm: dpo, preferences: rankings, updates: 100, beta: 0.1} +- name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_update: 2 + group_size: 4 + max_steps: 8 + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 + metric: score_mean +- name: ppo_improvement + algorithm: ppo + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + actor_learning_rate: 0.0003 + value_learning_rate: 0.0003 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 +sources: + decisions: + paths: [data/control-decisions.csv] + columns: + episode_id: episodeId + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {cue: next_cue} + group: contextId +preferences: {rankings: data/control-preferences.json} diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml index 5b02c79f..636a537c 100644 --- a/documentation/example_inputs/configs/train-policy-from-table.yaml +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -2,8 +2,6 @@ project_root: . model_name: table-policy device: cpu seed: 1010 - -# The ordinary model component schemas are also used for policy training. model: backbone: architecture: @@ -25,32 +23,18 @@ model: ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - position: 1 - velocity: 1 - __policy_previous_action: 6 + feature_embedding_dims: {position: 1, velocity: 1, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} - -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: [data/demonstrations.parquet] - # BC requires sequenceId, itemPosition, position, velocity and action. - # Rewards, boundaries and simulator recording metadata are optional. - -global_training: - training_objective: policy - context_length: 16 +policy_interface: actor +context_length: 16 +sources: + demonstrations.recorded: + paths: [data/demonstrations.parquet] +validation_fraction: 0.2 +phases: +- name: behavior_cloning + algorithm: policy_imitation + epochs: 80 + sources: [demonstrations.recorded] batch_size: 64 - learning_rate: 0.005 - validation_fraction: 0.2 - -training_plan: - phases: - - name: behavior_cloning - algorithm: policy_imitation - policy: actor - epochs: 80 - sources: [{source: demonstrations.recorded}] + actor_learning_rate: 0.005 diff --git a/documentation/example_inputs/configs/train-ppo-toy.yaml b/documentation/example_inputs/configs/train-ppo-toy.yaml index 7dd574f2..451140c0 100644 --- a/documentation/example_inputs/configs/train-ppo-toy.yaml +++ b/documentation/example_inputs/configs/train-ppo-toy.yaml @@ -26,32 +26,33 @@ model: output_dim: 8 feature_embedding_dims: {cue: 2, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} -training_plan: - phases: - - name: online_control - algorithm: ppo - policy: actor - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - normalization: {source: identity} - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - actor_learning_rate: 0.0003 - value_learning_rate: 0.0003 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 - resume: {policy: if_exists, mode: exact} +policy_interface: actor +normalization: {source: identity} +resume: {policy: if_exists, mode: exact} +phases: +- name: online_control + algorithm: ppo + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + actor_learning_rate: 0.0003 + value_learning_rate: 0.0003 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index 94b55469..b40f0015 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -8,71 +8,40 @@ model: dim_model: 32 num_layers: 2 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 128 - activation: swiglu + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: &id001 - - OBSERVATION_COLUMN - target_columns: - - action + input_columns: [OBSERVATION_COLUMN] + target_columns: [action] policy: - observation_columns: *id001 + observation_columns: [OBSERVATION_COLUMN] action_column: action - action_values: - - 0.0 - - 1.0 + action_values: [0.0, 1.0] return_conditioned: true - returns: - gamma: 1.0 - scale: 1.0 + returns: {gamma: 1.0, scale: 1.0} ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - OBSERVATION_COLUMN: 1 - __policy_previous_action: 6 - __policy_return: 1 - decoder: - type: linear - prediction_length: 1 - support: 1 -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: - - data/demonstrations.parquet -global_training: - training_objective: policy - context_length: 4 + feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} + decoder: {type: linear, prediction_length: 1, support: 1} +model_path: models/policy/actor.pt +report_path: outputs/policy/training-report.json +policy_interface: actor +context_length: 4 +sources: + demonstrations.recorded: + paths: [data/demonstrations.parquet] +validation_fraction: 0.2 +episode_cache_size: 8 +resume: {policy: never} +phases: +- name: offline_policy + algorithm: policy_imitation + epochs: 80 + sources: [demonstrations.recorded] batch_size: 64 - num_workers: 0 - episode_cache_size: 8 - resume: - policy: never - learning_rate: 0.005 gradient_clip: 1.0 - validation_fraction: 0.2 - optimizer: - name: Adam - scheduler: - name: ConstantLR - factor: 1.0 - total_iters: 1 -training_plan: - phases: - - name: offline_policy - algorithm: policy_imitation - epochs: 80 - sources: - - source: demonstrations.recorded - policy: actor -model_path: models/policy/policy.pt -report_path: outputs/policy/training-report.json + actor_learning_rate: 0.005 diff --git a/documentation/example_inputs/policy_control_data.py b/documentation/example_inputs/policy_control_data.py new file mode 100644 index 00000000..77e0c217 --- /dev/null +++ b/documentation/example_inputs/policy_control_data.py @@ -0,0 +1,89 @@ +"""Generate curated sequence preferences for the multi-phase control examples. + +Usage: python documentation/example_inputs/policy_control_data.py data +Preferences are supplied by a sign-matching rule, independent of reward sums. +This script only creates example data; it does not train or evaluate a model. +""" + +import csv +import json +import sys +from pathlib import Path + + +def generate(directory): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + pairs = [] + with (directory / "control-decisions.csv").open( + "w", newline="", encoding="utf-8" + ) as handle: + fields = [ + "sequenceId", + "episodeId", + "contextId", + "itemPosition", + "cue", + "action", + "reward", + "terminated", + "truncated", + "bootstrap_allowed", + "next_cue", + ] + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + for context in range(40): + initial = (-1 if context % 2 else 1) * (1 + context / 100) + alternatives = {} + for kind, length in (("chosen", 8), ("rejected", 6)): + identity = f"context-{context}-{kind}" + cue = initial + for step in range(length): + matched = "right" if cue > 0 else "left" + action = ( + matched + if kind == "chosen" + else ("left" if matched == "right" else "right") + ) + writer.writerow( + { + "sequenceId": identity, + "episodeId": identity, + "contextId": f"context-{context}", + "itemPosition": step, + "cue": cue, + "action": action, + "reward": 1 if action == matched else -1, + "terminated": step + 1 == 8, + "truncated": step + 1 == length and length != 8, + "bootstrap_allowed": step + 1 != 8, + "next_cue": -cue, + } + ) + cue = -cue + alternatives[kind] = { + "source": "decisions", + "episode_id": identity, + "start": 0, + "end": length, + } + pairs.append( + { + "pair_id": f"pair-{context}", + "context_id": f"context-{context}", + **alternatives, + } + ) + (directory / "control-preferences.json").write_text( + json.dumps( + {"format": "sequifier.preference-pairs", "version": 1, "pairs": pairs}, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + +if __name__ == "__main__": + generate(sys.argv[1] if len(sys.argv) > 1 else "data") diff --git a/documentation/example_inputs/ppo_toy_control.py b/documentation/example_inputs/ppo_toy_control.py index f18a29f0..25128ca0 100644 --- a/documentation/example_inputs/ppo_toy_control.py +++ b/documentation/example_inputs/ppo_toy_control.py @@ -1,29 +1,24 @@ -"""Python PPO workflow. Run from the repository root after installing Sequifier. +"""Train the categorical control example with the canonical Python policy API.""" - python documentation/example_inputs/ppo_toy_control.py - -The bundled deterministic adapter is implemented in sequifier.rl.control. -Its random-policy expected return is 0; optimal eight-action return is 8. -This example trains a policy; it is not a test or a validation benchmark. -""" - -from typing import cast - -from sequifier.config.train_config import PPOTrainConfig, load_train_config -from sequifier.rl_api import evaluate_policy, load_policy, run_ppo_training +from sequifier.config.train_config import load_train_config +from sequifier.rl_api import ( + PolicyRunConfig, + evaluate_policy, + load_policy, + run_policy_training, +) def main(): - loaded_config = load_train_config( + config = load_train_config( "documentation/example_inputs/configs/train-ppo-toy.yaml", {}, False ) - if not isinstance(loaded_config, PPOTrainConfig): - raise TypeError("the example requires an online PPO config") - config = cast(PPOTrainConfig, loaded_config) - result = run_ppo_training(config) - actor = load_policy(config.model_path) - print("Best evaluated return:", result.best_metric) - print(evaluate_policy(actor, config.training_plan.phases[0].evaluation)) + if not isinstance(config, PolicyRunConfig): + raise TypeError("the example requires a policy run config") + result = run_policy_training(config) + actor = load_policy(result.model_path) + print("Selected return:", result.phases[-1].selected_metric) + print(evaluate_policy(actor, config.phases[0].evaluation)) if __name__ == "__main__": diff --git a/documentation/plans/dpo-grpo-multiphase-implementation-plan.md b/documentation/plans/dpo-grpo-multiphase-implementation-plan.md new file mode 100644 index 00000000..ee0ae73f --- /dev/null +++ b/documentation/plans/dpo-grpo-multiphase-implementation-plan.md @@ -0,0 +1,390 @@ +# DPO, GRPO, and multi-phase policy training implementation plan + +Status: proposed, 2026-11-09. This plan builds on the policy-learning changes +since `2d01d407cd2134c8197b1a18f0034d01486f18a9`, the +[IQL plan](offline-iql-implementation-plan.md), and the +[PPO plan](scoped-ppo-implementation-plan.md). It is an implementation plan, +not a claim that these features exist. All policy-learning changes since that +commit are unreleased and may be redesigned or removed. There is no requirement +to migrate their configs, APIs, artifacts, checkpoints, or adapter protocols. +The verification work described below is for a subsequent implementation; no +tests are to be written or run as part of producing this plan. + +## 1. Outcome and scope + +Support ordered, same-run combinations of `policy_imitation`, `iql`, `ppo`, +`dpo`, and `grpo` phases through the training CLI and Python API. Each phase +has an explicit budget, data/environment contract, evaluation, and selection +rule. A phase produces a selected actor and hands it to the next phase; the +final selected actor is a portable Sequifier model usable for policy inference +and rollout through the redesigned interfaces. + +The first release targets this policy boundary: one fixed +categorical action, scalar/categorical observation and static fields, causal +fixed-length history, local single-process execution, and one selected policy +interface. DPO compares alternatives from the same initial history; GRPO samples +multiple trajectories from the same initial condition. This is a control and +sequence-policy implementation, not a claim of LLM-tokenizer, text-prompt, +continuous-action, vectorized, distributed, or arbitrary reward-model support. +The design includes imitation, return-conditioned imitation, IQL, and PPO. +Return-conditioned policies are available for imitation; IQL, PPO, DPO, and GRPO initially require an +unconditioned actor, with an explicit preflight error for incompatible plans. + +A multi-phase plan is sequential, not a mixture of concurrently trained +algorithms. Supervised forecasting phases are not part of this policy plan: +reject mixing them with policy phases rather than implying an unspecified +conversion. Cross-run composition uses the same portable actor artifact contract. + +## 2. Product and execution contracts + +### 2.1 Authoring and dispatch + +Add a canonical `PolicyRunConfig` with run-wide model, policy interface, +context length, device, seed, output paths, source catalog, and a discriminated +list of phase configurations. Offline phases reference source names; online +phases declare an environment, evaluation, and frozen normalization. Specify +budgets in their natural units: epochs, offline updates, PPO environment steps, +or GRPO groups. Validate unique phase names and all references before training. +Do not overload supervised `ResolvedTrainingPhase`, which assumes epoch-based +execution and supports only supervised/imitation algorithms. + +Use `PolicyRunConfig` as the sole policy-training configuration, with a single +result type containing ordered phase results and the final actor. Route both CLI +and Python entry points through the same validation and coordinator. Delete or +rewrite the unreleased `PolicyTrainConfig`, `PPOTrainConfig`, their separate +dispatch paths, and redundant public entry points as needed. Design one coherent +CLI and `rl_api` around the new contracts; no old policy syntax, default, result +shape, output path, or import needs an adapter. Keep the pre-existing supervised +training path independent, and reject mixed supervised/policy plans. + +Replace duplicated closed-set branching with a **built-in explicit registry** +that maps algorithm identifiers to phase config type and runtime factory. +Pydantic's static discriminated union still explicitly lists built-in phase +classes; the registry is not a promise of third-party dynamic config plugins. +One registration inventory drives dispatch and a consistency check verifies +that it agrees with the union. Algorithm-specific configuration and validation +remain in their own phase models; avoid a generic dictionary of arbitrary +options or an algorithm-independent `update()` abstraction that conceals +required inputs. Update typing, `rl_api`, docs and CLI examples together. + +### 2.2 Frozen actor representation and phase handoff + +Resolve one immutable actor representation before the first phase: architecture, +interface, context length, feature ordering/types, categorical vocabularies, +action IDs and ordering, normalization and units, and return conditioning. +The first offline phase may fit codecs and statistics on its training partition +only; persist the resulting complete representation before any update. An +online-first run must declare a frozen representation or initialize from a +compatible artifact. A later phase must reuse the frozen representation: never +refit from its own data or silently substitute identity statistics. Preflight +all explicitly declared phase representations and artifact contracts against it; +validate data-dependent details when the first phase has fitted them and before +starting later phases. Reject incompatible architecture, codec, normalization, +units, context, or conditioning with an actionable field-level diagnostic. + +Each phase receives a deep-copied **selected** actor from the previous phase, +not a mutable handle to the previous trainer. The first phase starts from the +run's `initialize_from` actor if supplied, otherwise from a seeded fresh actor. +Each phase declares `handoff: best | last` (default `best`), its supported +monitor and direction, and a deterministic tie rule: retain the earliest best. +`best` requires a finite metric and an evaluation schedule capable of producing +one; fail rather than silently use `last`. `last` means the most recently +completed optimizer transaction. Save both selected provenance and metric. +The final export is the final phase's selected actor, not the globally best +actor across incomparable metrics. + +Optimizers, schedulers, samplers, critics and mutable environment state are +phase-local and start fresh at a phase boundary. Do not transfer optimizer state +or reuse old on-policy samples between algorithms. Warm-starting critics or +optimizer state is outside this release. Use explicit phase indices and names in +paths and reports to prevent collisions. Protect all inputs, including +preference manifests and reference artifacts, from output overwrites. + +### 2.3 Reference policies + +DPO and, in this plan, GRPO require a frozen reference actor. Each declares +`reference: {source: phase_start}` or `{source: artifact, path: ...}`; +`phase_start` is the default. A reference is a deep copy of the incoming actor +*before* any update, in evaluation mode, with gradients disabled. For a fresh +first phase it is the initialized actor. Validate the full representation and +architecture against the trainable actor and fingerprint the actual weights, +not merely the path. Never silently retarget a reference at the end of an +epoch or phase. Reference weights or an immutable content-addressed artifact +and its verified hash must survive resume even if the original file changes or +disappears; use embedded checkpoint weights in the first implementation. + +DPO's reference enters the chosen/rejected log-ratio. GRPO's reference is used +only for its explicitly configured KL penalty; it is distinct from the frozen +**old collection policy** whose scores define the importance ratio. GRPO still +records a reference identity when its KL coefficient is zero so its run +contract is unambiguous. + +## 3. DPO data and objective + +### 3.1 Preference contract + +Add a versioned, inspectable preference-pair manifest and an offline dataset +view in `trajectory/` and `io/`. A pair has a unique ID, a declared common +initial history/context identity, and chosen and rejected alternatives, each +referencing an episode or a contiguous decision span in a supported +sequence source. Define span bounds as zero-based start-inclusive/end-exclusive; +require at least one scored action per side. Alternative histories must have +identical canonical initial observations, static inputs, valid masks and prior +action history at the branch point, under the frozen codec. After branching, +each alternative uses its own actual observations and prior actions. The pair +record need not invent rewards, successor states, or terminal flags for DPO; +design sequence sources to support reward-free preference data as well as +reward-bearing trajectories. +Never fabricate a reward-bearing `Trajectory` merely to satisfy its validator. +Reject missing actions, unsupported labels, ambiguous alignment, non-finite +features, duplicate pair IDs, overlapping chosen/rejected identities that make a +preference meaningless, and incompatible initial contexts. Permit unequal +alternative lengths; record exact decision masks and length diagnostics. + +Split before fitting representation. Form connected components across shared +context/group IDs, episode IDs, and duplicate content, then put each component +entirely in train or validation. Explicit validation sources must satisfy the +same no-overlap checks. If too few independent components remain, fail with a +clear error. Avoid materializing all windows: reuse `PolicyWindowView` and +bounded episode caches; persist source identities, span definitions, content +fingerprints, split membership and codec identity for exact resume. Comparison +semantics are supplied by the data: do not infer preference from total reward. + +### 3.2 Sequence likelihood and optimization + +Add a differentiable sequence-scoring helper over `CategoricalPolicy`: score +the recorded executable action at each selected decision using the causal +window valid at that decision, then sum FP32 log probabilities per alternative. +Padding and unselected context positions contribute zero. Ensure start/special +tokens are never executable labels. Chunk windows to bound memory while +preserving gradients for trainable scores. Reference scoring uses no gradients, +identical input transformations, and evaluation mode. Stochastic network layers +remain disabled during both reference and trainable scoring for reproducible +likelihood ratios; gradients are still enabled on the trainable actor. + +For each pair, with `s_theta(y)` the summed action log likelihood and `s_ref` +the frozen reference score, implement canonical sigmoid DPO: + +```text +margin = (s_theta(chosen) - s_theta(rejected)) + - (s_ref(chosen) - s_ref(rejected)) +loss = -logsigmoid(beta * margin) +``` + +Require finite `beta > 0`; average losses over pairs. Do not silently switch to +per-token averaging, add a supervised objective, or treat a trajectory reward +as a preference. A pair contributes equally even when alternatives have unequal +lengths; report lengths, margins and preference accuracy to expose that choice. +Support a phase-local optimizer/scheduler, gradient accumulation weighted by +actual pair count, clipping, deterministic sampler, checkpoint interval, and +held-out DPO loss as the default minimizing selection metric. Define evaluation +frequency and save/checkpoint boundaries in optimizer updates. Reference scores +may be cached only against content, span, codec and reference weight hashes; +a cache miss or mismatch must recompute, never silently reuse stale scores. + +## 4. GRPO collection and objective + +### 4.1 Reproducible group starts + +Define an online environment contract with an explicit group-start capability. It must produce an opaque, serializable start token and initialize +independent episodes from that token, yielding the same initial observation, +static inputs and units. Specify whether the token captures simulator randomness: +independent rollout action/environment RNG streams must be derived from run +seed, group index and sample index, while the initial condition remains the +same. An adapter may implement this by restoring a captured initial snapshot or +by a deterministic scenario/reset factory, but must explicitly declare the +capability. A repeated seed alone is insufficient. Validate initial histories +and units across a group and reject unsupported adapters before collection. +Use one coherent adapter protocol for online algorithms, with group-start as an +explicit capability required by GRPO. Redesign the current unreleased adapter +protocol and bridge as needed; do not infer capabilities from method names alone. + +One group contains `group_size >= 2` independent trajectories from the same +start token, generated sequentially in the first implementation under one +frozen old actor version. Use distinct action RNG streams. Each episode ends at +a real terminal or truncation, or at an explicit per-sample `max_steps` cutoff; +the scalar score is the finite sum of observed rewards up to that boundary, +with a declared fixed reward scale. A cutoff is an explicitly accepted +finite-horizon score, not an invented terminal or a bootstrap. If an adapter +fails, a score is non-finite, or a requested complete group cannot fit within +the configured memory/step budget, abort the whole uncommitted group and never +update from a partial group. Close adapters in `finally`. `total_groups` is the +hard authored budget; optional `max_env_steps` must be checked before starting +a group against the conservative `group_size * max_steps` bound. Report actual +executed actions as well as groups, including discarded actions if a failure is +reported. No automatic retry that silently changes data. + +Store group/start identity, episode and step IDs, action histories/masks, +rewards, cutoff reason, fixed old per-action log probabilities, reference +identity, score, and actor generation in a bounded sealed group buffer. Retain +only enough causal frames to reconstruct each action history. Verify old scores +against the current actor before the first update; enforce one generation and +consume each sealed buffer only once. Keep archival optional and distinct from +training input, as with PPO. Groups are collected under `no_grad` and no actor +updates happen until all groups for an iteration have been sealed. + +### 4.2 Chosen GRPO variant + +Specify this exact first variant rather than treating all GRPO variants as +interchangeable. Collect `groups_per_update` complete groups. For group scores +`R_i`, compute FP32 population mean and standard deviation, and assign the same +advantage to every valid decision of trajectory `i`: + +```text +A_i = (R_i - mean(R)) / (std_population(R) + advantage_epsilon) +r_it = exp(log pi_theta(a_it|h_it) - log pi_old(a_it|h_it)) +K_it = exp(log pi_ref(a_it|h_it) - log pi_theta(a_it|h_it)) + - (log pi_ref(a_it|h_it) - log pi_theta(a_it|h_it)) - 1 +loss = mean_over_trajectories(mean_over_valid_decisions( + -min(r_it * A_i, clamp(r_it, 1-epsilon, 1+epsilon) * A_i) + + kl_coefficient * K_it)) +``` + +This sampled reverse-KL estimator is nonnegative per decision, but its +expectation is a KL only under the corresponding sampling distribution; after +multiple optimization epochs the samples are from `pi_old`, so report it as a +sampled reference penalty, not an exact current-policy KL. Scores from the old +actor and fixed reference remain detached for every optimization epoch. +`advantage_epsilon > 0` is explicit; identical group scores yield zero +advantages, though a nonzero reference penalty can still update the actor. This +variant weights each trajectory equally irrespective of length and each group +equally because group sizes are fixed. Minibatches contain whole trajectories; +gradient accumulation weights by actual trajectory count, including the final +partial minibatch. Compute ratio, penalty and loss in FP32 with explicit finite +checks and fail before applying an invalid update. `kl_coefficient >= 0` and +`0 < epsilon < 1` are phase fields. Do not reuse PPO's old-policy approximate +KL as the reference penalty; an optional old-policy KL early-stop is separately +named and documented. GRPO has no value model, GAE, or value optimizer. + +Evaluation uses a separate adapter and independent deterministic evaluation +seeds/scenarios, never the training group RNG. Define score mean as the default +maximizing selection metric, evaluation interval in completed updates, and an +explicit final evaluation so `handoff: best` is meaningful. Report score +spread, zero-variance groups, sample lengths/cutoffs, KL penalty, clip fraction, +policy version, optimizer steps and real environment steps. + +## 5. Runtime, checkpoints, and artifacts + +Introduce a policy run coordinator responsible for validating the plan, +preparing the immutable actor representation, invoking a phase runtime, +committing phase results, and advancing to the next phase. Phase runtimes own +their natural lifecycle and algorithm-specific state. A narrow runtime +contract should accept a phase context and return a selected actor, typed phase +result, provenance and resumable state; it need not impose identical update +loops on imitation, IQL, PPO, DPO and GRPO. Factor shared actor loading and +compatibility, reference freezing, optimizer/scheduler construction, RNG +capture, safe atomic writes and report conventions only where their semantics +actually match. Keep PPO's collection-policy version separate from generic +phase/run IDs. Refactor imitation, IQL and PPO directly into the target runtime; simplify or +replace their current interfaces and implementation where this improves the +architecture. Specify their objectives, selection rules and defaults as part of +the new contracts rather than preserving incidental current behavior. + +Create a versioned multi-phase run checkpoint envelope containing the resolved +plan and immutable fingerprint, actor representation, run RNG state, current +phase index and status, completed phase results and selected actors, current +phase's algorithm identifier/version and opaque typed state, phase-local RNGs, +and the last committed transaction. Each algorithm owns validation of its +state: DPO includes actor/reference, optimizer/scheduler, sampler, dataset/split +identity, clocks and best selection; GRPO includes actor/reference, optimizer, +scheduler, group/seed counters, evaluation state and clocks. Write to a +temporary file and atomically replace only after a completed optimizer update +or phase handoff. No partial group or partially applied update is a checkpoint +boundary. On resume, check plan, data, model, reference hashes and capabilities +before mutating outputs. Resume at the current phase without rerunning completed +phases or reselecting their actor; a committed handoff must be idempotent. + +For GRPO, checkpoints are taken only after updates and with no live group. +Subsequent groups are derived from persisted group counters/start tokens and RNG +streams. Exact continuation requires the adapter's documented group-start +reproducibility and compatible implementation identity; validate both. If the +adapter cannot guarantee it, reject exact resume rather than claim equivalence. +An explicitly selected non-exact restart may begin new groups with new derived +seeds, report the discontinuity and preserve completed updates, but may never +reuse a partial group. PPO must likewise declare whether resuming restores +the environment exactly or resets it; exact continuation requires a validated +adapter snapshot, and a reset must be reported as a discontinuity. + +Use the new checkpoint envelope as the only policy-training checkpoint format. +Remove the unreleased algorithm-specific readers and migration paths where they +are no longer needed; do not add legacy format detection or conversion. +Portable inference artifacts remain actor-only. The run report records phase +order/config fingerprints, source/reference provenance, chosen metric and actor, +phase clocks, continuation mode and final artifact path. Checkpoint payloads can +contain training-only networks and references; inference loading must not depend +on them. + +## 6. Dependency-ordered delivery + +1. **Target contracts:** define the policy run/phase schemas, algorithm + registry, adapter capabilities, actor representation, phase-scoped paths and + structured results. Remove redundant policy configuration and dispatch. + Reject mixed supervised/policy plans and incompatible representations early. +2. **Coordinator and handoff:** implement the phase runtime contract, immutable + phase context, selected actor handoff and checkpoint envelope. Restructure + imitation, IQL and PPO to fit directly into the target architecture; define + their objectives and selection behavior explicitly. Demonstrate imitation → + IQL and imitation → PPO. +3. **DPO inputs and scoring:** implement pair manifest, source adapters, + component-safe splits, causal sequence log scoring and frozen reference + loading. Validate representation and prevent data leakage before training. +4. **DPO runtime:** add loss, optimizer, metrics, evaluation and resume; wire + the phase through CLI/Python and export. Exercise imitation → DPO and DPO → + another compatible algorithm as well as single-phase DPO. +5. **GRPO environment and buffers:** implement the shared online environment + contract, explicit group-start capability, reproducible independent rollout + streams, bounded complete-group collection and old/reference scoring. +6. **GRPO runtime:** add the specified group-normalized objective, actor-only + optimizer, separate evaluation and checkpoint/resume. Exercise imitation → + GRPO, DPO → GRPO, and a longer mixed plan with phase handoffs. +7. **Product integration:** document the new schemas, limitations, checkpoint + behavior, monitor meanings and sample configs. Update CLI help, public Python + exports and release notes; ensure final actors work with the redesigned + inference and rollout interfaces. Remove superseded unreleased policy code, + examples and documentation instead of maintaining parallel interfaces. + +## 7. Acceptance and verification for implementation + +The implementation is complete only when all items below have evidence. These +are future verification requirements, not tests executed for this plan. Use the +repository's actual test ownership and commands discovered at implementation +time; do not assume an external test checkout exists. + +- **Algorithm coverage:** single-phase BC, return-conditioned imitation, IQL, + PPO, DPO and GRPO work under the new contracts. Verify their stated objectives + and selection behavior; unsupported combinations fail before training. The + supervised training path predating this policy work continues to function. +- **DPO data and math:** hand-computed pair scores, causal masking, unequal + lengths, objective and gradient direction match the equations; actor equal to + reference gives zero margin and loss `log(2)`. Reference parameters never + receive gradients. Reject invalid pairs and train/validation leakage. +- **GRPO data and math:** identical group starts and independent reproducible + streams are established; analytic group normalization, clipping for both + advantage signs, reference penalty, per-trajectory weighting and zero-spread + groups match the declared variant. No critic is allocated, stale policy + versions and incomplete groups are rejected, and buffers obey memory limits. +- **Lifecycle:** every supported phase transition transfers exactly the chosen + actor and frozen codec; optimizers, samplers, reference policies and critics + have the stated ownership. Best/last behavior, evaluation failures, source + overlaps and output path collisions are explicit. A deterministic toy control + task demonstrates usable DPO and GRPO learning without making a universal + learning-quality guarantee. +- **Resume:** interruption at each committed update and phase boundary produces + equivalent subsequent state under the supported exact-resume contract; + changed data, reference weights, codec, config or adapter identity is + rejected. Non-exact environment reset is labeled, and partial groups are + never consumed. Only the new policy checkpoint format is required. +- **End-to-end:** run representative single- and multi-phase YAML configs via + CLI and Python, check phase reports and counters, then load the final ordinary + actor for inference and rollout through the redesigned interfaces. Verify deterministic reproducibility + on a supported local device and document any backend-specific limits. + +## 8. Deliberately deferred + +LLM/prompt-completion support, continuous or multi-component action spaces, +remote/vectorized environments, distributed/mixed-precision execution, +learned reward models, automatic preference generation, online preference +collection, asynchronous groups, dynamic external algorithm plugins, +supervised-to-policy phases, cross-phase optimizer/critic inheritance, and +mid-group exact checkpointing require their own contracts. Their omission must +not be represented as general support for those workflows. diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 87695d3c..80d6c1e7 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -1,698 +1,428 @@ -# Policy learning through `sequifier train` and `sequifier infer` - -Policy learning uses Sequifier's configurable ingestion, transformer network, -decoders, portable model artifacts, optimizer transactions, and run checkpoints. -A policy adds a decision-alignment contract to a model interface: at time `t`, -the inputs contain the current observation, the **previous** action, and, -optionally, a desired return. The current action is the supervised target. - -## Train from ordinary sequence tables - -The phase algorithm is `policy_imitation` for both behavior cloning and -return-conditioned imitation. The policy's `return_conditioned` setting selects -between them. Offline IQL uses a separate `iql` phase with an unconditioned -categorical actor. - -Use the existing `sequenceId` and `itemPosition` conventions. Each row describes -**the current observation, the action executed at that observation, and the -reward/boundary resulting from that action**. One sequence is one episode or an -explicit episode fragment. CSV and Parquet files can contain many episodes; -files are not assumed to be episode boundaries. Positions must be nonnegative, -unique and contiguous within each sequence, including across input shards. - -```yaml -project_root: . -model_name: table-policy -device: cpu -seed: 1010 - -model: - backbone: - architecture: - dim_model: 32 - num_layers: 2 - max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} - positional_encoding_scope: global - interfaces: - actor: - input_columns: [position, velocity] - target_columns: [action] - policy: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - return_conditioned: false - ingestion: - type: embedding - output_dim: 8 - feature_embedding_dims: - position: 1 - velocity: 1 - __policy_previous_action: 6 - decoder: {type: linear, prediction_length: 1, support: 1} - -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: [data/demonstrations.parquet] - -global_training: - training_objective: policy - context_length: 16 - batch_size: 64 - learning_rate: 0.005 - validation_fraction: 0.2 - -training_plan: - phases: - - name: behavior_cloning - algorithm: policy_imitation - policy: actor - epochs: 80 - sources: [{source: demonstrations.recorded}] -``` - -Run `sequifier train --config-path configs/train-policy.yaml`. BC -(`return_conditioned: false`) needs observations and action labels, without -rewards, successor observations, bindings, simulator metadata or fabricated seed -windows. A recorded current action is never also accepted as an observation from -the same source column. Return-conditioned modeling additionally needs recorded -rewards and a known terminal outcome or an explicitly configured finite horizon. -Unknown/truncated reward tails are excluded and reported; missing rewards never -become zero. Setting `return_conditioned: true` selects this mode. - -Defaults use the policy feature names and the columns `sequenceId`, -`itemPosition`, `reward`, `terminated` and `truncated`. Map different source names -explicitly: - -```yaml -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: [data/recorded.csv] - columns: - sequence_id: episode - item_position: step - observations: {position: measured_position, velocity: measured_velocity} - action: command - reward: score - terminated: task_finished - truncated: time_limit - previous_action: preceding_command - group: collection_session -``` - -Boundary columns, when present, contain booleans. No row may follow a boundary -within the same episode. Both termination and truncation may be true; termination -disables bootstrap. `episode_end: terminated` or `truncated` is an explicit claim -about each sequence's final row and must agree with recorded flags. The default -is `unknown`, never an inferred terminal outcome at file end. - -Position zero with no preceding action is an episode start. Fragments require -`columns.previous_action` at their first position; subsequent entries must lag -current actions by exactly one decision. At window boundaries, the preceding -action is fetched from the sequence rather than replaced with a start token. -Static features can be constant inline columns or supplied in a separate -`static_data_path` CSV/Parquet table keyed by the sequence ID. The table must cover -every selected ID without duplicates; fitting reads only training episodes. - -Optional `columns.successors` maps every observation name to its next-observation -column. An all-null successor is explicitly missing; partial successors are -invalid. Adjacent observations must agree with supplied successors. Optional -`columns.bootstrap_allowed` cannot permit bootstrap after termination or at a -final decision without an observed successor. BC and return-conditioned modeling -do not require successors. Task outcomes and collection groups remain data -metadata rather than model features. - -Without `validation_sources`, training splits whole episodes deterministically, -keeping repeated demonstrations and declared collection groups together. At -least two independent groups are required. Alternatively declare a held-out source -and set `validation_sources: [held_out.recorded]` on the phase. Duplicate paths and episode -IDs, overlapping collection groups and repeated demonstrations across partitions -are rejected. Normalization and categorical vocabulary fitting use training data. - -Declare raw categorical observation/static fields with `categorical_columns` on -the policy. They use the original preprocessing vocabulary fitter and reserved -IDs. Alternatively supply fixed `observation_n_classes` / `static_n_classes` for -already encoded integer IDs and optional `id_maps` for fixed domain labels. -String action labels are supported through `action_values`. Executable action -IDs use the usual global-to-decoder mapping; the start marker is input-only and -cannot be sampled as an action. Previously saved policy artifacts retain their -legacy vocabulary layout. - -Sources accept `format: auto` (default), `table`, `trajectory` or `prepared`. -Automatic selection recognizes CSV/Parquet tables, JSON rollout logs and PT -prepared histories. Prepared CSV/Parquet windows require `format: prepared`. -Prepared sources require `metadata_config_path`; overlapping windows are checked -and reconstructed into unique, split-owned decisions. A preceding action can be -recovered from stored context for a fragment. Categorical IDs and normalized -values are decoded using the original metadata, then fitted for the selected -training partition. Missing reward/boundary information remains unknown. -For a metadata-referenced prepared split, use `metadata_config_path` and -`split_index` instead of `paths`. Scalar prepared histories are currently supported; -policy image/depth ingestion remains planned; IQL and PPO support the scalar -transition-capable sources described below. - -A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy -`manifest` listing version-1 JSON episodes. These are validated rollout recordings -adapted into the same sequence representation, with bindings retained in their -recording format. Forecast recordings require -`action_semantics: executed_predictions`; this does not convert forecast checkpoint -alignment into policy alignment. - -Run paths resolve relative to `project_root`; manifest entries resolve relative -to the manifest directory. `global_training.num_workers` controls loader workers; -`episode_cache_size` bounds cached episodes per worker. Table-backed episodes use -filtered lazy scans. Prepared window reconstruction is eager per source and JSON -records still require one episode to fit in memory. - -The best model is selected by held-out action NLL. `model_path` defaults to -`models//.pt` and contains an ordinary `sequifier_model`. -The report records partitions, exclusions, the fitted -policy codec, decision counts and per-epoch NLL. Epochs belong to the phase, seed -and device to the run, and optimizer/scheduler settings to `global_training`. - -## Optional committed import - -Direct training from a table needs no conversion. For reusable imported data, -`sequifier preprocess` supports a separate operation: - -```yaml -operation: import_sequences -project_root: . -source: - paths: [data/demonstrations.csv] -policy: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - return_conditioned: false -output_path: data/imported-demonstrations -episodes_per_shard: 64 -``` - -The output directory must be new. Import writes raw Parquet shards and a -checksummed `manifest.json`, then atomically publishes the directory. It does not -fit statistics, pad contexts or invent missing transitions. Train with -`dataset_training.demonstrations.parts.recorded.manifest: data/imported-demonstrations/manifest.json`; -the recorded column mappings are loaded automatically. The manifest retains -source provenance, episode identities, optional successors and collection groups. - -## Shared model and dataset authoring - -Policy training uses the ordinary `model.backbone` and `model.interfaces` -schemas, with a `policy` data schema on each interface. Ingestion and decoder -components keep their existing syntax; the policy adapter adds previous-action -and return features. `global_training.context_length` selects a context no larger -than the backbone maximum. `global_training` supplies optimizer settings and -`training_objective: policy`. The former compact policy registries and aliases -have been consolidated into this shared authoring surface. - -Likewise, `dataset_training.` accepts `model_interface` and `parts`, with each -part containing a sequence source specification. Reference it with the ordinary -`{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these -parts do not need fabricated criteria. Model/interface references are validated. -The shared phase schema preserves supervised defaults; policy training supports -one `policy_imitation`, `iql`, or `ppo` phase, with BC selected on its policy. -Mixed algorithms, backbone repositories, and image/depth policies remain planned. -Use `initialize_from` for backbone artifact transfer. - -See [train-policy-from-table.yaml](example_inputs/configs/train-policy-from-table.yaml) -for a complete example using the shared model, dataset and phase authoring surface. - -## Configure the shared model - -Configure the backbone at `model.backbone.architecture` and ingestion/decoder -components at `model.interfaces.`. For a larger model, for example: - -```yaml -model: - backbone: - architecture: - dim_model: 64 - num_layers: 3 - max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} - positional_encoding_scope: global +# Categorical policy learning + +`sequifier train` and `sequifier.rl_api.run_policy_training` execute the same +`PolicyRunConfig`. Its ordered `phases` may contain `policy_imitation`, `iql`, +`ppo`, `dpo`, and `grpo` in any compatible order. Each phase receives a deep copy +of its predecessor's selected actor. Optimizers, schedulers, critics, samplers, +and environments start fresh at phase boundaries. + +This release supports one fixed categorical action, scalar or categorical +observations/static fields, causal fixed-length histories, FP32, and local +single-process CPU/CUDA/MPS execution. Return conditioning is available only +for imitation. A run containing another algorithm requires an unconditioned +actor throughout. Forecasting and policy phases cannot share a training plan. +Text tokenizers, continuous/joint actions, images/depth, distributed training, +vectorized environments, and reward models are outside this contract. + +## Authoring a run + +Start with the [multi-phase example](example_inputs/configs/train-multiphase-control.yaml), +[single-phase DPO](example_inputs/configs/train-dpo-control.yaml), or +[single-phase GRPO](example_inputs/configs/train-grpo-control.yaml). +Generate their curated data with: + +```shell +python documentation/example_inputs/policy_control_data.py data +sequifier train --config-path documentation/example_inputs/configs/train-multiphase-control.yaml ``` -Use an embedding ingestion to project real features and embed categorical -features before adapting to the backbone width. Supply ingestion and decoder -components explicitly as in the complete examples. A policy decoder must have -`support: 1` and `prediction_length: 1`. - -The internal input names `__policy_previous_action` and `__policy_return` are -reserved. The latter exists only for return-conditioned policies. Static columns -are broadcast over the decision window. Real observation and static sample statistics -are persisted in the model interface's preprocessing metadata. Encoded categorical -observations can be declared with `observation_n_classes: {state_id: 8}`; their -recorded values must be integral IDs in `[0, 8)` and are not normalized. -When authoring a mixed real/categorical embedding ingestion, provide -`feature_embedding_dims` under the normal ingestion rules, including the internal -previous-action column and, when enabled, return column. - -Actions use decoder-local IDs in the order of `action_values`, mapped to the -standard global category IDs. A separate input-only ID represents episode start -and is excluded from decoder outputs. -An observation may have the same external name as the action; in that case the -model uses `__policy_action` as its separate target name. Default decoders infer -the target automatically. - -Context length must fit `max_context_length`. -The standard ingestion also supports -feature-scoped learned positions when configured. -The standard attention, normalization, position encodings, dropout, and layer -sharing options remain available. - -To transfer a forecast or policy backbone, set a top-level -`initialize_from: models/forecast.pt`. Portable models and run checkpoints are -accepted. The source must have the same backbone architecture. Ingestion and -decoder parameters are initialized for the policy's own interface. Resumption -restores the entire existing policy instead of applying initialization again. - -## Resume training - -Every completed epoch saves a standard `sequifier_run_checkpoint` at -`checkpoints/runs//-latest.pt`. It contains current model -weights, preprocessing and policy semantics, optimizer/scheduler/scaler state, -random state, data-order generator state, best weights, and training progress. -The portable model at `model_path` contains the best validation weights; the run -checkpoint contains the latest training weights. - -```yaml -global_training: - resume: - policy: if_exists -``` - -`never` is the default. `required` also requires an explicit `checkpoint_path`, -using the shared resume configuration. Policy runs use the same `TrainingRun`, -`TrainingEngine`, evaluation and checkpoint services as supervised runs. Resume -restores the saved optimizer boundary and loader positions. Trajectory content, partition order, policy, -and learning settings must match. Device, output paths, worker/cache settings, -and total epoch budget may change. Restored scheduler state retains its existing -schedule when extending the epoch budget. Numerical reproducibility across -devices is governed by PyTorch. - -Private policy run checkpoints written before the shared-engine consolidation -use a different run-state schema. Start a new run with `initialize_from` pointing -to the old model or checkpoint; its execution weights remain loadable. New runs -write ordinary Sequifier run state and loader state. - -## Predict actions with `sequifier infer` - -Inference reads policy alignment and preprocessing directly from a portable -model or run checkpoint; a training YAML or preprocessing metadata file is not -needed. It accepts `model_type: generative` (or the explicit alias `policy`): +The example defines imitation → IQL → DPO → GRPO → PPO. Remove or reorder phase +entries to author smaller runs, such as imitation → IQL, imitation → PPO, +imitation → DPO, imitation → GRPO, or DPO → GRPO. The input preferences come from +an explicitly supplied sign-matching rule; Sequifier never derives preferences +from recorded reward. These are illustrative configurations, without a claim +that every algorithm order improves every task. ```yaml project_root: . -model_path: models/policy/policy.pt -data_path: data/observations.csv -read_format: csv -write_format: csv +model_name: control +policy_interface: actor +context_length: 4 +seed: 1010 device: cpu -deterministic: true -desired_return: 10.0 +# model uses the ordinary backbone/interfaces architecture schema. +model: { ... } +sources: + decisions: + paths: [data/control-decisions.csv] + columns: + episode_id: episodeId + group: contextId +preferences: + rankings: data/control-preferences.json +phases: + - name: imitate + algorithm: policy_imitation + sources: [decisions] + epochs: 4 + - name: rank + algorithm: dpo + preferences: rankings + updates: 100 + beta: 0.1 + reference: {source: phase_start} +resume: {policy: if_exists, mode: exact} ``` -See [infer-policy.yaml](example_inputs/configs/infer-policy.yaml). CSV and Parquet -inputs contain observations and any static columns in original units. Without -a `sequenceId` column, each row is an independent first decision. Within each -sequence, `itemPosition` defines decision order when present and must be unique -and contiguous; otherwise row order is used. Static inputs must be constant. -Provide the recorded `action_column` to derive previous actions by shifting one -row, or set `previous_action_column` to a column containing previous **domain -action values**, with null for episode start. Use a new sequence ID for each -episode; null previous actions inside a sequence are rejected. When observation -and action names coincide, use an explicit `previous_action_column` for sequential -CSV/Parquet input. Current recorded actions never -enter the inputs for the same decision. Historical context is bounded by the -model's context length. - -Return-conditioned models accept a fixed `desired_return` or a per-row column -named by `desired_return_column` (default `desired_return`). Per-row returns are -supplied by the caller; offline inference does not update budgets from rewards. -BC requires no return input and rejects a scalar `desired_return`. - -`inference_batch_size` bounds batched model calls (default 64). Predictions go to `outputs/predictions//part-000.csv` by default. -`output_path` overrides that location. Sequence IDs and the configured `item_position_column` (default `itemPosition`), when -present, are retained in input order. `output_probabilities: true` adds one -`__probability_` column per action. Set -`deterministic: false` for seeded categorical sampling. Select `model_interface` -when an artifact contains multiple policy interfaces. - -## Execute an environment rollout - -Add the ordinary `rollout` configuration to policy inference: +Run-wide fields: + +| Field | Meaning / default | +| --- | --- | +| `model`, `policy_interface`, `context_length`, `model_name`, `phases` | Required architecture, selected interface, history length, run name, ordered phases | +| `project_root`, `device`, `seed`, `enforce_determinism` | `.`, `cpu`, 1010, true; seed in `[0, 2**32)` | +| `sources` | Named `SequenceSourceConfig` objects; paths/manifest, format, explicit column mappings, prepared metadata | +| `preferences` | Named paths to version-1 preference-pair JSON manifests | +| `units` | Frozen observation/static units; `{}` means unitless; online training/evaluation adapters must match | +| `initialize_from` | Optional portable actor with matching full representation and architecture | +| `normalization` | Optional explicit frozen `source: identity` or `source: statistics` with exact real-field `statistics: {column: {mean: ..., std: ...}}` | +| `validation_fraction`, `episode_cache_size` | 0.2 and 8; split independent components before fitting codecs/statistics | +| `resume` | `{policy: never, mode: reset_environments}`; policy also `if_exists`/`required`, mode also `exact` | +| `model_path` | `models//actor.pt` | +| `report_path` | `outputs/training//report.json` | +| `checkpoint_path` | `checkpoints//run.pt` | +| `phase_directory` | `outputs/training//phases`; subdirectories `000-`, etc. | + +The model must contain exactly `policy_interface`. Its input columns list the +policy observations followed by static fields; its target is the action column. +Use a scalar categorical decoder with `support: 1`, `prediction_length: 1`. +Declare `return_conditioned: false` for BC or any improvement algorithm. Standard +ingestion/backbone/decoder configuration remains available. Previous executable +actions enter the input as `__policy_previous_action`; its start token is +input-only. Return-conditioned imitation additionally uses `__policy_return`. + +The first offline phase fits missing categorical vocabularies and normalization +on its training episodes only. A declared normalization takes precedence over +fitting statistics. An online-first run must supply frozen normalization and +categorical counts/codecs, or a compatible `initialize_from` actor. Later phases +reuse the complete frozen representation. They may optionally declare +`representation` and online `normalization` as assertions; any disagreement +fails with a field-specific diagnostic before training. An online phase after +an offline phase normally omits `normalization` to reuse the fitted values. + +All input files, preference manifests, metadata, initial actors and references +are protected from output collisions. Source bytes, partitions, codecs, model, +units, reference weights, and adapter identities are checked on resume. + +## Selection and optimization + +Every phase has a unique identifier `name` and `handoff: best | last` (default +`best`). Best selection retains the earliest finite best on ties. The final +update always evaluates. Missing finite evaluation for `best` fails explicitly. +`last` selects the most recently completed optimizer transaction. The final +export uses the last phase's selected actor, with metric, transaction and weight +hash recorded; metrics from different phases are never compared globally. + +Imitation, DPO and GRPO share these actor optimization fields: + +| Field | Default | +| --- | --- | +| `batch_size`, `accumulation_steps` | 32, 1; DPO counts pairs, GRPO counts whole trajectories | +| `actor_learning_rate`, `actor_optimizer` | 0.0003, `{name: Adam}` | +| `actor_scheduler` | `{name: ConstantLR, factor: 1.0, total_iters: 1}` | +| `gradient_clip` | 1.0 | +| `evaluation_interval`, `checkpoint_interval` | 1; optimizer updates for imitation/DPO, completed collection/optimization updates for GRPO | + +Optimizers support Adam, AdamW, SGD and RMSprop. Their constructor arguments +belong alongside `name` (or in `arguments`); `lr` and parameters belong to the +phase. Schedulers advance on completed optimizer steps; plateau/compound +schedulers are unsupported. `OneCycleLR.total_steps` must equal the authored +maximum optimizer-step budget. Accumulation weights by the actual number of +examples, including final partial batches. Network stochastic layers are disabled +while gradients remain enabled; likelihood ratios use reproducible scores. + +## Offline sequences and imitation + +Sources accept CSV/Parquet tables, JSON rollout logs, committed sequence +manifests, or scalar prepared histories. A row has current observations and its +executed action; optional rewards/successors describe the resulting transition. +`PolicyTableColumns` exposes explicit observation/static/action/reward/group/ID +mappings. Temporal positions must be contiguous, zero-based at episode starts. +Fragments require a known preceding action. Reward-free sequences are ordinary +`SequenceEpisode` objects with absent transition capabilities, rather than +fabricated reward-bearing trajectories. + +`policy_imitation` uses `sources`, optional disjoint `validation_sources`, and +`epochs`. It minimizes action negative log likelihood at each valid causal +decision. The monitor is `validation_action_nll`, direction `min`. BC requires no +rewards. Return conditioning requires recorded rewards and either true task +termination or an explicit finite return horizon, using the declared discount +and scale. Unknown return tails are excluded. The input return label at decision +`t` includes reward `r_t`; the current action is never an input at that decision. + +Episode/group/duplicate-content components remain wholly in training or +validation. Explicit validation sources undergo the same overlap checks. +Episodes load through bounded caches; windows are constructed on demand. -```yaml -autoregressive: true -generation_steps: 1000 -rollout: - mode: policy - adapter: {factory: my_environment:create_adapter, config: {}} - bindings: - - {input: position, source: environment, key: position} - - {input: velocity, source: environment, key: velocity} - trajectory_path: outputs/trajectories/episode.json - seed_index: 0 -``` +## Offline IQL -`seed_index` selects one raw input row, forming a single-observation reset seed. -The shared rollout coordinator owns reset, step ordering, boundaries, and adapter -closure. Each predicted action enters policy history only after it is executed. -The [adapter protocol](rollout.md#adapter-protocol) is the same as for forecasts. -The resulting trajectory can be used as a later training source. +`iql` requires complete rewards, termination/truncation and bootstrap flags, +with genuine successors for bootstrapped decisions. Nonterminal transitions +bootstrap; true terminal transitions do not. Never substitute a reset +observation for a successor. + +The phase has `updates` (required), `batch_size` (64), `gamma` (0.99), +`expectile` (0.7), `inverse_temperature` (3), `max_weight` (100), and +`target_update_rate` (0.005). Independent value and twin-Q networks use +expectile regression and one-step Bellman regression with Polyak target-Q +updates. Actor extraction minimizes action NLL weighted by +`min(exp(inverse_temperature * (min(target_Q1,target_Q2)-V)), max_weight)`. +Each update samples transitions uniformly with replacement. Held-out unweighted +`validation_action_nll` is the minimizing selection monitor; Bellman residual, +values, Q and weight diagnostics are reported separately. + +`actor_`, `value_`, and `critic_learning_rate` default to 0.0003; each role has +its own optimizer/scheduler. `gradient_clip` defaults to 1.0, evaluation and +checkpoint intervals to 100 updates. Critics are fresh even after another IQL +phase and are absent from the portable actor. + +## DPO preferences + +The inspectable manifest is separate from its sequence sources: + +```json +{ + "format": "sequifier.preference-pairs", + "version": 1, + "pairs": [{ + "pair_id": "pair-1", + "context_id": "initial-condition-1", + "chosen": {"source": "decisions", "episode_id": "good", "start": 0, "end": 8}, + "rejected": {"source": "decisions", "episode_id": "bad", "start": 0, "end": 6} + }] +} +``` -The raw return budget starts at `desired_return` (or the seed row's return -column) and updates after each executed reward: +Span bounds index the loaded episode/fragment rows, zero-based, start-inclusive +and end-exclusive. Both sides need +at least one executable action. Unequal lengths are permitted. Canonical +observations, static fields, prior-action histories and valid masks must match +at the branch point. Each subsequent score uses that alternative's actual +observations and prior actions. Duplicate pair IDs, identical/overlapping +alternatives, unsupported labels, non-finite features and incompatible initial +contexts fail before updates. + +Splitting joins shared context/group IDs, episode IDs and duplicate learning +content transitively; explicit validation manifests obey the same constraints. +At least two independent components are required for automatic splitting. +Source identities, exact spans, fingerprints and membership persist in the run +checkpoint. The manifest must state the preferences; reward is not interpreted +as a preference. + +`dpo` requires `preferences` and `updates`; optional `validation_preferences` +selects an explicit validation manifest. `beta` defaults to 0.1 and must be +finite positive. `scoring_chunk_size` defaults to 64 causal windows. For the +summed FP32 action log likelihood `s`, it minimizes equally weighted pair losses: ```text -G_next = (G_current - reward) / gamma +margin = (s_actor(chosen)-s_actor(rejected)) - (s_ref(chosen)-s_ref(rejected)) +loss = -logsigmoid(beta * margin) ``` -Each reset restores the requested budget and clears policy history. The trajectory -records action log probabilities, previous-action IDs, and remaining raw budgets. -Termination and truncation remain separate; a collection cutoff preserves the -unfinished-episode flag. The coordinator always closes the adapter. - -## Labels, windows, and boundaries - -Raw return labels obey `G_t = r_t + gamma * G_(t+1)`, with terminal `G_T = 0`. -`returns.gamma` defaults to 1 and is in `(0, 1]`; `returns.scale` defaults to 1 -and divides the return only at ingestion. For rewards `[1, 2, 3]`, gamma 1 gives -`[6, 5, 3]`, and gamma 0.5 gives `[2.75, 3.5, 3]`. - -Return-conditioned training excludes truncations and collection cutoffs with -unknown tails. `returns.finite_horizon: H` can label a fully observed prefix of -H decisions. Shorter fragments need true termination. Online finite-horizon -execution requires `rollout.max_episode_steps: H`. No unknown tail is assigned -zero. BC uses **all observed decisions**, including truncated and cutoff episodes, -because its loss does not consume return labels. +Padding and unselected context positions contribute zero. Losses are averaged +over pairs, without length normalization or an added supervised term. Reports +include lengths, margin and preference accuracy. Default selection minimizes +`validation_dpo_loss`. Reference scores are recomputed, avoiding stale caches. +A two-pass backward calculation uses the exact scalar margin derivative and +recomputes one chunk's differentiable likelihood at a time, bounding activation +memory without changing the summed sequence objective. -Offline and online paths share `PolicyWindowView.inputs`, producing left-padded, -causal histories. The dataset slices to the context window before building -inputs, retaining the actual preceding action at a cropped boundary. Each -training example labels only its last position, so every decision contributes -once. Current actions, current rewards, and future observations are excluded -from inputs; return-to-go is the intentional future-derived condition. +DPO and GRPO each declare `reference: {source: phase_start}` (default) or +`{source: artifact, path: models/reference/actor.pt}`. A phase-start reference is +a deep copy before any phase update. References stay in evaluation mode with +gradients disabled. Full representations and actual weight hashes must match. +Embedded checkpoint reference weights survive removal of the original file; +a still-present artifact with changed weights is rejected on resume. -Policy trajectories use `sequifier.scalar-policy-trajectory`, version 1. -Forecast trajectories can be demonstrations with explicit -`action_semantics: executed_predictions` and a finite `action_values` vocabulary. -They retain the latest reset observation and start previous-action history with -the start token, because earlier forecast seed actions are unknown. +## Online adapters -## Python interfaces and scope +PPO and GRPO share the version-3 `OnlineEnvironmentAdapter` contract. Configure +`environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. +Factories must create fresh independent adapters for evaluation. An adapter +explicitly declares: ```python -from sequifier.config.train_config import load_train_config -from sequifier.train import run_training -from sequifier.rl_api import load_policy, PolicyWindowView - -config = load_train_config("configs/train-policy.yaml", {}, False) -run_training(config) -policy = load_policy(config.model_path, device="cpu") -# inputs = PolicyWindowView(policy.config).inputs( -# observations, previous_action_ids, desired_returns, static_inputs -# ).to(policy.device) -# distribution = policy.distribution(inputs) -``` - -Python callers use `SequifierInputs` for named model tensors and -`PolicyWindowView.inputs` for raw histories. `PolicySequenceDataset` accepts -`SequenceSourceConfig` entries or in-memory `SequenceEpisode` records; JSON logs -are adapted with `source_episodes`. The low-level `train_policy` takes training -and validation datasets plus a `PolicyTrainConfig` from -`sequifier.config.train_config`, sharing the same -options and checkpoint lifecycle as `run_policy_training`. - -`CategoricalPolicy` is an alignment adapter over the ordinary network, available -as `policy.network`. `policy.execution_config` is the canonical model config, -compatible with the shared artifact and export services. Distribution methods -include `mode`, `sample`, `log_prob`, and `entropy`. `collect_policy_rollout` -remains available for application-owned adapters. Generic analysis/loading tools -also accept policy model and run artifacts. - -The previous standalone numeric-policy artifacts (including versions 1 and 2) -use a different architecture and must be retrained. This workflow currently -supports one categorical actor phase, including the online PPO route below. Mixed supervised/policy plans, -distributed policy learning, custom run integrations, multimodal -trajectory storage, and ONNX policy execution through the CLI remain outside -this workflow. PyTorch policy execution supports CPU, CUDA, and MPS. - -## Offline IQL - -Use a single `iql` phase to improve an unconditioned categorical actor from a -fixed offline dataset. See the complete -[IQL configuration](example_inputs/configs/train-iql-from-table.yaml). The -actor can start from a compatible BC model with `initialize_from`. IQL creates -separate value, Q1, Q2, and target-Q networks; the portable model artifact -contains only the selected actor. The run checkpoint contains the full learner. -In this categorical slice, V and Q use independent copies of the configured -actor history encoder plus their own scalar heads; their architecture is not -authored in a separate model registry. - -Each selected decision needs a finite scalar reward, `terminated`, `truncated`, -and `bootstrap_allowed`. An ordinary nonterminal decision must bootstrap from -its genuine successor observation. An interior row may obtain that successor -from the following row; a final nonterminal or truncated row needs explicitly -mapped successor columns. A true terminal may omit its successor and cannot -bootstrap. Termination takes precedence when both boundary flags are true. -Unknown boundaries and missing rewards are errors for IQL even though BC can -use those demonstrations. Current observations and previous actions form the -actor/value history; Q alone receives the current recorded action. Splits, -categorical fitting, and numeric statistics come only from training episodes. - -`updates` is the number of optimizer cycles, each containing one value, critic, -and actor step. The phase controls uniform transition batch size, discount, -expectile, inverse temperature, maximum actor weight, target-Q update rate, -separate optimizer learning rates/specs and scheduler specs, gradient clipping, -and evaluation/checkpoint intervals. Schedulers advance after their own -optimizer update. The held-out action likelihood selects the portable actor; -the report also records value, Q, Bellman, advantage, and weight diagnostics. -The supported per-role optimizers are Adam, AdamW, SGD, and RMSprop. -`global_training.batch_size` and `global_training.learning_rate` remain required -by the shared policy config; the IQL phase's batch size and three learning rates -control its actual updates. -Use a separate application evaluation with `collect_policy_rollout` to compare -returns against BC in a local environment. -The [toy control example](example_inputs/iql_toy_control.py) generates a fixed -mixed-quality dataset for its [BC](example_inputs/configs/train-bc-toy.yaml) -and [IQL](example_inputs/configs/train-iql-toy.yaml) configs. Train BC first; -the IQL config warm-starts from its actor artifact. The example compares saved -actors over 100 fixed initial states and reports mean and standard deviation -rather than asserting an improvement for every seed. - -```console -python documentation/example_inputs/iql_toy_control.py generate data/offline-transitions.csv -sequifier train --config-path documentation/example_inputs/configs/train-bc-toy.yaml -sequifier train --config-path documentation/example_inputs/configs/train-iql-toy.yaml -python documentation/example_inputs/iql_toy_control.py evaluate models/toy-bc/toy-bc.pt models/toy-iql/toy-iql.pt +from sequifier.rl_api import EnvironmentCapabilities, OnlineResetResult, OnlineStepResult + +class Adapter: + protocol_version = 3 + implementation_id = "my-simulator.v1" + capabilities = EnvironmentCapabilities( + exact_snapshot=True, + group_start="scenario", + reproducible_group_start=True, + group_start_randomness="independent", + ) + + def reset(self, seed: int) -> OnlineResetResult: ... + def step(self, action: dict) -> OnlineStepResult: ... + def close(self) -> None: ... + def snapshot_state(self) -> dict: ... + def restore_state(self, state: dict) -> None: ... + def create_group_start(self, seed: int) -> object: ... + def reset_group(self, token: object, rollout_seed: int) -> OnlineResetResult: ... ``` -`global_training.resume: {policy: if_exists}` resumes the IQL learner from its -complete update checkpoint. `policy: required` needs an explicit -`checkpoint_path`. Exact resume validates the configuration, source content, -split, fitted vocabulary, and normalization. A BC model passed to -`initialize_from` is a warm start, not a learner checkpoint; the actor's policy -schema, normalization, model architecture, and context must match. IQL requires -float32 layers without autocast and supports scalar histories, one categorical -action, and local execution; a -return-conditioned actor, distributed training, mixed precision, continuous or -joint actions, and mixed phase plans are rejected. - -Python callers can use `run_iql_training(config)` from `sequifier.rl_api` with a -validated `PolicyTrainConfig`. The return value contains the selected -`CategoricalPolicy`, recorded update metrics, and training/validation episode IDs. +Reset returns the current observation, static inputs and units. Step returns +immediate scalar reward, actual successor observation, boundary flags, units +and optional boolean success. Only true termination may omit the successor. +`close()` runs in `finally`. `implementation_id` must match the authored config +and change whenever simulator/reset/RNG semantics change. + +GRPO requires `group_start: snapshot | scenario`, explicit methods, and an opaque +finite JSON-serializable token. Resetting a sample must reproduce the token's +initial condition while independently seeding future environment randomness. +`group_start_randomness` documents whether the token captures initial simulator +randomness or only the scenario. Group/sample indices and run/phase seeds derive +separate environment and action streams. Repeating a reset seed alone does not +establish this capability. Exact GRPO continuation additionally requires the +adapter's documented `reproducible_group_start` guarantee. The bundled +`CategoricalControlAdapter` demonstrates a deterministic scenario token. + +`SeededAdapterBridge` can adapt an existing seeded forecast-rollout adapter with +an explicit initial-state factory; it declares no group-start or exact-snapshot +capability. Forecast rollout bindings keep their separate version-1 contract. ## Online categorical PPO -PPO runs one persistent local environment with one categorical action. Start -with the [complete control config](example_inputs/configs/train-ppo-toy.yaml): +`ppo` requires `environment`, `evaluation`, `total_env_steps` and `rollout_length`. +Normalization is inherited from the run representation; a phase assertion is +optional. One persistent training environment collects compact sealed horizons. +True successors bootstrap fixed-length horizon/truncation boundaries; termination +disables bootstrap. FP32 GAE feeds the clipped policy objective, entropy bonus, +and separate value regression, with optional value clipping. Actor and value +optimizers share no parameters. Old collection scores are checked before update, +and collection generations are independent of phase/run indices. + +Defaults: `optimization_epochs: 4`, `minibatch_size: 64`, `accumulation_steps: 1`, +`gamma: 0.99`, `gae_lambda: 0.95`, `clip_epsilon: 0.2`, +`value_coefficient: 0.5`, `entropy_coefficient: 0.01`, +`normalize_advantages: true`, `reward_scale: 1`. Optional positive `target_kl` +stops optimization against the old collection policy; `value_clip` is optional. +`actor_learning_rate`/`value_learning_rate` default to 0.0003, with independent +optimizers/schedulers and `actor_gradient_clip`/`value_gradient_clip` of 1.0. +`allow_partial_rollout: true` executes exactly the remaining action budget; +false requires divisibility. `max_buffer_bytes` defaults to 536870912. +`archive_directory` optionally writes separate versioned collection records; +`archive_scores` defaults to false. Archived records are not training input. + +`evaluation_interval` and `checkpoint_interval` default to one complete +collection/optimization iteration. Evaluation uses a separate adapter, +independent seeds and a whole-evaluation POSIX/main-thread deadline. It requires +`environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; +`selection: deterministic | stochastic` defaults to deterministic. Default +`metric: mean_completed_return`, `direction: max` selects true completed returns. +`mean_completed_length` and `success_rate` are also supported. Censored episodes +and unavailable metrics are explicit; best selection fails when no finite +monitored value is available. The final iteration always evaluates. + +## GRPO groups and objective + +`grpo` requires `environment`, `evaluation`, `total_groups` and `max_steps`. +Defaults: `group_size: 4` (at least 2), `groups_per_update: 1`, +`optimization_epochs: 4`, `advantage_epsilon: 1e-6`, `clip_epsilon: 0.2`, +`kl_coefficient: 0.04`, `reward_scale: 1`, `scoring_chunk_size: 64`. +`old_policy_target_kl` optionally limits old-policy drift independently of the +reference penalty. No value network, value optimizer, or GAE is allocated. + +Each group contains independent sequential trajectories from the same opaque +start token under one frozen old actor generation. All groups in an iteration +are sealed before any update. Scores sum observed rewards times the fixed +reward scale, through termination, truncation or the explicit `max_steps` +finite-horizon cutoff. Cutoffs do not invent termination or bootstrapped rewards. +Incomplete/failed/non-finite groups are discarded and never optimized. Errors +report issued environment actions, including actions discarded with the group; +there is no retry. Optional `max_env_steps` is checked before collection using +the conservative `group_size * max_steps` bound for every requested group. +Memory is bounded by `max_buffer_bytes` (default 536870912), checked before +collection and as frames are appended. Optional archives are separate from the +training buffer. + +The precise first variant is: -```sh -sequifier train --config-path documentation/example_inputs/configs/train-ppo-toy.yaml +```text +A_i = (R_i - mean(R)) / (std_population(R) + advantage_epsilon) +r_it = exp(log pi_actor(a_it|h_it) - log pi_old(a_it|h_it)) +d_it = log pi_ref(a_it|h_it) - log pi_actor(a_it|h_it) +K_it = exp(d_it) - d_it - 1 +loss = mean_trajectories(mean_valid_decisions( + -min(r_it*A_i, clamp(r_it,1-epsilon,1+epsilon)*A_i) + kl_coefficient*K_it)) ``` -`sequifier make` also writes `configs/train-ppo.yaml`. The PPO root schema is -`PPOTrainConfig`: it has `model`, `context_length`, `seed`, `device`, and exactly -one `training_plan.phases` entry with `algorithm: ppo`. It has no -`dataset_training`, offline `sources`, `epochs`, or `global_training`. Phase -fields own both optimizers and their update clocks. See the -[PPO configuration reference](configs/train.md#online-ppo-configuration) and -[Python workflow](example_inputs/ppo_toy_control.py). - -The bundled `sequifier.rl.control:CategoricalControlAdapter` alternates a scalar -cue between -1 and +1. Choosing left for -1 and right for +1 earns +1; the other -action earns -1. Each episode executes eight actions. A random policy's expected -return is zero; the optimum is eight. The example config uses CPU, seed 1010, -context length 4, a 32-dimensional one-layer transformer, 256 decisions per -collection, and 8,192 total actions. The training report records hardware, -parameter counts, actual throughput and peak process memory. Learning quality -and multiple-seed improvement have not been measured for this implementation; -no training or test jobs were run for this increment. No CI job was added, -because the implementation request explicitly excluded writing or running tests. - -### Online adapter contract - -A factory receives `environment.config` as keyword arguments. It must return a -fresh adapter with `protocol_version = 2`, implementing: +Each trajectory has equal weight regardless of length; group sizes are fixed. +Minibatches contain whole trajectories; accumulation uses actual trajectory +counts. FP32 ratios/objectives/gradients are checked before applying an update. +Equal group scores give zero advantages; the reference penalty can still update +the actor. Reference and old-policy scores stay detached across optimization +epochs. The reported `sampled_reference_penalty` is a sampled estimator under +old-policy samples, not exact current-policy KL. + +Evaluation requires `metric: score_mean`, `direction: max` and scores every +independent evaluation episode through its real boundary or declared horizon. +The evaluation score uses the same fixed reward scale. Evaluation/checkpoint +intervals count completed updates, with an explicit final evaluation. Reports +include score spread, zero-variance groups, lengths, cutoff reasons, start +tokens, sampled penalty, clip fraction, generations, optimizer steps and issued +environment actions. Sealed buffers are consumed once; stale generations fail. + +## Checkpoints, resume, and artifacts + +The only policy-training checkpoint is `sequifier_policy_run`, format version 1. +It contains the resolved plan/fingerprint, immutable representation, data and +adapter identities, run RNG, current phase/index/status, completed results and +selected actors, algorithm/version and phase-owned state, and the last committed +transaction. DPO/GRPO reference weights are embedded. Optimizer, scheduler, +sampler, selection, evaluation and phase counters remain phase-local. + +Writes use a temporary file and atomic replacement at completed updates or +handoffs. No partial group is checkpointable. A committed handoff advances the +phase index exactly once; resume never reruns completed phases or reselects their +actor. Phase artifacts/reports can be republished idempotently from the stored +selection. Representation and data/adapter/reference mismatches fail before +output mutation. + +`resume.mode: exact` requires validated PPO simulator/RNG snapshots or GRPO's +reproducible group-start capability and matching implementation identity. PPO +checks the restored snapshot exactly; GRPO derives subsequent group tokens and +independent streams from persisted counters. `reset_environments` explicitly +resets PPO environments or restarts GRPO with a new derived stream, preserving +completed updates and reporting a trajectory discontinuity. Neither mode reuses +partial groups or old on-policy samples. Offline continuation restores its +sampler/order and optimizer transaction exactly. + +Portable artifacts are ordinary actor-only `sequifier_model` bundles. They +carry normalization, codecs, architecture, policy and unit provenance, with no +critics, references or trainer dependency. The final artifact comes from the +final phase's selected actor. The run report records order/config fingerprints, +source/reference provenance, selections, clocks and continuation modes. + +## Python and inference ```python -from sequifier.rl.environment import OnlineResetResult, OnlineStepResult - -class Environment: - protocol_version = 2 - supports_exact_snapshot = False - - def reset(self, seed: int) -> OnlineResetResult: - # Return initial domain values and episode-static inputs. - return OnlineResetResult({"observation": 0.0}) - - def step(self, action: dict) -> OnlineStepResult: - # Execute exactly once. Return the genuine successor, immediate finite - # scalar reward, and independent termination/truncation flags. - return OnlineStepResult({"observation": 1.0}, reward=1.0) +from sequifier.config.train_config import load_train_config +from sequifier.rl_api import PolicyRunConfig, run_policy_training, load_policy - def close(self): - pass +config = load_train_config("configs/train-policy.yaml", {}, False) +if not isinstance(config, PolicyRunConfig): + raise TypeError("expected a policy run") +result = run_policy_training(config) +actor = load_policy(result.model_path) +for phase in result.phases: + print(phase.name, phase.selected_metric, phase.clocks) ``` -The actor reads observations through the current decision and previous actions -only. After executing an action, its genuine successor and that action enter -the next causal window. An environment must never substitute an autoreset -observation for a final successor. A true terminal may return `observation=None` -and never bootstraps; truncation and collection cuts require a genuine successor -and bootstrap V from it. If both flags are true, termination wins. A horizon -cut preserves the live environment and its bounded causal history for the next -policy version. Only recorded episode boundaries trigger resets before the next -decision. Rewards must be final and immediate; delayed/provisional rewards and -reward providers are outside this contract. - -Observations and static fields must exactly match the frozen scalar schema. -Real values must fit finite FP32, categorical values must belong to declared -vocabularies, and units returned at every reset/step must equal the authored -`environment.units` mapping (empty for unitless values). Declare units for each -field whose units matter. Stochastic augmentation is unavailable. Dropout stays -disabled during collection and optimization, while scoring retains gradients. -The existing version-1 seeded inference adapter API is unchanged. -`SeededAdapterBridge` can wrap a v1 environment with same-named environment -observation bindings when an application supplies an initial-state factory. - -### Frozen representation and optimization - -The policy's observation/static schema, action order, class counts, label maps, -model architecture and context are fixed before the first reset. Author -`normalization: {source: identity}` deliberately, or use -`source: statistics` with a `statistics` map containing exactly the real fields' -`mean` and nonnegative `std`. Zero standard deviations from constant features -use the shared normalizer's epsilon, preserving BC/IQL warm-start statistics. -PPO never estimates statistics from changing online data. `initialize_from` -accepts an ordinary unconditioned BC/IQL actor only if -its policy schema, action ordering, normalization, model and context match -exactly. It initializes actor weights, not the learner state. A -return-conditioned actor is rejected. - -A separate history-only V owns independent parameters; PPO creates no Q or -target-Q networks. Each buffer stores a single policy generation, old action -log-probabilities, old values and true successor values. Compact frames retain -pre-buffer context; minibatches reconstruct ordered, complete windows. -FP32 GAE breaks its recurrence at episode boundaries and horizon cuts, while -bootstrapping at nonterminal boundaries. Advantage normalization occurs once per -iteration. Old scores, advantages and targets stay fixed for all epochs. - -The objective uses clipped probability ratios, independent squared value loss, -and current-policy categorical entropy. Optional value clipping is centered on -old V; it defaults off. Probability math is FP32. Gradient accumulation and the -last partial minibatch are weighted by their number of actual decisions. Actor -and V schedulers advance independently after their optimizer steps. Optional -KL limits stop further optimizer groups. Non-finite inputs, scores, losses, -gradients and parameters fail the run instead of admitting invalid decisions. - -### Evaluation, artifacts and resuming - -Evaluation uses a fresh adapter, frozen actor snapshot, fixed evaluation seeds, -and separate action/reset seed streams. It restores Python, NumPy and torch RNG -state afterward. Default evaluation selects mode actions; use -`selection: stochastic` explicitly for sampled actions. `max_episode_steps` -and `timeout_seconds` are mandatory. Evaluation runs on the main thread of a -local POSIX process, with an interval-timer deadline across creation, resets -and steps. Adapter methods must be interruptible and `close()` must return -promptly. A factory must return a different object from the live training slot. - -Reports include completed returns and their spread, episode lengths, completed -and censored counts, and success rates only when the adapter supplies success. -An environment truncation or evaluation step cap is censored; it does not count -as a completed episode. Best-actor selection defaults to maximizing -`mean_completed_return`; selection can instead use `mean_completed_length` or -`success_rate` with an explicit direction. No completed episodes means the -return metric is unavailable. If no evaluation supplies the selection metric, -the latest actor is exported with `latest_actor_metric_unavailable` recorded; -it is not described as a best actor. Initial evaluation also participates in -best-actor selection. - -The default outputs are `models//.pt`, -`outputs/training//report.json`, and `checkpoints//ppo.pt`. -The portable model contains actor weights only and loads with `load_policy`, -`sequifier infer`, and `collect_policy_rollout`. The example includes ordinary -[inference](example_inputs/configs/infer-ppo-toy.yaml) and seeded -[rollout](example_inputs/configs/rollout-ppo-toy.yaml) configs: - -```sh -sequifier infer --config-path documentation/example_inputs/configs/infer-ppo-toy.yaml -sequifier infer --config-path documentation/example_inputs/configs/rollout-ppo-toy.yaml -``` - The learner checkpoint contains -actor/V weights and specs, both optimizers/schedulers and step clocks, the -frozen codec, config fingerprint, best actor, seed/RNG state, live history and -episode counters. Checkpoints commit atomically only at complete iteration -boundaries. A crash during collection or optimization restarts from the last -committed iteration; partial buffers are not resumed. - -Set the phase's `resume.policy` to `never`, `if_exists`, or `required` and choose -one of these declared `resume.mode` values: - -- `exact` requires `supports_exact_snapshot = True` and working - `snapshot_state()` / `restore_state(state)` methods. The snapshot must include - simulator and simulator RNG state using dictionaries, lists, tuples, finite - scalars, numeric NumPy arrays and torch tensors. Restore is checked against a fresh snapshot. - Merely exposing `state_dict` does not declare this capability. Exact mode - rejects unsupported adapters before collection. It preserves the live - history and episode counters; equivalence still depends on deterministic - adapter and device operations. -- `reset_environments` restores learner, action sampler and optimizer state, - then resets the environment and causal history. It discards incomplete - episode metrics and records a non-trajectory-equivalent continuation. Reset - counts continue forward, so the next reset uses a new derived seed. - -Resume validates the full run contract, including budget, model, codec, reward -scale, adapter identity/config and normalization. Resume policy and mode may -change; other changes require a new run. Evaluation seeds never affect training -reset counts. Optional `archive_directory` writes atomic, version-1 -`sequifier.ppo-collection` records, with pre-fragment context, identities, -policy generation, cutoff reason, raw/scaled rewards and true successors. `archive_scores: true` -also retains old log-probability, entropy and values. The independent -`load_collection_record` reader does not change existing v1 JSON episode -readers. Archives are never fed back into online updates. - -The report records return/evaluation curves, policy/value losses, entropy, clip -fraction, approximate KL, explained variance, gradient norms, ratio tails, -optimizer clocks, counts and separate collection/update/evaluation timing. -Invalid data fails immediately; successful reports therefore have zero invalid -episodes. A nonterminated final training episode is explicitly censored. - -This scope is one in-process adapter slot and one fixed categorical action on -CPU or a supported local CUDA/MPS device. Vector/Gym/remote adapters, -continuous/joint actions, media/depth, distributed learning, reward providers, -mixed phases, same-run phase transfer, generalized monitors/search and ONNX -policy execution remain deferred. This does not complete the broader rollout -and reinforcement-learning plan. +`PolicyRunConfig.model_validate` accepts the same mapping directly. The single +`PolicyRunResult` exposes `policy`, ordered `phases`, `representation` and output +paths. `rl_api` also exports the phase schemas, preference records, online +capabilities, `PolicyWindowView`, and model loading/execution helpers. + +Use `sequifier infer` with the final artifact for categorical sequence inference +or rollout. `PolicyExecutor` and `collect_policy_rollout` build the same causal +histories; return-conditioned actors require an explicit desired return. +See [policy inference](configs/infer.md), [rollout](rollout.md), and the +[Python PPO workflow](example_inputs/ppo_toy_control.py). + +CPU determinism is the reference execution contract. CUDA and MPS reproducibility +is limited by the selected PyTorch backend/operators and matching device/RNG +topology. This implementation was statically checked; no tests or training runs +were written/executed for this change, so numerical and end-to-end acceptance +verification remains outstanding. diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index dad71d42..d45e07e0 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -1,83 +1,33 @@ -# Integrated policy training and inference (unreleased) - -Offline IQL now accepts one fixed-dataset `iql` phase for an unconditioned, -single-categorical actor. It adds strict transition/successor validation, -independent value and twin-Q networks, target-Q updates, weighted actor -extraction, a complete learner checkpoint, and actor-only model export. -Continuous/joint actions, distributed IQL, and mixed plans remain future -work. No tests were written or run for this increment, as requested. - -For imitation, `sequifier train` accepts one trajectory-backed `policy_imitation` phase; -`return_conditioned: false` selects behavior cloning. Policies now use the -standard configurable ingestion, backbone, decoder, and network implementation. -Encoded categorical observations and matching forecast-backbone transfer are -supported. PyTorch execution supports CPU, CUDA, and MPS. - -The selected best model is an ordinary `sequifier_model` artifact, usable with -`sequifier infer`, Python policy APIs, and generic model-loading tools. Inference -supports raw CSV/Parquet decisions and environment execution through the shared -rollout coordinator. Each training epoch writes a standard resumable run -checkpoint, including optimizer/scheduler/scaler, random/data-order state, -training progress, and best weights. The older standalone actor format requires -retraining. The default model path is now `models//.pt`. - -File-backed training uses a bounded episode cache, incremental normalization, -and context-sized window construction. Behavior cloning retains observed -state/action pairs from truncated and cutoff episodes without return labels. -Grouped episode splitting and training-only normalization remain in place. - -`sequifier make` includes policy train, IQL train, PPO train, and infer templates. Mixed algorithm -phases, distributed policy learning, multimodal trajectories, -and CLI ONNX policy execution remain future work. - -Policy sequence representation follow-up: - -- Train BC directly from CSV/Parquet observation/action tables. Rewards, - successors, bindings and seed histories are not required for BC. -- JSON recordings and existing scalar prepared histories adapt into the same - columnar episode representation, with optional transition capabilities. -- Policy windows use explicit same-position labels and causal attention, named - tensors, standard sequence batches and the supervised loss service. -- Categorical maps and sample statistics use shared preprocessing functions and - fit only training episodes. New action vocabularies use global/decoder mappings; - existing policy artifacts retain their legacy layout. -- Policy configuration uses shared model/dataset/phase authoring. Redundant - compact registries and dense input APIs have been removed. Preprocess can - optionally commit raw Parquet sequence shards - with a checksummed manifest. -- Repository tests were moved to `../sequifier-test/tests/sequifier-refactor-unit`; - no tests were added. Verification uses lint, compilation and workflow smoke runs. - -Architecture consolidation: - -- Policy runs use the ordinary TrainingRun, TrainingEngine, evaluation, run state, - checkpoint restore and loader-state services. The private policy loop is gone. -- Model components and global optimization settings use their shared schemas; - policy records retain observation/action alignment rather than copying models. -- CSV inference and training share table-to-episode validation and causal windows. - Tabular prepared windows share the stored-tensor reader with supervised data. -- Sequence import writes columnar episodes directly, retaining optional transitions. -- Earlier private policy run checkpoints require a new run initialized from their - model weights. Portable models and execution weights remain readable. - - -Scoped online PPO: - -- A single `ppo` phase trains an unconditioned categorical actor against one - persistent local version-2 adapter, without datasets or supervised budgets. -- Shared bounded causal histories, compact sealed buffers, true-successor - bootstrap, FP32 GAE and ratios, clipped policy/value objectives, entropy, - sample-weighted gradient accumulation and independent actor/V optimizers. -- Frozen codecs and explicit normalization; strict BC/IQL actor warm starts. -- Separate seeded, bounded rollout evaluation selects actors by completed return - or another declared environment metric, with censored metrics kept explicit. -- Atomic version-1 PPO learner checkpoints support declared exact simulator/RNG - snapshots or non-trajectory-equivalent environment resets. Actor export stays - an ordinary portable `sequifier_model` and includes no learner networks. -- Structured reports, optional independent collection archives, a deterministic - categorical control adapter, CLI template and Python workflow are included. -- No tests were written or run. No training/benchmark runs or learning claims - accompany this implementation; only static validation was performed. -- Vector/remote/Gym adapters, continuous/joint actions, multimodal/distributed - PPO, mixed phases, reward providers, generalized monitors/search and ONNX - policy execution remain deferred. The parent RL plan remains incomplete. +# Multi-phase categorical policy training (unreleased) + +- One `PolicyRunConfig`, CLI dispatch and Python `run_policy_training` entry point + support ordered imitation, IQL, PPO, DPO and GRPO phases with natural budgets. +- A frozen architecture/interface/codec/normalization/context/units contract spans + phases. Selected actors are copied at handoff; optimizer/critic state is local. + Best/last selection is explicit, ties retain the earliest best, and final export + uses the final phase's selection. +- DPO adds versioned reward-free preference spans, connected-component-safe + partitions, branch-history validation, summed causal action likelihoods and + canonical sigmoid DPO. Chunked backward scoring bounds activation memory. +- GRPO adds explicit reproducible group-start capabilities, independent RNG + streams, bounded complete groups, population-normalized advantages, clipped + ratios and a separately named sampled frozen-reference penalty, without a critic. +- Version-3 local online adapters serve PPO and GRPO. Exact continuation requires + validated snapshots or documented reproducible group starts; non-exact restarts + report a discontinuity. Failed/partial groups are never reused. +- One atomic `sequifier_policy_run` checkpoint embeds phase references, selected + actors, source/representation hashes, sampler/RNG state and committed progress. + The former unreleased single-algorithm configuration/checkpoint paths are removed. +- Actor-only portable artifacts remain usable for inference, rollout and cross-run + initialization. Default output is `models//actor.pt`; phase reports and + selected actors use indexed directories. +- Templates, examples, public exports and guides use the canonical schema. The + pre-existing supervised training runtime remains independent. +- No tests or training jobs were written/run for this change. Static compilation, + lint and type analysis are the verification scope; numerical learning and exact + continuation acceptance evidence remains to be established. + +Scope: one categorical action, scalar/categorical inputs, causal fixed histories, +local single-process FP32. Return conditioning supports imitation only. Text/LLM, +continuous/joint actions, images/depth, vector/distributed execution, reward models, +optimizer/critic inheritance and mid-group checkpoints remain deferred. diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index bd23595b..dac8ddf8 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -95,8 +95,6 @@ def resolved_config_from_model_config( } ) authored_interfaces[name] = ModelInterfaceSpecModel.model_validate(authored) - if policy_spec is not None: - policy_spec = {"categorical_encoding": "legacy", **policy_spec} target_decoder_ids = interface.get("target_decoder_ids", {}) target_n_classes = interface.get( "target_n_classes", diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 554ff562..745daf66 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -82,11 +82,6 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": raise ValueError("Unsupported checkpoint: expected new Sequifier format.") from sequifier.config.train_config import ResolvedSequifierConfig - if "trajectory_sources" in payload["training_config"]: - raise ValueError( - "Private policy run state requires a new run using initialize_from" - ) - if ( payload.get("initialization_policy_version") != 1 or payload.get("initialization_seed_derivation_version") != 1 diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 10235351..2e90d33e 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -440,9 +440,9 @@ def load_inferer_config( from sequifier.config.train_config import load_train_config training = load_train_config(training_config_path, {}, False) - from sequifier.config.train_config import PolicyTrainConfig, PPOTrainConfig + from sequifier.config.train_config import PolicyRunConfig - if isinstance(training, (PolicyTrainConfig, PPOTrainConfig)): + if isinstance(training, PolicyRunConfig): raise ValueError( "policy inference requires model_path pointing to its portable artifact" ) diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 0fa14158..62187779 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -35,7 +35,7 @@ class PolicySchemaConfig(PolicyConfigBase): ) # The same label -> global ID convention as preprocessing metadata. id_maps: dict[str, dict[str | int, int]] = Field(default_factory=dict) - categorical_encoding: Literal["global", "legacy"] = "global" + categorical_encoding: Literal["global"] = "global" action_column: str = Field(default="action", min_length=1) action_values: list[DomainValue] = Field( default_factory=lambda: [0.0, 1.0], min_length=2 @@ -95,7 +95,5 @@ def categorical_n_classes(self) -> dict[str, int]: class CategoricalPolicyConfig(PolicySchemaConfig): """Resolved alignment metadata; architecture belongs to the ordinary model.""" - # Older artifacts embedded model components here. The model bundle owns them. - model_config = ConfigDict(extra="ignore", frozen=True) context_length: int = Field(gt=0) interface_name: NonemptyStr diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py new file mode 100644 index 00000000..76d27c85 --- /dev/null +++ b/src/sequifier/config/policy_plan.py @@ -0,0 +1,341 @@ +"""Built-in policy phase schemas; budgets and objectives are algorithm-specific.""" + +import math +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from sequifier.config.components import ComponentSpec +from sequifier.config.identifiers import identifier +from sequifier.config.policy_config import CategoricalPolicyConfig + + +class PhaseConfig(BaseModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + name: str + handoff: Literal["best", "last"] = "best" + representation: CategoricalPolicyConfig | None = None + + def optimizer_step_budget(self, training_size=None): + raise NotImplementedError("built-in phase must declare its optimizer budget") + + @field_validator("name") + @classmethod + def phase_name(cls, value): + return identifier(value, "Policy phase name") + + +class ActorOptimizationConfig(PhaseConfig): + batch_size: int = Field(default=32, gt=0) + accumulation_steps: int = Field(default=1, gt=0) + actor_learning_rate: float = Field(default=3e-4, gt=0) + actor_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + actor_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + gradient_clip: float = Field(default=1.0, gt=0) + evaluation_interval: int = Field(default=1, gt=0) + checkpoint_interval: int = Field(default=1, gt=0) + + +class OfflinePhaseConfig(PhaseConfig): + sources: list[str] = Field(min_length=1) + validation_sources: list[str] | None = Field(default=None, min_length=1) + + +class PolicyImitationPhaseConfig(ActorOptimizationConfig, OfflinePhaseConfig): + algorithm: Literal["policy_imitation"] + epochs: int = Field(gt=0) + monitor: Literal["validation_action_nll"] = "validation_action_nll" + direction: Literal["min"] = "min" + + def optimizer_step_budget(self, training_size=None): + if training_size is None: + raise ValueError("imitation optimizer budget requires training decisions") + size = self.batch_size * self.accumulation_steps + return self.epochs * ((training_size + size - 1) // size) + + +class IQLPhaseConfig(OfflinePhaseConfig): + model_config = ConfigDict(extra="forbid") + monitor: Literal["validation_action_nll"] = "validation_action_nll" + direction: Literal["min"] = "min" + algorithm: Literal["iql"] + name: str = Field(default="offline_iql", min_length=1) + updates: int = Field(gt=0) + batch_size: int = Field(default=64, gt=0) + gamma: float = Field(default=0.99, gt=0, le=1, allow_inf_nan=False) + expectile: float = Field(default=0.7, gt=0, lt=1, allow_inf_nan=False) + inverse_temperature: float = Field(default=3.0, gt=0, allow_inf_nan=False) + max_weight: float = Field(default=100.0, ge=1, allow_inf_nan=False) + target_update_rate: float = Field(default=0.005, gt=0, le=1, allow_inf_nan=False) + actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + critic_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + actor_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + value_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + critic_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + actor_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + value_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + critic_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) + evaluation_interval: int = Field(default=100, gt=0) + checkpoint_interval: int = Field(default=100, gt=0) + + def optimizer_step_budget(self, training_size=None): + return self.updates + + +class OnlineEnvironmentConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + factory: str + config: dict = Field(default_factory=dict) + protocol_version: Literal[3] = 3 + implementation_id: str = Field(min_length=1) + # Empty means unitless; adapters must report this exact declaration. + units: dict[str, str] = Field(default_factory=dict) + + @field_validator("factory") + @classmethod + def validate_factory(cls, value): + module, separator, attribute = value.partition(":") + if not separator or not module or not attribute.isidentifier(): + raise ValueError("environment.factory must use module:factory") + return value + + +class FrozenNormalizationConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + source: Literal["identity", "statistics"] + statistics: dict[str, dict[str, float]] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_statistics(self): + if self.source == "identity" and self.statistics: + raise ValueError("identity normalization cannot include statistics") + for column, values in self.statistics.items(): + if ( + set(values) != {"mean", "std"} + or not all(math.isfinite(value) for value in values.values()) + or values["std"] < 0 + ): + raise ValueError( + f"{column}: statistics require finite mean and nonnegative std" + ) + return self + + +class EvaluationEpisodesConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + environment: OnlineEnvironmentConfig + seeds: list[int] = Field(min_length=1) + max_episode_steps: int = Field(gt=0) + timeout_seconds: float = Field(gt=0, allow_inf_nan=False) + selection: Literal["deterministic", "stochastic"] = "deterministic" + + @field_validator("seeds") + @classmethod + def validate_seeds(cls, values): + if any(value < 0 or value >= 2**63 for value in values): + raise ValueError("evaluation seeds must be in [0, 2**63)") + return values + + +class OnlineEvaluationConfig(EvaluationEpisodesConfig): + metric: Literal[ + "mean_completed_return", "success_rate", "mean_completed_length", "score_mean" + ] = "mean_completed_return" + direction: Literal["min", "max"] = "max" + + +class PolicyResumeConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + policy: Literal["never", "if_exists", "required"] = "never" + mode: Literal["exact", "reset_environments"] = "reset_environments" + + +class PPOPhaseConfig(PhaseConfig): + model_config = ConfigDict(extra="forbid") + algorithm: Literal["ppo"] + name: str = Field(default="online_ppo", min_length=1) + environment: OnlineEnvironmentConfig + evaluation: OnlineEvaluationConfig + normalization: FrozenNormalizationConfig | None = None + seed_policy: Literal["derived_per_reset"] = "derived_per_reset" + total_env_steps: int = Field(gt=0) + rollout_length: int = Field(gt=0) + allow_partial_rollout: bool = True + optimization_epochs: int = Field(default=4, gt=0) + minibatch_size: int = Field(default=64, gt=0) + accumulation_steps: int = Field(default=1, gt=0) + gamma: float = Field(default=0.99, ge=0, le=1, allow_inf_nan=False) + gae_lambda: float = Field(default=0.95, ge=0, le=1, allow_inf_nan=False) + clip_epsilon: float = Field(default=0.2, gt=0, lt=1, allow_inf_nan=False) + value_clip: float | None = Field(default=None, gt=0, allow_inf_nan=False) + value_coefficient: float = Field(default=0.5, gt=0, allow_inf_nan=False) + entropy_coefficient: float = Field(default=0.01, ge=0, allow_inf_nan=False) + target_kl: float | None = Field(default=None, gt=0, allow_inf_nan=False) + normalize_advantages: bool = True + reward_scale: float = Field(default=1.0, gt=0, allow_inf_nan=False) + actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) + actor_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + value_optimizer: ComponentSpec = Field( + default_factory=lambda: ComponentSpec(name="Adam") + ) + actor_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + value_scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} + ) + ) + actor_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) + value_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) + evaluation_interval: int = Field(default=1, gt=0) + checkpoint_interval: int = Field(default=1, gt=0) + archive_directory: str | None = Field(default=None, min_length=1) + archive_scores: bool = False + max_buffer_bytes: int = Field(default=536870912, gt=0) + + def optimizer_step_budget(self, training_size=None): + full, remainder = divmod(self.total_env_steps, self.rollout_length) + size = self.minibatch_size * self.accumulation_steps + return self.optimization_epochs * ( + full * ((self.rollout_length + size - 1) // size) + + ((remainder + size - 1) // size if remainder else 0) + ) + + @model_validator(mode="after") + def validate_plans(self): + if ( + not self.allow_partial_rollout + and self.total_env_steps % self.rollout_length + ): + raise ValueError( + "rollout_length must divide total_env_steps unless allow_partial_rollout" + ) + for optimizer in (self.actor_optimizer, self.value_optimizer): + if optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"}: + raise ValueError("PPO optimizers must be Adam, AdamW, SGD, or RMSprop") + if "lr" in optimizer.arguments or "params" in optimizer.arguments: + raise ValueError( + "PPO learning rates and parameter ownership belong to the phase" + ) + max_steps = self.optimizer_step_budget() + for scheduler in (self.actor_scheduler, self.value_scheduler): + if ( + scheduler.name == "OneCycleLR" + and scheduler.arguments.get("total_steps") != max_steps + ): + raise ValueError( + "PPO OneCycleLR.total_steps must equal the maximum authored optimizer-step budget" + ) + if ( + scheduler.name + in {"ReduceLROnPlateau", "ChainedScheduler", "SequentialLR"} + or "optimizer" in scheduler.arguments + ): + raise ValueError( + "PPO schedulers must advance independently on optimizer steps" + ) + return self + + +class ReferenceConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + source: Literal["phase_start", "artifact"] = "phase_start" + path: str | None = None + + @model_validator(mode="after") + def location(self): + if (self.source == "artifact") != (self.path is not None): + raise ValueError("reference.path is required only for source: artifact") + return self + + +class DPOPhaseConfig(ActorOptimizationConfig): + algorithm: Literal["dpo"] + preferences: str + validation_preferences: str | None = None + updates: int = Field(gt=0) + beta: float = Field(default=0.1, gt=0) + scoring_chunk_size: int = Field(default=64, gt=0) + reference: ReferenceConfig = Field(default_factory=ReferenceConfig) + monitor: Literal["validation_dpo_loss"] = "validation_dpo_loss" + direction: Literal["min"] = "min" + + def optimizer_step_budget(self, training_size=None): + return self.updates + + +class GRPOEvaluationConfig(EvaluationEpisodesConfig): + metric: Literal["score_mean"] = "score_mean" + direction: Literal["max"] = "max" + + +class GRPOPhaseConfig(ActorOptimizationConfig): + scoring_chunk_size: int = Field(default=64, gt=0) + algorithm: Literal["grpo"] + environment: OnlineEnvironmentConfig + evaluation: GRPOEvaluationConfig + normalization: FrozenNormalizationConfig | None = None + reference: ReferenceConfig = Field(default_factory=ReferenceConfig) + total_groups: int = Field(gt=0) + groups_per_update: int = Field(default=1, gt=0) + group_size: int = Field(default=4, ge=2) + max_steps: int = Field(gt=0) + max_env_steps: int | None = Field(default=None, gt=0) + max_buffer_bytes: int = Field(default=536870912, gt=0) + optimization_epochs: int = Field(default=4, gt=0) + clip_epsilon: float = Field(default=0.2, gt=0, lt=1) + advantage_epsilon: float = Field(default=1e-6, gt=0) + kl_coefficient: float = Field(default=0.04, ge=0) + old_policy_target_kl: float | None = Field(default=None, gt=0) + reward_scale: float = Field(default=1.0, gt=0) + archive_directory: str | None = None + + def optimizer_step_budget(self, training_size=None): + full, remainder = divmod(self.total_groups, self.groups_per_update) + size = self.batch_size * self.accumulation_steps + return self.optimization_epochs * ( + full * ((self.groups_per_update * self.group_size + size - 1) // size) + + ((remainder * self.group_size + size - 1) // size if remainder else 0) + ) + + +PolicyPhase = Annotated[ + PolicyImitationPhaseConfig + | IQLPhaseConfig + | PPOPhaseConfig + | DPOPhaseConfig + | GRPOPhaseConfig, + Field(discriminator="algorithm"), +] diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 8c573c2a..07c5fd5e 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -27,6 +27,7 @@ class PolicyTableColumns(PolicyConfigBase): class SequenceSourceConfig(PolicyConfigBase): + units: dict[str, str] | None = None paths: list[NonemptyStr] = Field(default_factory=list) manifest: NonemptyStr | None = None format: Literal["auto", "trajectory", "table", "prepared"] = "auto" @@ -100,8 +101,3 @@ def resolve_sequence_source( split_index=None, ) return SequenceSourceConfig.model_validate(values) - - -class PolicyDatasetConfig(PolicyConfigBase): - model_interface: NonemptyStr - parts: dict[str, SequenceSourceConfig] = Field(min_length=1) diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 0279490f..e228d6d3 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -10,7 +10,6 @@ import copy import inspect -import json import math import os import warnings @@ -65,11 +64,17 @@ NonemptyStr, PolicySchemaConfig, ) +from sequifier.config.policy_plan import ( + FrozenNormalizationConfig, + GRPOPhaseConfig, + PolicyPhase, + PolicyResumeConfig, + PPOPhaseConfig, +) from sequifier.config.policy_train_config import ( - PolicyDatasetConfig, + SequenceSourceConfig, resolve_sequence_source, ) -from sequifier.config.training_plan import PolicyTrainingPlanModel, PPOTrainingPlanModel from sequifier.config.training_plan import ( SupervisedTrainingPlanModel as TrainingPlanModel, ) @@ -741,113 +746,74 @@ def validate_interface_names(cls, value): return value -class PolicyTrainConfig(BaseModel): - """One policy phase using the shared model and global training schemas.""" +class PolicyRunConfig(BaseModel): + """Canonical local policy run, shared by the CLI and Python API.""" - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) project_root: str = "." model_name: NonemptyStr = Field(pattern=r"^[^/\\]+$") device: str = "cpu" - seed: int = Field(default=1010, ge=0) - initialize_from: str | None = None + seed: int = Field(default=1010, ge=0, lt=2**32) + enforce_determinism: bool = True model: ModelSpecModel - dataset_training: dict[str, PolicyDatasetConfig] = Field(min_length=1) - global_training: GlobalTrainingSpecModel - training_plan: PolicyTrainingPlanModel - model_path: NonemptyStr - report_path: NonemptyStr - - @model_validator(mode="before") - @classmethod - def defaults(cls, values): - values = dict(values) - model = ModelSpecModel.model_validate(values.get("model")) - values["model"] = model - training = values.get("global_training", {}) - if isinstance(training, GlobalTrainingSpecModel): - training = training.model_dump() - values["global_training"] = { - "training_objective": "policy", - "target_offset": 0, - "context_length": model.backbone.architecture.max_context_length, - "batch_size": 64, - "inference_batch_size": 1, - "learning_rate": 0.005, - "gradient_clip": 1.0, - "accumulation_steps": 1, - "torch_compile": "none", - "reset_optimization_on_phase": False, - "calculate_validation_loss_on_initialization": False, - "scheduler": {"name": "ConstantLR", "factor": 1.0, "total_iters": 1}, - **training, - } - root = Path(values.get("project_root", ".")).resolve() - values["project_root"] = str(root) - name = values.get("model_name", "") - for key, default in ( - ("model_path", f"models/{name}/{name}.pt"), - ("report_path", f"outputs/training/{name}/report.json"), - ): - path = values.get(key) - if path == "": - raise ValueError("output paths must be nonempty") - values[key] = str((root / (path or default)).resolve()) - if values.get("initialize_from") is not None: - values["initialize_from"] = str( - (root / values["initialize_from"]).resolve() - ) - return values - - @property - def sources(self): - return { - f"{name}.{part}": source - for name, dataset in self.dataset_training.items() - for part, source in dataset.parts.items() - } - - @property - def policy(self) -> CategoricalPolicyConfig: - name = self.training_plan.phases[0].policy - schema = self.model.interfaces[name].policy - if schema is None: - raise ValueError("selected interface requires a policy schema") - return CategoricalPolicyConfig( - **schema.model_dump(), - interface_name=name, - context_length=self.global_training.context_length, - ) + policy_interface: str + context_length: int = Field(gt=0) + units: dict[str, str] = Field(default_factory=dict) + initialize_from: str | None = None + normalization: FrozenNormalizationConfig | None = None + sources: dict[str, SequenceSourceConfig] = Field(default_factory=dict) + preferences: dict[str, str] = Field(default_factory=dict) + phases: list[PolicyPhase] = Field(min_length=1) + validation_fraction: float = Field(default=0.2, gt=0, lt=1) + episode_cache_size: int = Field(default=8, gt=0) + resume: PolicyResumeConfig = Field(default_factory=PolicyResumeConfig) + model_path: str = "" + report_path: str = "" + checkpoint_path: str = "" + phase_directory: str = "" @model_validator(mode="after") def resolve(self): + root = Path(self.project_root).resolve() + self.project_root = str(root) if self.model_name in {".", ".."}: - raise ValueError("model_name must name a single directory") - if len(self.training_plan.phases) != 1: - raise ValueError("policy training requires one phase") - phase = self.training_plan.phases[0] - if phase.policy not in self.model.interfaces: - raise ValueError(f"unknown policy interface {phase.policy!r}") - if phase.algorithm == "iql": - selected_policy = self.model.interfaces[phase.policy].policy - if selected_policy is not None and selected_policy.return_conditioned: - raise ValueError("IQL requires an unconditioned categorical actor") - if self.global_training.data_parallelism is not None: - raise ValueError("IQL does not yet support distributed execution") - policy = self.policy - interface = self.model.interfaces[phase.policy] - if ( - interface.input_columns - and interface.input_columns - != policy.observation_columns + policy.static_columns + raise ValueError("model_name must name one directory") + for field, default in ( + ("model_path", f"models/{self.model_name}/actor.pt"), + ("report_path", f"outputs/training/{self.model_name}/report.json"), + ("checkpoint_path", f"checkpoints/{self.model_name}/run.pt"), + ("phase_directory", f"outputs/training/{self.model_name}/phases"), ): + setattr( + self, field, str((root / (getattr(self, field) or default)).resolve()) + ) + if self.initialize_from: + self.initialize_from = str((root / self.initialize_from).resolve()) + self.sources = { + name: resolve_sequence_source(source, root) + for name, source in self.sources.items() + } + self.preferences = { + name: str((root / path).resolve()) + for name, path in self.preferences.items() + } + if set(self.model.interfaces) != {self.policy_interface}: raise ValueError( - "policy inputs must list observations and static columns in order" + "policy runs require exactly the selected policy_interface" ) - if interface.target_columns != [policy.action_column]: - raise ValueError("policy targets must name its action column") + spec = self.policy + if set(self.units) - set(spec.observation_columns + spec.static_columns): + raise ValueError("units must name policy observation/static fields") + interface = self.model.interfaces[self.policy_interface] + if interface.input_columns != spec.observation_columns + spec.static_columns: + raise ValueError( + "model.interfaces inputs must match policy observations and static fields in order" + ) + if interface.target_columns != [spec.action_column]: + raise ValueError("model.interfaces targets must contain the policy action") if interface.decoder.support != 1 or interface.decoder.prediction_length != 1: raise ValueError( - "policy decoders require support=1 and prediction_length=1" + "policy decoder requires support=1 and prediction_length=1" ) if any( ( @@ -856,215 +822,88 @@ def resolve(self): interface.preconditioning_image_encoder, interface.feature_layout, ) - ): - raise ValueError("policy learning supports scalar inputs") - if ( - self.global_training.distributed - or self.global_training.training_objective != "policy" - or self.global_training.target_offset != 0 - or self.global_training.window_stride not in {None, 1} ): raise ValueError( - "policy training requires local policy alignment, target_offset=0 and stride=1" + "policy runs support scalar observation/static fields only" ) - if ( - self.global_training.context_length - > self.model.backbone.architecture.max_context_length - ): - raise ValueError("policy context exceeds backbone max_context_length") - if self.model_path == self.report_path: - raise ValueError("model_path and report_path must differ") if self.model.backbone.repository is not None: - raise ValueError("policy runs use initialize_from for backbone transfer") - names = phase.trajectory_sources + (phase.validation_sources or []) - if len(names) != len(set(names)) or set(names) - self.sources.keys(): + raise ValueError("policy initialization uses initialize_from") + if self.context_length > self.model.backbone.architecture.max_context_length: raise ValueError( - "phase sources must be unique, disjoint dataset.part references" + "context_length exceeds model.backbone.architecture.max_context_length" ) - for name in names: - dataset = self.dataset_training[name.partition(".")[0]] - if dataset.model_interface != phase.policy: - raise ValueError("dataset model_interface disagrees with phase policy") - root = Path(self.project_root) - self.dataset_training = { - name: dataset.model_copy( - update={ - "parts": { - part: resolve_sequence_source(source, root) - for part, source in dataset.parts.items() - } - } + if torch.device(self.device).type not in {"cpu", "cuda", "mps"}: + raise ValueError("policy runs require a local CPU, CUDA or MPS device") + for name, source in self.sources.items(): + if source.units is not None and source.units != self.units: + raise ValueError(f"sources.{name}.units differs from run units") + names = [phase.name for phase in self.phases] + if len(names) != len(set(names)): + raise ValueError("policy phase names must be unique") + for phase in self.phases: + if phase.algorithm != "policy_imitation" and spec.return_conditioned: + raise ValueError( + f"phases.{phase.name}: {phase.algorithm} requires an unconditioned actor" + ) + refs = getattr(phase, "sources", []) + ( + getattr(phase, "validation_sources", None) or [] ) - for name, dataset in self.dataset_training.items() - } + if len(refs) != len(set(refs)) or set(refs) - self.sources.keys(): + raise ValueError( + f"phases.{phase.name}.sources: unknown, repeated or overlapping source names" + ) + if phase.algorithm == "dpo": + for name in [phase.preferences, phase.validation_preferences]: + if name is not None and name not in self.preferences: + raise ValueError( + f"phases.{phase.name}.preferences: unknown manifest {name!r}" + ) + reference = getattr(phase, "reference", None) + if reference is not None and reference.path is not None: + reference.path = str((root / reference.path).resolve()) + if isinstance(phase, (PPOPhaseConfig, GRPOPhaseConfig)): + for environment in (phase.environment, phase.evaluation.environment): + if environment.units != self.units: + raise ValueError( + f"phases.{phase.name}.environment.units differs from run units" + ) + archive = getattr(phase, "archive_directory", None) + if ( + isinstance(phase, (PPOPhaseConfig, GRPOPhaseConfig)) + and archive is not None + ): + phase.archive_directory = str((root / archive).resolve()) + for role in ("actor", "value", "critic"): + optimizer = getattr(phase, f"{role}_optimizer", None) + scheduler = getattr(phase, f"{role}_scheduler", None) + if optimizer is not None and ( + optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"} + or {"lr", "params"}.intersection(optimizer.arguments) + ): + raise ValueError( + f"phases.{phase.name}.{role}_optimizer: use a supported optimizer and phase learning rate" + ) + if scheduler is not None and ( + scheduler.name + in {"ReduceLROnPlateau", "SequentialLR", "ChainedScheduler"} + or "optimizer" in scheduler.arguments + ): + raise ValueError( + f"phases.{phase.name}.{role}_scheduler must advance on optimizer steps" + ) return self - -class PPOTrainConfig(BaseModel): - """Online learning has no dataset loaders or supervised optimization fields.""" - - model_config = ConfigDict(extra="forbid") - project_root: str = "." - model_name: NonemptyStr = Field(pattern=r"^[^/\\]+$") - device: str = "cpu" - seed: int = Field(default=1010, ge=0, lt=2**32) - enforce_determinism: bool = True - context_length: int = Field(gt=0) - initialize_from: str | None = None - model: ModelSpecModel - training_plan: PPOTrainingPlanModel - model_path: NonemptyStr - report_path: NonemptyStr - checkpoint_path: NonemptyStr - - @model_validator(mode="before") - @classmethod - def defaults(cls, values): - values = dict(values) - model = ModelSpecModel.model_validate(values.get("model")) - values["model"] = model - values.setdefault( - "context_length", model.backbone.architecture.max_context_length - ) - root = Path(values.get("project_root", ".")).resolve() - values["project_root"] = str(root) - name = values.get("model_name", "") - for key, default in ( - ("model_path", f"models/{name}/{name}.pt"), - ("report_path", f"outputs/training/{name}/report.json"), - ("checkpoint_path", f"checkpoints/{name}/ppo.pt"), - ): - if values.get(key) == "": - raise ValueError("PPO output paths must be nonempty") - values[key] = str((root / (values.get(key) or default)).resolve()) - if values.get("initialize_from") is not None: - values["initialize_from"] = str( - (root / values["initialize_from"]).resolve() - ) - return values - @property def policy(self) -> CategoricalPolicyConfig: - name = self.training_plan.phases[0].policy - schema = self.model.interfaces[name].policy + schema = self.model.interfaces[self.policy_interface].policy if schema is None: - raise ValueError("PPO requires a categorical policy schema") + raise ValueError("policy_interface requires a categorical policy schema") return CategoricalPolicyConfig( **schema.model_dump(), - interface_name=name, + interface_name=self.policy_interface, context_length=self.context_length, ) - @property - def normalization_statistics(self): - declaration = self.training_plan.phases[0].normalization - real = ( - set(self.policy.observation_columns + self.policy.static_columns) - - self.policy.categorical_n_classes.keys() - ) - if declaration.source == "identity": - return {column: {"mean": 0.0, "std": 1.0} for column in sorted(real)} - if set(declaration.statistics) != real: - raise ValueError("PPO statistics must cover exactly the real policy fields") - return declaration.statistics - - @model_validator(mode="after") - def resolve(self): - if self.model_name in {".", ".."} or len(self.training_plan.phases) != 1: - raise ValueError("PPO requires a model name and exactly one online phase") - phase = self.training_plan.phases[0] - if set(self.model.interfaces) != {phase.policy}: - raise ValueError("PPO requires exactly its selected actor interface") - policy = self.policy - interface = self.model.interfaces[phase.policy] - if policy.return_conditioned: - raise ValueError( - "PPO cannot drop the desired-return contract; use an unconditioned actor" - ) - if set(policy.categorical_columns) - policy.categorical_n_classes.keys(): - raise ValueError( - "PPO categorical fields require explicit frozen class counts" - ) - if ( - interface.input_columns - != policy.observation_columns + policy.static_columns - or interface.target_columns != [policy.action_column] - ): - raise ValueError( - "PPO interface columns must match its policy schema in order" - ) - if interface.decoder.support != 1 or interface.decoder.prediction_length != 1: - raise ValueError("PPO requires a one-step scalar categorical decoder") - if any( - ( - interface.categorical_hashing, - interface.static_conditioning, - interface.preconditioning_image_encoder, - interface.feature_layout, - ) - ): - raise ValueError("PPO supports only scalar observation and static fields") - if self.model.backbone.repository is not None: - raise ValueError("PPO actor initialization uses initialize_from") - if self.context_length > self.model.backbone.architecture.max_context_length: - raise ValueError("PPO context exceeds the model context limit") - device = torch.device(self.device) - if device.type not in {"cpu", "cuda", "mps"}: - raise ValueError("PPO requires a supported local torch device") - if not self.normalization_statistics and any( - column not in policy.categorical_n_classes - for column in policy.observation_columns + policy.static_columns - ): - raise ValueError("real PPO fields require frozen normalization") - json.dumps(self.model_dump(mode="json"), allow_nan=False) - outputs = [self.model_path, self.report_path, self.checkpoint_path] - if len(set(outputs)) != len(outputs) or self.initialize_from in outputs: - raise ValueError( - "PPO outputs must be distinct from each other and actor initialization" - ) - for output in outputs: - destination = Path(output) - if destination.is_dir() or any( - parent.exists() and not parent.is_dir() - for parent in destination.parents - ): - raise ValueError("PPO output paths require writable file destinations") - if any( - destination in Path(other).parents - for other in outputs - if other != output - ): - raise ValueError("PPO output files cannot contain another output path") - for environment in (phase.environment, phase.evaluation.environment): - if set(environment.units) - set( - policy.observation_columns + policy.static_columns - ): - raise ValueError("environment units must refer to policy fields") - if phase.environment.units != phase.evaluation.environment.units: - raise ValueError("training and evaluation units must match") - if phase.archive_directory is not None: - phase.archive_directory = str( - (Path(self.project_root) / phase.archive_directory).resolve() - ) - archive = Path(phase.archive_directory) - if (archive.exists() and not archive.is_dir()) or any( - parent.exists() and not parent.is_dir() for parent in archive.parents - ): - raise ValueError( - "PPO archive_directory requires a directory destination" - ) - if any( - archive == Path(path) - or archive in Path(path).parents - or Path(path) in archive.parents - for path in outputs - + ([self.initialize_from] if self.initialize_from else []) - ): - raise ValueError( - "PPO archive directory must be separate from artifacts" - ) - return self - class DatasetPartSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -1232,6 +1071,12 @@ def validate_model_name(cls, value): @model_validator(mode="after") @beartype def validate_relationships(self): + if self.global_training.training_objective == "policy" or any( + interface.policy is not None for interface in self.model.interfaces.values() + ): + raise ValueError( + "policy training requires PolicyRunConfig; supervised and policy phases cannot be mixed" + ) if self.global_training.early_stopping_epochs is not None and ( self.evaluation is None or self.evaluation.monitor is None ): @@ -1492,7 +1337,7 @@ class ResolvedTrainingSource(BaseModel): class ResolvedTrainingPhase(BaseModel): model_config = ConfigDict(extra="forbid") - algorithm: Literal["supervised", "policy_imitation"] = "supervised" + algorithm: Literal["supervised"] = "supervised" name: str epochs: int @@ -1625,8 +1470,8 @@ def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: @dataclass(frozen=True) class LoadedTrainConfig: - config: SequifierConfig | PolicyTrainConfig | PPOTrainConfig - resolved: ResolvedSequifierConfig | PolicyTrainConfig | PPOTrainConfig + config: SequifierConfig | PolicyRunConfig + resolved: ResolvedSequifierConfig | PolicyRunConfig metadata: dict[str, DatasetMetadata] @@ -2499,26 +2344,25 @@ def load_train_config_with_source( if args: raise ValueError(f"Unsupported training CLI overrides: {sorted(args)}") - plan = raw.get("training_plan", {}) - phases = plan.get("phases", [plan]) if isinstance(plan, dict) else [] - algorithms = { - phase.get("algorithm", "supervised") - for phase in phases - if isinstance(phase, dict) - } - if algorithms - {"supervised"}: - if len(phases) != 1 or not algorithms <= {"policy_imitation", "iql", "ppo"}: + if "phases" in raw: + if ( + "training_plan" in raw + or "dataset_training" in raw + or "global_training" in raw + ): raise ValueError( - "Policy training supports exactly one policy_imitation, iql, or ppo phase." + "policy phases cannot be mixed with supervised training configuration" ) if sensitive or metadata_inline_values or skip_metadata: - raise ValueError( - "trajectory training does not accept supervised metadata overrides" - ) - authored = ( - PPOTrainConfig if algorithms == {"ppo"} else PolicyTrainConfig - ).model_validate(raw) + raise ValueError("policy runs do not accept supervised metadata overrides") + authored = PolicyRunConfig.model_validate(raw) return LoadedTrainConfig(config=authored, resolved=authored, metadata={}) + plan = raw.get("training_plan", {}) + phases = plan.get("phases", [plan]) if isinstance(plan, dict) else [] + if any(phase.get("algorithm", "supervised") != "supervised" for phase in phases): + raise ValueError( + "policy training requires PolicyRunConfig with top-level phases" + ) config = try_catch_excess_keys(config_path, SequifierConfig, raw) part_overrides: dict[str, dict[str, str]] = {} @@ -2582,7 +2426,7 @@ def load_train_config_with_source( @beartype def load_train_config( config_path: str, args_config: dict[str, Any], skip_metadata: bool -) -> ResolvedSequifierConfig | PolicyTrainConfig | PPOTrainConfig: +) -> ResolvedSequifierConfig | PolicyRunConfig: return load_train_config_with_source( config_path, args_config, skip_metadata ).resolved diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py index 0927dd45..215dacfd 100644 --- a/src/sequifier/config/training_plan.py +++ b/src/sequifier/config/training_plan.py @@ -1,11 +1,9 @@ """Shared authored phase contracts; algorithm-specific budgets remain explicit.""" -import math -from typing import Annotated, Generic, Literal, Optional, TypeVar +from typing import Generic, Literal, Optional, TypeVar from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from sequifier.config.components import ComponentSpec from sequifier.config.identifiers import identifier as _identifier from sequifier.typechecking import beartype @@ -67,253 +65,9 @@ def validate_mode_fields(self): return self -class PolicyImitationPhaseConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - algorithm: Literal["policy_imitation"] - name: str = Field(default="policy", min_length=1) - epochs: int = Field(gt=0) - sources: list[TrainingSourceSpecModel] = Field(min_length=1) - - @model_validator(mode="after") - def validate_sources(self): - if any( - value.weight is not None or value.batches_per_selection is not None - for value in self.sources - ): - raise ValueError( - "policy sources do not support supervised source scheduling" - ) - return self - - @property - def trajectory_sources(self) -> list[str]: - return [value.source for value in self.sources] - - validation_sources: list[str] | None = Field(default=None, min_length=1) - policy: str = Field(min_length=1) - - -class IQLPhaseConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - algorithm: Literal["iql"] - name: str = Field(default="offline_iql", min_length=1) - policy: str = Field(min_length=1) - sources: list[TrainingSourceSpecModel] = Field(min_length=1) - validation_sources: list[str] | None = Field(default=None, min_length=1) - updates: int = Field(gt=0) - batch_size: int = Field(default=64, gt=0) - gamma: float = Field(default=0.99, gt=0, le=1, allow_inf_nan=False) - expectile: float = Field(default=0.7, gt=0, lt=1, allow_inf_nan=False) - inverse_temperature: float = Field(default=3.0, gt=0, allow_inf_nan=False) - max_weight: float = Field(default=100.0, ge=1, allow_inf_nan=False) - target_update_rate: float = Field(default=0.005, gt=0, le=1, allow_inf_nan=False) - actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - critic_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - actor_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - value_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - critic_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - actor_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - value_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - critic_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) - evaluation_interval: int = Field(default=100, gt=0) - checkpoint_interval: int = Field(default=100, gt=0) - - @model_validator(mode="after") - def validate_sources(self): - if any( - source.weight is not None or source.batches_per_selection is not None - for source in self.sources - ): - raise ValueError("IQL sources use uniform transition sampling") - return self - - @property - def trajectory_sources(self) -> list[str]: - return [value.source for value in self.sources] - - -class OnlineEnvironmentConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - factory: str - config: dict = Field(default_factory=dict) - protocol_version: Literal[2] = 2 - # Empty means unitless; adapters must report this exact declaration. - units: dict[str, str] = Field(default_factory=dict) - - @field_validator("factory") - @classmethod - def validate_factory(cls, value): - module, separator, attribute = value.partition(":") - if not separator or not module or not attribute.isidentifier(): - raise ValueError("environment.factory must use module:factory") - return value - - -class PPONormalizationConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - source: Literal["identity", "statistics"] - statistics: dict[str, dict[str, float]] = Field(default_factory=dict) - - @model_validator(mode="after") - def validate_statistics(self): - if self.source == "identity" and self.statistics: - raise ValueError("identity normalization cannot include statistics") - for column, values in self.statistics.items(): - if ( - set(values) != {"mean", "std"} - or not all(math.isfinite(value) for value in values.values()) - or values["std"] < 0 - ): - raise ValueError( - f"{column}: statistics require finite mean and nonnegative std" - ) - return self - - -class PPOEvaluationConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - environment: OnlineEnvironmentConfig - seeds: list[int] = Field(min_length=1) - max_episode_steps: int = Field(gt=0) - timeout_seconds: float = Field(gt=0, allow_inf_nan=False) - selection: Literal["deterministic", "stochastic"] = "deterministic" - metric: Literal[ - "mean_completed_return", "success_rate", "mean_completed_length" - ] = "mean_completed_return" - direction: Literal["min", "max"] = "max" - - @field_validator("seeds") - @classmethod - def validate_seeds(cls, values): - if any(value < 0 or value >= 2**63 for value in values): - raise ValueError("evaluation seeds must be in [0, 2**63)") - return values - - -class PPOResumeConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - policy: Literal["never", "if_exists", "required"] = "never" - mode: Literal["exact", "reset_environments"] = "reset_environments" - - -class PPOPhaseConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - algorithm: Literal["ppo"] - name: str = Field(default="online_ppo", min_length=1) - policy: str = Field(min_length=1) - environment: OnlineEnvironmentConfig - evaluation: PPOEvaluationConfig - normalization: PPONormalizationConfig - resume: PPOResumeConfig = Field(default_factory=PPOResumeConfig) - seed_policy: Literal["derived_per_reset"] = "derived_per_reset" - total_env_steps: int = Field(gt=0) - rollout_length: int = Field(gt=0) - allow_partial_rollout: bool = True - optimization_epochs: int = Field(default=4, gt=0) - minibatch_size: int = Field(default=64, gt=0) - accumulation_steps: int = Field(default=1, gt=0) - gamma: float = Field(default=0.99, ge=0, le=1, allow_inf_nan=False) - gae_lambda: float = Field(default=0.95, ge=0, le=1, allow_inf_nan=False) - clip_epsilon: float = Field(default=0.2, gt=0, lt=1, allow_inf_nan=False) - value_clip: float | None = Field(default=None, gt=0, allow_inf_nan=False) - value_coefficient: float = Field(default=0.5, gt=0, allow_inf_nan=False) - entropy_coefficient: float = Field(default=0.01, ge=0, allow_inf_nan=False) - target_kl: float | None = Field(default=None, gt=0, allow_inf_nan=False) - normalize_advantages: bool = True - reward_scale: float = Field(default=1.0, gt=0, allow_inf_nan=False) - actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - actor_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - value_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - actor_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - value_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - actor_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) - value_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) - evaluation_interval: int = Field(default=1, gt=0) - checkpoint_interval: int = Field(default=1, gt=0) - archive_directory: str | None = Field(default=None, min_length=1) - archive_scores: bool = False - max_buffer_bytes: int = Field(default=536870912, gt=0) - - @model_validator(mode="after") - def validate_plans(self): - if ( - not self.allow_partial_rollout - and self.total_env_steps % self.rollout_length - ): - raise ValueError( - "rollout_length must divide total_env_steps unless allow_partial_rollout" - ) - for optimizer in (self.actor_optimizer, self.value_optimizer): - if optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"}: - raise ValueError("PPO optimizers must be Adam, AdamW, SGD, or RMSprop") - if "lr" in optimizer.arguments or "params" in optimizer.arguments: - raise ValueError( - "PPO learning rates and parameter ownership belong to the phase" - ) - full, remainder = divmod(self.total_env_steps, self.rollout_length) - group_size = self.minibatch_size * self.accumulation_steps - max_steps = self.optimization_epochs * ( - full * ((self.rollout_length + group_size - 1) // group_size) - + ((remainder + group_size - 1) // group_size if remainder else 0) - ) - for scheduler in (self.actor_scheduler, self.value_scheduler): - if ( - scheduler.name == "OneCycleLR" - and scheduler.arguments.get("total_steps") != max_steps - ): - raise ValueError( - "PPO OneCycleLR.total_steps must equal the maximum authored optimizer-step budget" - ) - if ( - scheduler.name - in {"ReduceLROnPlateau", "ChainedScheduler", "SequentialLR"} - or "optimizer" in scheduler.arguments - ): - raise ValueError( - "PPO schedulers must advance independently on optimizer steps" - ) - return self - - Phase = TypeVar( "Phase", - bound=TrainingPhaseSpecModel - | PolicyImitationPhaseConfig - | IQLPhaseConfig - | PPOPhaseConfig, + bound=TrainingPhaseSpecModel, ) @@ -356,12 +110,3 @@ def validate_unique_names(self): SupervisedTrainingPlanModel = TrainingPlanModel[TrainingPhaseSpecModel] -PolicyTrainingPlanModel = TrainingPlanModel[PolicyImitationPhaseConfig | IQLPhaseConfig] -PPOTrainingPlanModel = TrainingPlanModel[PPOPhaseConfig] -AlgorithmPhase = Annotated[ - TrainingPhaseSpecModel - | PolicyImitationPhaseConfig - | IQLPhaseConfig - | PPOPhaseConfig, - Field(discriminator="algorithm"), -] diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index c5589372..64c7be85 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -104,7 +104,7 @@ There are seven standalone commands within sequifier: `make`, `preprocess`, `tra | --- | --- | | `make` | Create a new sequifier project with config templates. | | `preprocess` | Convert input data into fixed-length subsequences. | -| `train` | Train a supervised model or a categorical policy from sequence tables or rollout recordings. | +| `train` | Train supervised models or ordered imitation, IQL, PPO, DPO and GRPO policy phases. | | `infer` | Generate predictions, probabilities, embeddings, or policy actions and rollouts. | | `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | | `visualize-training` | Generate interactive HTML plots from structured training metrics. | @@ -116,7 +116,7 @@ The processing and visualization commands have dedicated documentation pages: - [preprocess documentation](./documentation/configs/preprocess.md) - [train documentation](./documentation/configs/train.md) -- [policy training](./documentation/policy-learning.md), including BC, offline IQL and [online categorical PPO](./documentation/policy-learning.md#online-categorical-ppo) +- [policy training](./documentation/policy-learning.md), including BC, IQL, DPO, GRPO and [online categorical PPO](./documentation/policy-learning.md#online-categorical-ppo) - [infer documentation](./documentation/configs/infer.md) - [hyperparameter-search documentation](./documentation/configs/hyperparameter-search.md) - [visualize-training documentation](./documentation/commands/visualize-training.md) @@ -423,6 +423,8 @@ Please cite with: ``` +Policy runs support ordered imitation, IQL, DPO, GRPO and PPO phases; see the [policy learning guide](documentation/policy-learning.md). + # Preprocess Command Guide @@ -1719,82 +1721,32 @@ healthy active update. Momentum and weight decay cannot update parameters whose gradients are absent. -## Policy sequence training - -`sequifier train` accepts one `policy_imitation` or `iql` phase. Put the observation/action -schema in `model.interfaces..policy`, a sequence source in each -`dataset_training..parts.`, and optimizer/context settings in -`global_training` with `training_objective: policy`. Reference sources with -`{source: dataset.part}`. The algorithm supplies action NLL. - -`return_conditioned: false` selects BC, which needs observations and action labels. -Return-conditioned training additionally requires rewards and a known return -endpoint. CSV/Parquet tables, JSON rollout logs and scalar prepared histories all -use the same episode representation. Models support CPU/CUDA/MPS, backbone -transfer via `initialize_from`, and shared-engine checkpoint resume via -`global_training.resume`. IQL additionally requires complete transition data -and an unconditioned categorical policy. PPO uses the separate online schema below. -Mixed phases and image/depth policies remain planned. The former compact registries and configuration aliases have been -consolidated into the shared model/dataset authoring surface. - -See the [complete table example](../example_inputs/configs/train-policy-from-table.yaml), -[return-conditioned example](../example_inputs/configs/train-return-conditioned.yaml), -[IQL example](../example_inputs/configs/train-iql-from-table.yaml), -and [policy learning guide](../policy-learning.md). - -## Online PPO configuration - -Online PPO uses `PPOTrainConfig`, selected automatically by one `algorithm: ppo` -phase. It requires no datasets, sources, validation loaders, `epochs` or -`global_training`. See the [runnable YAML](../example_inputs/configs/train-ppo-toy.yaml) -and [adapter/resume guide](../policy-learning.md#online-categorical-ppo). - -Root fields are `project_root` (default `.`), `model_name`, `model`, -`context_length` (defaults to the backbone maximum), `device` (default `cpu`), -`seed` (default 1010, in `[0, 2**32)`), `enforce_determinism` (default true), -optional `initialize_from`, and optional `model_path`, `report_path`, -`checkpoint_path`. Paths resolve against `project_root`; outputs must be -distinct. Exactly one scalar policy interface is allowed and -`return_conditioned` must be false. Schema/class counts, action order, -normalization and architecture must be explicit and frozen. - -The phase accepts these fields; extra fields are rejected: - -| Fields | Default / contract | -| --- | --- | -| `algorithm`, `policy`, `name` | `ppo`, selected interface, `online_ppo` | -| `environment` | Required `factory: module:factory`, `config: {}`, `protocol_version: 2`, `units: {}` | -| `normalization` | Required `source: identity` or `statistics`; the latter requires exact real-column `statistics: {column: {mean: ..., std: ...}}` | -| `total_env_steps`, `rollout_length` | Required positive executed-action budgets | -| `allow_partial_rollout` | true; false requires a divisible total budget | -| `optimization_epochs`, `minibatch_size`, `accumulation_steps` | 4, 64, 1; all positive | -| `gamma`, `gae_lambda` | 0.99, 0.95; both in `[0, 1]` | -| `clip_epsilon`, `value_clip` | 0.2 in `(0, 1)`; null or positive value clip | -| `value_coefficient`, `entropy_coefficient` | 0.5 positive; 0.01 nonnegative | -| `target_kl`, `normalize_advantages` | null or positive KL limit; true | -| `reward_scale` | 1.0; fixed positive finite multiplier, checkpointed | -| `actor_learning_rate`, `value_learning_rate` | 0.0003 each, positive finite | -| `actor_optimizer`, `value_optimizer` | `{name: Adam}`; Adam/AdamW/SGD/RMSprop with constructor arguments; phase owns `lr` and parameters | -| `actor_scheduler`, `value_scheduler` | `{name: ConstantLR, factor: 1.0, total_iters: 1}`; scheduler constructor arguments, stepped after each role's optimizer step | -| `actor_gradient_clip`, `value_gradient_clip` | 1.0 each; positive finite norm limit | -| `evaluation_interval`, `checkpoint_interval` | 1 each, measured in complete iterations; final iteration always runs both | -| `evaluation.environment` | Required separate adapter factory/config, same frozen units as training | -| `evaluation.seeds` | Required nonempty list of integers in `[0, 2**63)` | -| `evaluation.max_episode_steps`, `evaluation.timeout_seconds` | Required positive per-episode action cap and whole-evaluation deadline | -| `evaluation.selection` | `deterministic` or explicitly `stochastic` | -| `evaluation.metric`, `evaluation.direction` | `mean_completed_return`, `max`; alternatives `mean_completed_length` / `success_rate`, direction `min` / `max` | -| `resume` | `{policy: never, mode: reset_environments}`; policy also `if_exists`/`required`, mode also `exact` | -| `seed_policy` | `derived_per_reset`; separate training/evaluation streams | -| `max_buffer_bytes` | 536870912; positive conservative host-buffer budget, estimated before collection and enforced per insertion | -| `archive_directory`, `archive_scores` | null (disabled), false; separate versioned collection records | +## Policy runs + +Policy training uses one `PolicyRunConfig` with top-level ordered `phases`, +`model`, `policy_interface`, `context_length`, optional named `sources` and +`preferences`, and run-wide output/resume settings. Supported algorithms are +`policy_imitation`, `iql`, `ppo`, `dpo`, and `grpo`. Epochs, offline updates, +PPO environment steps and GRPO groups are explicit per-phase budgets. + +Each phase selects `handoff: best | last` (default `best`). One frozen actor +representation and causal history contract spans the run; all optimizer/critic +state starts fresh at each handoff. Return conditioning supports imitation only. +Policy runs are local FP32 runs and cannot contain supervised forecasting phases. +The supervised `training_plan`/`dataset_training`/`global_training` schema above +remains independent. + +See the [policy learning guide](../policy-learning.md) for full fields, objective +formulas, environment capabilities, preference manifests, checkpoint boundaries, +selection monitors and exact/non-exact continuation. Examples: -`OneCycleLR.total_steps` must equal the maximum authored optimizer-step budget -(including accumulation and a final partial rollout); KL stopping may use fewer -steps. Plateau/compound schedulers, distributed modes, layer autocast and ONNX are not -PPO config fields. Optimizer and scheduler constructor errors are raised before -collection. A final shorter rollout executes exactly the remaining actions; -terminal actions count toward the budget. Checkpoint/export paths and resume -modes are algorithm-owned and do not change the supervised or offline schemas. +- [BC](../example_inputs/configs/train-bc-toy.yaml) +- [Return-conditioned imitation](../example_inputs/configs/train-return-conditioned.yaml) +- [IQL](../example_inputs/configs/train-iql-from-table.yaml) +- [PPO](../example_inputs/configs/train-ppo-toy.yaml) +- [DPO](../example_inputs/configs/train-dpo-control.yaml) +- [GRPO](../example_inputs/configs/train-grpo-control.yaml) +- [Imitation → IQL → DPO → GRPO → PPO](../example_inputs/configs/train-multiphase-control.yaml) # Infer Command Guide @@ -2066,12 +2018,12 @@ ORT installation. Outputs stay indexed by outer item coordinates. ## Policy model interfaces -`sequifier infer` recognizes a policy interface in a portable PyTorch model or -run checkpoint. Use raw CSV/Parquet observation rows and optional recorded action -histories; alignment, vocabulary, and normalization come from the artifact. +`sequifier infer` recognizes a policy interface in a portable PyTorch actor. +Use the selected actor export rather than the policy run checkpoint. Use raw +CSV/Parquet observation rows and optional recorded action histories; alignment, vocabulary, and normalization come from the artifact. `model_type: generative` and `model_type: policy` are accepted. Environment execution uses the existing `rollout` section with `mode: policy`. See the -[policy guide](../policy-learning.md#predict-actions-with-sequifier-infer) and +[policy guide](../policy-learning.md#python-and-inference) and [inference example](../example_inputs/configs/infer-policy.yaml). @@ -2393,701 +2345,431 @@ srun torchrun \ * **Data Access:** All nodes must have access to the same shared filesystem (e.g., NFS, GPFS) where the `project_root` and the sharded preprocessing output are stored. -# Policy learning through `sequifier train` and `sequifier infer` +# Categorical policy learning -Policy learning uses Sequifier's configurable ingestion, transformer network, -decoders, portable model artifacts, optimizer transactions, and run checkpoints. -A policy adds a decision-alignment contract to a model interface: at time `t`, -the inputs contain the current observation, the **previous** action, and, -optionally, a desired return. The current action is the supervised target. +`sequifier train` and `sequifier.rl_api.run_policy_training` execute the same +`PolicyRunConfig`. Its ordered `phases` may contain `policy_imitation`, `iql`, +`ppo`, `dpo`, and `grpo` in any compatible order. Each phase receives a deep copy +of its predecessor's selected actor. Optimizers, schedulers, critics, samplers, +and environments start fresh at phase boundaries. -## Train from ordinary sequence tables - -The phase algorithm is `policy_imitation` for both behavior cloning and -return-conditioned imitation. The policy's `return_conditioned` setting selects -between them. Offline IQL uses a separate `iql` phase with an unconditioned -categorical actor. - -Use the existing `sequenceId` and `itemPosition` conventions. Each row describes -**the current observation, the action executed at that observation, and the -reward/boundary resulting from that action**. One sequence is one episode or an -explicit episode fragment. CSV and Parquet files can contain many episodes; -files are not assumed to be episode boundaries. Positions must be nonnegative, -unique and contiguous within each sequence, including across input shards. - -```yaml -project_root: . -model_name: table-policy -device: cpu -seed: 1010 +This release supports one fixed categorical action, scalar or categorical +observations/static fields, causal fixed-length histories, FP32, and local +single-process CPU/CUDA/MPS execution. Return conditioning is available only +for imitation. A run containing another algorithm requires an unconditioned +actor throughout. Forecasting and policy phases cannot share a training plan. +Text tokenizers, continuous/joint actions, images/depth, distributed training, +vectorized environments, and reward models are outside this contract. -model: - backbone: - architecture: - dim_model: 32 - num_layers: 2 - max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} - positional_encoding_scope: global - interfaces: - actor: - input_columns: [position, velocity] - target_columns: [action] - policy: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - return_conditioned: false - ingestion: - type: embedding - output_dim: 8 - feature_embedding_dims: - position: 1 - velocity: 1 - __policy_previous_action: 6 - decoder: {type: linear, prediction_length: 1, support: 1} +## Authoring a run -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: [data/demonstrations.parquet] +Start with the [multi-phase example](example_inputs/configs/train-multiphase-control.yaml), +[single-phase DPO](example_inputs/configs/train-dpo-control.yaml), or +[single-phase GRPO](example_inputs/configs/train-grpo-control.yaml). +Generate their curated data with: -global_training: - training_objective: policy - context_length: 16 - batch_size: 64 - learning_rate: 0.005 - validation_fraction: 0.2 - -training_plan: - phases: - - name: behavior_cloning - algorithm: policy_imitation - policy: actor - epochs: 80 - sources: [{source: demonstrations.recorded}] +```shell +python documentation/example_inputs/policy_control_data.py data +sequifier train --config-path documentation/example_inputs/configs/train-multiphase-control.yaml ``` -Run `sequifier train --config-path configs/train-policy.yaml`. BC -(`return_conditioned: false`) needs observations and action labels, without -rewards, successor observations, bindings, simulator metadata or fabricated seed -windows. A recorded current action is never also accepted as an observation from -the same source column. Return-conditioned modeling additionally needs recorded -rewards and a known terminal outcome or an explicitly configured finite horizon. -Unknown/truncated reward tails are excluded and reported; missing rewards never -become zero. Setting `return_conditioned: true` selects this mode. - -Defaults use the policy feature names and the columns `sequenceId`, -`itemPosition`, `reward`, `terminated` and `truncated`. Map different source names -explicitly: +The example defines imitation → IQL → DPO → GRPO → PPO. Remove or reorder phase +entries to author smaller runs, such as imitation → IQL, imitation → PPO, +imitation → DPO, imitation → GRPO, or DPO → GRPO. The input preferences come from +an explicitly supplied sign-matching rule; Sequifier never derives preferences +from recorded reward. These are illustrative configurations, without a claim +that every algorithm order improves every task. ```yaml -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: [data/recorded.csv] - columns: - sequence_id: episode - item_position: step - observations: {position: measured_position, velocity: measured_velocity} - action: command - reward: score - terminated: task_finished - truncated: time_limit - previous_action: preceding_command - group: collection_session -``` - -Boundary columns, when present, contain booleans. No row may follow a boundary -within the same episode. Both termination and truncation may be true; termination -disables bootstrap. `episode_end: terminated` or `truncated` is an explicit claim -about each sequence's final row and must agree with recorded flags. The default -is `unknown`, never an inferred terminal outcome at file end. - -Position zero with no preceding action is an episode start. Fragments require -`columns.previous_action` at their first position; subsequent entries must lag -current actions by exactly one decision. At window boundaries, the preceding -action is fetched from the sequence rather than replaced with a start token. -Static features can be constant inline columns or supplied in a separate -`static_data_path` CSV/Parquet table keyed by the sequence ID. The table must cover -every selected ID without duplicates; fitting reads only training episodes. - -Optional `columns.successors` maps every observation name to its next-observation -column. An all-null successor is explicitly missing; partial successors are -invalid. Adjacent observations must agree with supplied successors. Optional -`columns.bootstrap_allowed` cannot permit bootstrap after termination or at a -final decision without an observed successor. BC and return-conditioned modeling -do not require successors. Task outcomes and collection groups remain data -metadata rather than model features. - -Without `validation_sources`, training splits whole episodes deterministically, -keeping repeated demonstrations and declared collection groups together. At -least two independent groups are required. Alternatively declare a held-out source -and set `validation_sources: [held_out.recorded]` on the phase. Duplicate paths and episode -IDs, overlapping collection groups and repeated demonstrations across partitions -are rejected. Normalization and categorical vocabulary fitting use training data. - -Declare raw categorical observation/static fields with `categorical_columns` on -the policy. They use the original preprocessing vocabulary fitter and reserved -IDs. Alternatively supply fixed `observation_n_classes` / `static_n_classes` for -already encoded integer IDs and optional `id_maps` for fixed domain labels. -String action labels are supported through `action_values`. Executable action -IDs use the usual global-to-decoder mapping; the start marker is input-only and -cannot be sampled as an action. Previously saved policy artifacts retain their -legacy vocabulary layout. - -Sources accept `format: auto` (default), `table`, `trajectory` or `prepared`. -Automatic selection recognizes CSV/Parquet tables, JSON rollout logs and PT -prepared histories. Prepared CSV/Parquet windows require `format: prepared`. -Prepared sources require `metadata_config_path`; overlapping windows are checked -and reconstructed into unique, split-owned decisions. A preceding action can be -recovered from stored context for a fragment. Categorical IDs and normalized -values are decoded using the original metadata, then fitted for the selected -training partition. Missing reward/boundary information remains unknown. -For a metadata-referenced prepared split, use `metadata_config_path` and -`split_index` instead of `paths`. Scalar prepared histories are currently supported; -policy image/depth ingestion remains planned; IQL and PPO support the scalar -transition-capable sources described below. - -A source can still specify `paths: [episode-a.json, episode-b.json]` or a legacy -`manifest` listing version-1 JSON episodes. These are validated rollout recordings -adapted into the same sequence representation, with bindings retained in their -recording format. Forecast recordings require -`action_semantics: executed_predictions`; this does not convert forecast checkpoint -alignment into policy alignment. - -Run paths resolve relative to `project_root`; manifest entries resolve relative -to the manifest directory. `global_training.num_workers` controls loader workers; -`episode_cache_size` bounds cached episodes per worker. Table-backed episodes use -filtered lazy scans. Prepared window reconstruction is eager per source and JSON -records still require one episode to fit in memory. - -The best model is selected by held-out action NLL. `model_path` defaults to -`models//.pt` and contains an ordinary `sequifier_model`. -The report records partitions, exclusions, the fitted -policy codec, decision counts and per-epoch NLL. Epochs belong to the phase, seed -and device to the run, and optimizer/scheduler settings to `global_training`. - -## Optional committed import - -Direct training from a table needs no conversion. For reusable imported data, -`sequifier preprocess` supports a separate operation: - -```yaml -operation: import_sequences project_root: . -source: - paths: [data/demonstrations.csv] -policy: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - return_conditioned: false -output_path: data/imported-demonstrations -episodes_per_shard: 64 -``` - -The output directory must be new. Import writes raw Parquet shards and a -checksummed `manifest.json`, then atomically publishes the directory. It does not -fit statistics, pad contexts or invent missing transitions. Train with -`dataset_training.demonstrations.parts.recorded.manifest: data/imported-demonstrations/manifest.json`; -the recorded column mappings are loaded automatically. The manifest retains -source provenance, episode identities, optional successors and collection groups. - -## Shared model and dataset authoring - -Policy training uses the ordinary `model.backbone` and `model.interfaces` -schemas, with a `policy` data schema on each interface. Ingestion and decoder -components keep their existing syntax; the policy adapter adds previous-action -and return features. `global_training.context_length` selects a context no larger -than the backbone maximum. `global_training` supplies optimizer settings and -`training_objective: policy`. The former compact policy registries and aliases -have been consolidated into this shared authoring surface. - -Likewise, `dataset_training.` accepts `model_interface` and `parts`, with each -part containing a sequence source specification. Reference it with the ordinary -`{source: dataset.part}` phase syntax. The algorithm supplies action NLL, so these -parts do not need fabricated criteria. Model/interface references are validated. -The shared phase schema preserves supervised defaults; policy training supports -one `policy_imitation`, `iql`, or `ppo` phase, with BC selected on its policy. -Mixed algorithms, backbone repositories, and image/depth policies remain planned. -Use `initialize_from` for backbone artifact transfer. - -See [train-policy-from-table.yaml](example_inputs/configs/train-policy-from-table.yaml) -for a complete example using the shared model, dataset and phase authoring surface. - -## Configure the shared model - -Configure the backbone at `model.backbone.architecture` and ingestion/decoder -components at `model.interfaces.`. For a larger model, for example: - -```yaml -model: - backbone: - architecture: - dim_model: 64 - num_layers: 3 - max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} - positional_encoding_scope: global +model_name: control +policy_interface: actor +context_length: 4 +seed: 1010 +device: cpu +# model uses the ordinary backbone/interfaces architecture schema. +model: { ... } +sources: + decisions: + paths: [data/control-decisions.csv] + columns: + episode_id: episodeId + group: contextId +preferences: + rankings: data/control-preferences.json +phases: + - name: imitate + algorithm: policy_imitation + sources: [decisions] + epochs: 4 + - name: rank + algorithm: dpo + preferences: rankings + updates: 100 + beta: 0.1 + reference: {source: phase_start} +resume: {policy: if_exists, mode: exact} ``` -Use an embedding ingestion to project real features and embed categorical -features before adapting to the backbone width. Supply ingestion and decoder -components explicitly as in the complete examples. A policy decoder must have -`support: 1` and `prediction_length: 1`. - -The internal input names `__policy_previous_action` and `__policy_return` are -reserved. The latter exists only for return-conditioned policies. Static columns -are broadcast over the decision window. Real observation and static sample statistics -are persisted in the model interface's preprocessing metadata. Encoded categorical -observations can be declared with `observation_n_classes: {state_id: 8}`; their -recorded values must be integral IDs in `[0, 8)` and are not normalized. -When authoring a mixed real/categorical embedding ingestion, provide -`feature_embedding_dims` under the normal ingestion rules, including the internal -previous-action column and, when enabled, return column. - -Actions use decoder-local IDs in the order of `action_values`, mapped to the -standard global category IDs. A separate input-only ID represents episode start -and is excluded from decoder outputs. -An observation may have the same external name as the action; in that case the -model uses `__policy_action` as its separate target name. Default decoders infer -the target automatically. - -Context length must fit `max_context_length`. -The standard ingestion also supports -feature-scoped learned positions when configured. -The standard attention, normalization, position encodings, dropout, and layer -sharing options remain available. - -To transfer a forecast or policy backbone, set a top-level -`initialize_from: models/forecast.pt`. Portable models and run checkpoints are -accepted. The source must have the same backbone architecture. Ingestion and -decoder parameters are initialized for the policy's own interface. Resumption -restores the entire existing policy instead of applying initialization again. - -## Resume training - -Every completed epoch saves a standard `sequifier_run_checkpoint` at -`checkpoints/runs//-latest.pt`. It contains current model -weights, preprocessing and policy semantics, optimizer/scheduler/scaler state, -random state, data-order generator state, best weights, and training progress. -The portable model at `model_path` contains the best validation weights; the run -checkpoint contains the latest training weights. +Run-wide fields: -```yaml -global_training: - resume: - policy: if_exists -``` +| Field | Meaning / default | +| --- | --- | +| `model`, `policy_interface`, `context_length`, `model_name`, `phases` | Required architecture, selected interface, history length, run name, ordered phases | +| `project_root`, `device`, `seed`, `enforce_determinism` | `.`, `cpu`, 1010, true; seed in `[0, 2**32)` | +| `sources` | Named `SequenceSourceConfig` objects; paths/manifest, format, explicit column mappings, prepared metadata | +| `preferences` | Named paths to version-1 preference-pair JSON manifests | +| `units` | Frozen observation/static units; `{}` means unitless; online training/evaluation adapters must match | +| `initialize_from` | Optional portable actor with matching full representation and architecture | +| `normalization` | Optional explicit frozen `source: identity` or `source: statistics` with exact real-field `statistics: {column: {mean: ..., std: ...}}` | +| `validation_fraction`, `episode_cache_size` | 0.2 and 8; split independent components before fitting codecs/statistics | +| `resume` | `{policy: never, mode: reset_environments}`; policy also `if_exists`/`required`, mode also `exact` | +| `model_path` | `models//actor.pt` | +| `report_path` | `outputs/training//report.json` | +| `checkpoint_path` | `checkpoints//run.pt` | +| `phase_directory` | `outputs/training//phases`; subdirectories `000-`, etc. | + +The model must contain exactly `policy_interface`. Its input columns list the +policy observations followed by static fields; its target is the action column. +Use a scalar categorical decoder with `support: 1`, `prediction_length: 1`. +Declare `return_conditioned: false` for BC or any improvement algorithm. Standard +ingestion/backbone/decoder configuration remains available. Previous executable +actions enter the input as `__policy_previous_action`; its start token is +input-only. Return-conditioned imitation additionally uses `__policy_return`. + +The first offline phase fits missing categorical vocabularies and normalization +on its training episodes only. A declared normalization takes precedence over +fitting statistics. An online-first run must supply frozen normalization and +categorical counts/codecs, or a compatible `initialize_from` actor. Later phases +reuse the complete frozen representation. They may optionally declare +`representation` and online `normalization` as assertions; any disagreement +fails with a field-specific diagnostic before training. An online phase after +an offline phase normally omits `normalization` to reuse the fitted values. + +All input files, preference manifests, metadata, initial actors and references +are protected from output collisions. Source bytes, partitions, codecs, model, +units, reference weights, and adapter identities are checked on resume. + +## Selection and optimization + +Every phase has a unique identifier `name` and `handoff: best | last` (default +`best`). Best selection retains the earliest finite best on ties. The final +update always evaluates. Missing finite evaluation for `best` fails explicitly. +`last` selects the most recently completed optimizer transaction. The final +export uses the last phase's selected actor, with metric, transaction and weight +hash recorded; metrics from different phases are never compared globally. + +Imitation, DPO and GRPO share these actor optimization fields: + +| Field | Default | +| --- | --- | +| `batch_size`, `accumulation_steps` | 32, 1; DPO counts pairs, GRPO counts whole trajectories | +| `actor_learning_rate`, `actor_optimizer` | 0.0003, `{name: Adam}` | +| `actor_scheduler` | `{name: ConstantLR, factor: 1.0, total_iters: 1}` | +| `gradient_clip` | 1.0 | +| `evaluation_interval`, `checkpoint_interval` | 1; optimizer updates for imitation/DPO, completed collection/optimization updates for GRPO | + +Optimizers support Adam, AdamW, SGD and RMSprop. Their constructor arguments +belong alongside `name` (or in `arguments`); `lr` and parameters belong to the +phase. Schedulers advance on completed optimizer steps; plateau/compound +schedulers are unsupported. `OneCycleLR.total_steps` must equal the authored +maximum optimizer-step budget. Accumulation weights by the actual number of +examples, including final partial batches. Network stochastic layers are disabled +while gradients remain enabled; likelihood ratios use reproducible scores. + +## Offline sequences and imitation + +Sources accept CSV/Parquet tables, JSON rollout logs, committed sequence +manifests, or scalar prepared histories. A row has current observations and its +executed action; optional rewards/successors describe the resulting transition. +`PolicyTableColumns` exposes explicit observation/static/action/reward/group/ID +mappings. Temporal positions must be contiguous, zero-based at episode starts. +Fragments require a known preceding action. Reward-free sequences are ordinary +`SequenceEpisode` objects with absent transition capabilities, rather than +fabricated reward-bearing trajectories. + +`policy_imitation` uses `sources`, optional disjoint `validation_sources`, and +`epochs`. It minimizes action negative log likelihood at each valid causal +decision. The monitor is `validation_action_nll`, direction `min`. BC requires no +rewards. Return conditioning requires recorded rewards and either true task +termination or an explicit finite return horizon, using the declared discount +and scale. Unknown return tails are excluded. The input return label at decision +`t` includes reward `r_t`; the current action is never an input at that decision. + +Episode/group/duplicate-content components remain wholly in training or +validation. Explicit validation sources undergo the same overlap checks. +Episodes load through bounded caches; windows are constructed on demand. -`never` is the default. `required` also requires an explicit `checkpoint_path`, -using the shared resume configuration. Policy runs use the same `TrainingRun`, -`TrainingEngine`, evaluation and checkpoint services as supervised runs. Resume -restores the saved optimizer boundary and loader positions. Trajectory content, partition order, policy, -and learning settings must match. Device, output paths, worker/cache settings, -and total epoch budget may change. Restored scheduler state retains its existing -schedule when extending the epoch budget. Numerical reproducibility across -devices is governed by PyTorch. +## Offline IQL -Private policy run checkpoints written before the shared-engine consolidation -use a different run-state schema. Start a new run with `initialize_from` pointing -to the old model or checkpoint; its execution weights remain loadable. New runs -write ordinary Sequifier run state and loader state. +`iql` requires complete rewards, termination/truncation and bootstrap flags, +with genuine successors for bootstrapped decisions. Nonterminal transitions +bootstrap; true terminal transitions do not. Never substitute a reset +observation for a successor. -## Predict actions with `sequifier infer` +The phase has `updates` (required), `batch_size` (64), `gamma` (0.99), +`expectile` (0.7), `inverse_temperature` (3), `max_weight` (100), and +`target_update_rate` (0.005). Independent value and twin-Q networks use +expectile regression and one-step Bellman regression with Polyak target-Q +updates. Actor extraction minimizes action NLL weighted by +`min(exp(inverse_temperature * (min(target_Q1,target_Q2)-V)), max_weight)`. +Each update samples transitions uniformly with replacement. Held-out unweighted +`validation_action_nll` is the minimizing selection monitor; Bellman residual, +values, Q and weight diagnostics are reported separately. -Inference reads policy alignment and preprocessing directly from a portable -model or run checkpoint; a training YAML or preprocessing metadata file is not -needed. It accepts `model_type: generative` (or the explicit alias `policy`): +`actor_`, `value_`, and `critic_learning_rate` default to 0.0003; each role has +its own optimizer/scheduler. `gradient_clip` defaults to 1.0, evaluation and +checkpoint intervals to 100 updates. Critics are fresh even after another IQL +phase and are absent from the portable actor. -```yaml -project_root: . -model_path: models/policy/policy.pt -data_path: data/observations.csv -read_format: csv -write_format: csv -device: cpu -deterministic: true -desired_return: 10.0 -``` +## DPO preferences -See [infer-policy.yaml](example_inputs/configs/infer-policy.yaml). CSV and Parquet -inputs contain observations and any static columns in original units. Without -a `sequenceId` column, each row is an independent first decision. Within each -sequence, `itemPosition` defines decision order when present and must be unique -and contiguous; otherwise row order is used. Static inputs must be constant. -Provide the recorded `action_column` to derive previous actions by shifting one -row, or set `previous_action_column` to a column containing previous **domain -action values**, with null for episode start. Use a new sequence ID for each -episode; null previous actions inside a sequence are rejected. When observation -and action names coincide, use an explicit `previous_action_column` for sequential -CSV/Parquet input. Current recorded actions never -enter the inputs for the same decision. Historical context is bounded by the -model's context length. - -Return-conditioned models accept a fixed `desired_return` or a per-row column -named by `desired_return_column` (default `desired_return`). Per-row returns are -supplied by the caller; offline inference does not update budgets from rewards. -BC requires no return input and rejects a scalar `desired_return`. - -`inference_batch_size` bounds batched model calls (default 64). Predictions go to `outputs/predictions//part-000.csv` by default. -`output_path` overrides that location. Sequence IDs and the configured `item_position_column` (default `itemPosition`), when -present, are retained in input order. `output_probabilities: true` adds one -`__probability_` column per action. Set -`deterministic: false` for seeded categorical sampling. Select `model_interface` -when an artifact contains multiple policy interfaces. - -## Execute an environment rollout - -Add the ordinary `rollout` configuration to policy inference: +The inspectable manifest is separate from its sequence sources: -```yaml -autoregressive: true -generation_steps: 1000 -rollout: - mode: policy - adapter: {factory: my_environment:create_adapter, config: {}} - bindings: - - {input: position, source: environment, key: position} - - {input: velocity, source: environment, key: velocity} - trajectory_path: outputs/trajectories/episode.json - seed_index: 0 +```json +{ + "format": "sequifier.preference-pairs", + "version": 1, + "pairs": [{ + "pair_id": "pair-1", + "context_id": "initial-condition-1", + "chosen": {"source": "decisions", "episode_id": "good", "start": 0, "end": 8}, + "rejected": {"source": "decisions", "episode_id": "bad", "start": 0, "end": 6} + }] +} ``` -`seed_index` selects one raw input row, forming a single-observation reset seed. -The shared rollout coordinator owns reset, step ordering, boundaries, and adapter -closure. Each predicted action enters policy history only after it is executed. -The [adapter protocol](rollout.md#adapter-protocol) is the same as for forecasts. -The resulting trajectory can be used as a later training source. - -The raw return budget starts at `desired_return` (or the seed row's return -column) and updates after each executed reward: +Span bounds index the loaded episode/fragment rows, zero-based, start-inclusive +and end-exclusive. Both sides need +at least one executable action. Unequal lengths are permitted. Canonical +observations, static fields, prior-action histories and valid masks must match +at the branch point. Each subsequent score uses that alternative's actual +observations and prior actions. Duplicate pair IDs, identical/overlapping +alternatives, unsupported labels, non-finite features and incompatible initial +contexts fail before updates. + +Splitting joins shared context/group IDs, episode IDs and duplicate learning +content transitively; explicit validation manifests obey the same constraints. +At least two independent components are required for automatic splitting. +Source identities, exact spans, fingerprints and membership persist in the run +checkpoint. The manifest must state the preferences; reward is not interpreted +as a preference. + +`dpo` requires `preferences` and `updates`; optional `validation_preferences` +selects an explicit validation manifest. `beta` defaults to 0.1 and must be +finite positive. `scoring_chunk_size` defaults to 64 causal windows. For the +summed FP32 action log likelihood `s`, it minimizes equally weighted pair losses: ```text -G_next = (G_current - reward) / gamma +margin = (s_actor(chosen)-s_actor(rejected)) - (s_ref(chosen)-s_ref(rejected)) +loss = -logsigmoid(beta * margin) ``` -Each reset restores the requested budget and clears policy history. The trajectory -records action log probabilities, previous-action IDs, and remaining raw budgets. -Termination and truncation remain separate; a collection cutoff preserves the -unfinished-episode flag. The coordinator always closes the adapter. - -## Labels, windows, and boundaries - -Raw return labels obey `G_t = r_t + gamma * G_(t+1)`, with terminal `G_T = 0`. -`returns.gamma` defaults to 1 and is in `(0, 1]`; `returns.scale` defaults to 1 -and divides the return only at ingestion. For rewards `[1, 2, 3]`, gamma 1 gives -`[6, 5, 3]`, and gamma 0.5 gives `[2.75, 3.5, 3]`. +Padding and unselected context positions contribute zero. Losses are averaged +over pairs, without length normalization or an added supervised term. Reports +include lengths, margin and preference accuracy. Default selection minimizes +`validation_dpo_loss`. Reference scores are recomputed, avoiding stale caches. +A two-pass backward calculation uses the exact scalar margin derivative and +recomputes one chunk's differentiable likelihood at a time, bounding activation +memory without changing the summed sequence objective. -Return-conditioned training excludes truncations and collection cutoffs with -unknown tails. `returns.finite_horizon: H` can label a fully observed prefix of -H decisions. Shorter fragments need true termination. Online finite-horizon -execution requires `rollout.max_episode_steps: H`. No unknown tail is assigned -zero. BC uses **all observed decisions**, including truncated and cutoff episodes, -because its loss does not consume return labels. +DPO and GRPO each declare `reference: {source: phase_start}` (default) or +`{source: artifact, path: models/reference/actor.pt}`. A phase-start reference is +a deep copy before any phase update. References stay in evaluation mode with +gradients disabled. Full representations and actual weight hashes must match. +Embedded checkpoint reference weights survive removal of the original file; +a still-present artifact with changed weights is rejected on resume. -Offline and online paths share `PolicyWindowView.inputs`, producing left-padded, -causal histories. The dataset slices to the context window before building -inputs, retaining the actual preceding action at a cropped boundary. Each -training example labels only its last position, so every decision contributes -once. Current actions, current rewards, and future observations are excluded -from inputs; return-to-go is the intentional future-derived condition. +## Online adapters -Policy trajectories use `sequifier.scalar-policy-trajectory`, version 1. -Forecast trajectories can be demonstrations with explicit -`action_semantics: executed_predictions` and a finite `action_values` vocabulary. -They retain the latest reset observation and start previous-action history with -the start token, because earlier forecast seed actions are unknown. - -## Python interfaces and scope +PPO and GRPO share the version-3 `OnlineEnvironmentAdapter` contract. Configure +`environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. +Factories must create fresh independent adapters for evaluation. An adapter +explicitly declares: ```python -from sequifier.config.train_config import load_train_config -from sequifier.train import run_training -from sequifier.rl_api import load_policy, PolicyWindowView - -config = load_train_config("configs/train-policy.yaml", {}, False) -run_training(config) -policy = load_policy(config.model_path, device="cpu") -# inputs = PolicyWindowView(policy.config).inputs( -# observations, previous_action_ids, desired_returns, static_inputs -# ).to(policy.device) -# distribution = policy.distribution(inputs) +from sequifier.rl_api import EnvironmentCapabilities, OnlineResetResult, OnlineStepResult + +class Adapter: + protocol_version = 3 + implementation_id = "my-simulator.v1" + capabilities = EnvironmentCapabilities( + exact_snapshot=True, + group_start="scenario", + reproducible_group_start=True, + group_start_randomness="independent", + ) + + def reset(self, seed: int) -> OnlineResetResult: ... + def step(self, action: dict) -> OnlineStepResult: ... + def close(self) -> None: ... + def snapshot_state(self) -> dict: ... + def restore_state(self, state: dict) -> None: ... + def create_group_start(self, seed: int) -> object: ... + def reset_group(self, token: object, rollout_seed: int) -> OnlineResetResult: ... ``` -Python callers use `SequifierInputs` for named model tensors and -`PolicyWindowView.inputs` for raw histories. `PolicySequenceDataset` accepts -`SequenceSourceConfig` entries or in-memory `SequenceEpisode` records; JSON logs -are adapted with `source_episodes`. The low-level `train_policy` takes training -and validation datasets plus a `PolicyTrainConfig` from -`sequifier.config.train_config`, sharing the same -options and checkpoint lifecycle as `run_policy_training`. - -`CategoricalPolicy` is an alignment adapter over the ordinary network, available -as `policy.network`. `policy.execution_config` is the canonical model config, -compatible with the shared artifact and export services. Distribution methods -include `mode`, `sample`, `log_prob`, and `entropy`. `collect_policy_rollout` -remains available for application-owned adapters. Generic analysis/loading tools -also accept policy model and run artifacts. - -The previous standalone numeric-policy artifacts (including versions 1 and 2) -use a different architecture and must be retrained. This workflow currently -supports one categorical actor phase, including the online PPO route below. Mixed supervised/policy plans, -distributed policy learning, custom run integrations, multimodal -trajectory storage, and ONNX policy execution through the CLI remain outside -this workflow. PyTorch policy execution supports CPU, CUDA, and MPS. - -## Offline IQL - -Use a single `iql` phase to improve an unconditioned categorical actor from a -fixed offline dataset. See the complete -[IQL configuration](example_inputs/configs/train-iql-from-table.yaml). The -actor can start from a compatible BC model with `initialize_from`. IQL creates -separate value, Q1, Q2, and target-Q networks; the portable model artifact -contains only the selected actor. The run checkpoint contains the full learner. -In this categorical slice, V and Q use independent copies of the configured -actor history encoder plus their own scalar heads; their architecture is not -authored in a separate model registry. - -Each selected decision needs a finite scalar reward, `terminated`, `truncated`, -and `bootstrap_allowed`. An ordinary nonterminal decision must bootstrap from -its genuine successor observation. An interior row may obtain that successor -from the following row; a final nonterminal or truncated row needs explicitly -mapped successor columns. A true terminal may omit its successor and cannot -bootstrap. Termination takes precedence when both boundary flags are true. -Unknown boundaries and missing rewards are errors for IQL even though BC can -use those demonstrations. Current observations and previous actions form the -actor/value history; Q alone receives the current recorded action. Splits, -categorical fitting, and numeric statistics come only from training episodes. - -`updates` is the number of optimizer cycles, each containing one value, critic, -and actor step. The phase controls uniform transition batch size, discount, -expectile, inverse temperature, maximum actor weight, target-Q update rate, -separate optimizer learning rates/specs and scheduler specs, gradient clipping, -and evaluation/checkpoint intervals. Schedulers advance after their own -optimizer update. The held-out action likelihood selects the portable actor; -the report also records value, Q, Bellman, advantage, and weight diagnostics. -The supported per-role optimizers are Adam, AdamW, SGD, and RMSprop. -`global_training.batch_size` and `global_training.learning_rate` remain required -by the shared policy config; the IQL phase's batch size and three learning rates -control its actual updates. -Use a separate application evaluation with `collect_policy_rollout` to compare -returns against BC in a local environment. -The [toy control example](example_inputs/iql_toy_control.py) generates a fixed -mixed-quality dataset for its [BC](example_inputs/configs/train-bc-toy.yaml) -and [IQL](example_inputs/configs/train-iql-toy.yaml) configs. Train BC first; -the IQL config warm-starts from its actor artifact. The example compares saved -actors over 100 fixed initial states and reports mean and standard deviation -rather than asserting an improvement for every seed. - -```console -python documentation/example_inputs/iql_toy_control.py generate data/offline-transitions.csv -sequifier train --config-path documentation/example_inputs/configs/train-bc-toy.yaml -sequifier train --config-path documentation/example_inputs/configs/train-iql-toy.yaml -python documentation/example_inputs/iql_toy_control.py evaluate models/toy-bc/toy-bc.pt models/toy-iql/toy-iql.pt -``` - -`global_training.resume: {policy: if_exists}` resumes the IQL learner from its -complete update checkpoint. `policy: required` needs an explicit -`checkpoint_path`. Exact resume validates the configuration, source content, -split, fitted vocabulary, and normalization. A BC model passed to -`initialize_from` is a warm start, not a learner checkpoint; the actor's policy -schema, normalization, model architecture, and context must match. IQL requires -float32 layers without autocast and supports scalar histories, one categorical -action, and local execution; a -return-conditioned actor, distributed training, mixed precision, continuous or -joint actions, and mixed phase plans are rejected. - -Python callers can use `run_iql_training(config)` from `sequifier.rl_api` with a -validated `PolicyTrainConfig`. The return value contains the selected -`CategoricalPolicy`, recorded update metrics, and training/validation episode IDs. +Reset returns the current observation, static inputs and units. Step returns +immediate scalar reward, actual successor observation, boundary flags, units +and optional boolean success. Only true termination may omit the successor. +`close()` runs in `finally`. `implementation_id` must match the authored config +and change whenever simulator/reset/RNG semantics change. + +GRPO requires `group_start: snapshot | scenario`, explicit methods, and an opaque +finite JSON-serializable token. Resetting a sample must reproduce the token's +initial condition while independently seeding future environment randomness. +`group_start_randomness` documents whether the token captures initial simulator +randomness or only the scenario. Group/sample indices and run/phase seeds derive +separate environment and action streams. Repeating a reset seed alone does not +establish this capability. Exact GRPO continuation additionally requires the +adapter's documented `reproducible_group_start` guarantee. The bundled +`CategoricalControlAdapter` demonstrates a deterministic scenario token. + +`SeededAdapterBridge` can adapt an existing seeded forecast-rollout adapter with +an explicit initial-state factory; it declares no group-start or exact-snapshot +capability. Forecast rollout bindings keep their separate version-1 contract. ## Online categorical PPO -PPO runs one persistent local environment with one categorical action. Start -with the [complete control config](example_inputs/configs/train-ppo-toy.yaml): +`ppo` requires `environment`, `evaluation`, `total_env_steps` and `rollout_length`. +Normalization is inherited from the run representation; a phase assertion is +optional. One persistent training environment collects compact sealed horizons. +True successors bootstrap fixed-length horizon/truncation boundaries; termination +disables bootstrap. FP32 GAE feeds the clipped policy objective, entropy bonus, +and separate value regression, with optional value clipping. Actor and value +optimizers share no parameters. Old collection scores are checked before update, +and collection generations are independent of phase/run indices. + +Defaults: `optimization_epochs: 4`, `minibatch_size: 64`, `accumulation_steps: 1`, +`gamma: 0.99`, `gae_lambda: 0.95`, `clip_epsilon: 0.2`, +`value_coefficient: 0.5`, `entropy_coefficient: 0.01`, +`normalize_advantages: true`, `reward_scale: 1`. Optional positive `target_kl` +stops optimization against the old collection policy; `value_clip` is optional. +`actor_learning_rate`/`value_learning_rate` default to 0.0003, with independent +optimizers/schedulers and `actor_gradient_clip`/`value_gradient_clip` of 1.0. +`allow_partial_rollout: true` executes exactly the remaining action budget; +false requires divisibility. `max_buffer_bytes` defaults to 536870912. +`archive_directory` optionally writes separate versioned collection records; +`archive_scores` defaults to false. Archived records are not training input. + +`evaluation_interval` and `checkpoint_interval` default to one complete +collection/optimization iteration. Evaluation uses a separate adapter, +independent seeds and a whole-evaluation POSIX/main-thread deadline. It requires +`environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; +`selection: deterministic | stochastic` defaults to deterministic. Default +`metric: mean_completed_return`, `direction: max` selects true completed returns. +`mean_completed_length` and `success_rate` are also supported. Censored episodes +and unavailable metrics are explicit; best selection fails when no finite +monitored value is available. The final iteration always evaluates. + +## GRPO groups and objective + +`grpo` requires `environment`, `evaluation`, `total_groups` and `max_steps`. +Defaults: `group_size: 4` (at least 2), `groups_per_update: 1`, +`optimization_epochs: 4`, `advantage_epsilon: 1e-6`, `clip_epsilon: 0.2`, +`kl_coefficient: 0.04`, `reward_scale: 1`, `scoring_chunk_size: 64`. +`old_policy_target_kl` optionally limits old-policy drift independently of the +reference penalty. No value network, value optimizer, or GAE is allocated. + +Each group contains independent sequential trajectories from the same opaque +start token under one frozen old actor generation. All groups in an iteration +are sealed before any update. Scores sum observed rewards times the fixed +reward scale, through termination, truncation or the explicit `max_steps` +finite-horizon cutoff. Cutoffs do not invent termination or bootstrapped rewards. +Incomplete/failed/non-finite groups are discarded and never optimized. Errors +report issued environment actions, including actions discarded with the group; +there is no retry. Optional `max_env_steps` is checked before collection using +the conservative `group_size * max_steps` bound for every requested group. +Memory is bounded by `max_buffer_bytes` (default 536870912), checked before +collection and as frames are appended. Optional archives are separate from the +training buffer. + +The precise first variant is: -```sh -sequifier train --config-path documentation/example_inputs/configs/train-ppo-toy.yaml +```text +A_i = (R_i - mean(R)) / (std_population(R) + advantage_epsilon) +r_it = exp(log pi_actor(a_it|h_it) - log pi_old(a_it|h_it)) +d_it = log pi_ref(a_it|h_it) - log pi_actor(a_it|h_it) +K_it = exp(d_it) - d_it - 1 +loss = mean_trajectories(mean_valid_decisions( + -min(r_it*A_i, clamp(r_it,1-epsilon,1+epsilon)*A_i) + kl_coefficient*K_it)) ``` -`sequifier make` also writes `configs/train-ppo.yaml`. The PPO root schema is -`PPOTrainConfig`: it has `model`, `context_length`, `seed`, `device`, and exactly -one `training_plan.phases` entry with `algorithm: ppo`. It has no -`dataset_training`, offline `sources`, `epochs`, or `global_training`. Phase -fields own both optimizers and their update clocks. See the -[PPO configuration reference](configs/train.md#online-ppo-configuration) and -[Python workflow](example_inputs/ppo_toy_control.py). - -The bundled `sequifier.rl.control:CategoricalControlAdapter` alternates a scalar -cue between -1 and +1. Choosing left for -1 and right for +1 earns +1; the other -action earns -1. Each episode executes eight actions. A random policy's expected -return is zero; the optimum is eight. The example config uses CPU, seed 1010, -context length 4, a 32-dimensional one-layer transformer, 256 decisions per -collection, and 8,192 total actions. The training report records hardware, -parameter counts, actual throughput and peak process memory. Learning quality -and multiple-seed improvement have not been measured for this implementation; -no training or test jobs were run for this increment. No CI job was added, -because the implementation request explicitly excluded writing or running tests. - -### Online adapter contract - -A factory receives `environment.config` as keyword arguments. It must return a -fresh adapter with `protocol_version = 2`, implementing: +Each trajectory has equal weight regardless of length; group sizes are fixed. +Minibatches contain whole trajectories; accumulation uses actual trajectory +counts. FP32 ratios/objectives/gradients are checked before applying an update. +Equal group scores give zero advantages; the reference penalty can still update +the actor. Reference and old-policy scores stay detached across optimization +epochs. The reported `sampled_reference_penalty` is a sampled estimator under +old-policy samples, not exact current-policy KL. + +Evaluation requires `metric: score_mean`, `direction: max` and scores every +independent evaluation episode through its real boundary or declared horizon. +The evaluation score uses the same fixed reward scale. Evaluation/checkpoint +intervals count completed updates, with an explicit final evaluation. Reports +include score spread, zero-variance groups, lengths, cutoff reasons, start +tokens, sampled penalty, clip fraction, generations, optimizer steps and issued +environment actions. Sealed buffers are consumed once; stale generations fail. + +## Checkpoints, resume, and artifacts + +The only policy-training checkpoint is `sequifier_policy_run`, format version 1. +It contains the resolved plan/fingerprint, immutable representation, data and +adapter identities, run RNG, current phase/index/status, completed results and +selected actors, algorithm/version and phase-owned state, and the last committed +transaction. DPO/GRPO reference weights are embedded. Optimizer, scheduler, +sampler, selection, evaluation and phase counters remain phase-local. + +Writes use a temporary file and atomic replacement at completed updates or +handoffs. No partial group is checkpointable. A committed handoff advances the +phase index exactly once; resume never reruns completed phases or reselects their +actor. Phase artifacts/reports can be republished idempotently from the stored +selection. Representation and data/adapter/reference mismatches fail before +output mutation. + +`resume.mode: exact` requires validated PPO simulator/RNG snapshots or GRPO's +reproducible group-start capability and matching implementation identity. PPO +checks the restored snapshot exactly; GRPO derives subsequent group tokens and +independent streams from persisted counters. `reset_environments` explicitly +resets PPO environments or restarts GRPO with a new derived stream, preserving +completed updates and reporting a trajectory discontinuity. Neither mode reuses +partial groups or old on-policy samples. Offline continuation restores its +sampler/order and optimizer transaction exactly. + +Portable artifacts are ordinary actor-only `sequifier_model` bundles. They +carry normalization, codecs, architecture, policy and unit provenance, with no +critics, references or trainer dependency. The final artifact comes from the +final phase's selected actor. The run report records order/config fingerprints, +source/reference provenance, selections, clocks and continuation modes. + +## Python and inference ```python -from sequifier.rl.environment import OnlineResetResult, OnlineStepResult - -class Environment: - protocol_version = 2 - supports_exact_snapshot = False - - def reset(self, seed: int) -> OnlineResetResult: - # Return initial domain values and episode-static inputs. - return OnlineResetResult({"observation": 0.0}) - - def step(self, action: dict) -> OnlineStepResult: - # Execute exactly once. Return the genuine successor, immediate finite - # scalar reward, and independent termination/truncation flags. - return OnlineStepResult({"observation": 1.0}, reward=1.0) +from sequifier.config.train_config import load_train_config +from sequifier.rl_api import PolicyRunConfig, run_policy_training, load_policy - def close(self): - pass +config = load_train_config("configs/train-policy.yaml", {}, False) +if not isinstance(config, PolicyRunConfig): + raise TypeError("expected a policy run") +result = run_policy_training(config) +actor = load_policy(result.model_path) +for phase in result.phases: + print(phase.name, phase.selected_metric, phase.clocks) ``` -The actor reads observations through the current decision and previous actions -only. After executing an action, its genuine successor and that action enter -the next causal window. An environment must never substitute an autoreset -observation for a final successor. A true terminal may return `observation=None` -and never bootstraps; truncation and collection cuts require a genuine successor -and bootstrap V from it. If both flags are true, termination wins. A horizon -cut preserves the live environment and its bounded causal history for the next -policy version. Only recorded episode boundaries trigger resets before the next -decision. Rewards must be final and immediate; delayed/provisional rewards and -reward providers are outside this contract. - -Observations and static fields must exactly match the frozen scalar schema. -Real values must fit finite FP32, categorical values must belong to declared -vocabularies, and units returned at every reset/step must equal the authored -`environment.units` mapping (empty for unitless values). Declare units for each -field whose units matter. Stochastic augmentation is unavailable. Dropout stays -disabled during collection and optimization, while scoring retains gradients. -The existing version-1 seeded inference adapter API is unchanged. -`SeededAdapterBridge` can wrap a v1 environment with same-named environment -observation bindings when an application supplies an initial-state factory. - -### Frozen representation and optimization - -The policy's observation/static schema, action order, class counts, label maps, -model architecture and context are fixed before the first reset. Author -`normalization: {source: identity}` deliberately, or use -`source: statistics` with a `statistics` map containing exactly the real fields' -`mean` and nonnegative `std`. Zero standard deviations from constant features -use the shared normalizer's epsilon, preserving BC/IQL warm-start statistics. -PPO never estimates statistics from changing online data. `initialize_from` -accepts an ordinary unconditioned BC/IQL actor only if -its policy schema, action ordering, normalization, model and context match -exactly. It initializes actor weights, not the learner state. A -return-conditioned actor is rejected. - -A separate history-only V owns independent parameters; PPO creates no Q or -target-Q networks. Each buffer stores a single policy generation, old action -log-probabilities, old values and true successor values. Compact frames retain -pre-buffer context; minibatches reconstruct ordered, complete windows. -FP32 GAE breaks its recurrence at episode boundaries and horizon cuts, while -bootstrapping at nonterminal boundaries. Advantage normalization occurs once per -iteration. Old scores, advantages and targets stay fixed for all epochs. - -The objective uses clipped probability ratios, independent squared value loss, -and current-policy categorical entropy. Optional value clipping is centered on -old V; it defaults off. Probability math is FP32. Gradient accumulation and the -last partial minibatch are weighted by their number of actual decisions. Actor -and V schedulers advance independently after their optimizer steps. Optional -KL limits stop further optimizer groups. Non-finite inputs, scores, losses, -gradients and parameters fail the run instead of admitting invalid decisions. - -### Evaluation, artifacts and resuming - -Evaluation uses a fresh adapter, frozen actor snapshot, fixed evaluation seeds, -and separate action/reset seed streams. It restores Python, NumPy and torch RNG -state afterward. Default evaluation selects mode actions; use -`selection: stochastic` explicitly for sampled actions. `max_episode_steps` -and `timeout_seconds` are mandatory. Evaluation runs on the main thread of a -local POSIX process, with an interval-timer deadline across creation, resets -and steps. Adapter methods must be interruptible and `close()` must return -promptly. A factory must return a different object from the live training slot. - -Reports include completed returns and their spread, episode lengths, completed -and censored counts, and success rates only when the adapter supplies success. -An environment truncation or evaluation step cap is censored; it does not count -as a completed episode. Best-actor selection defaults to maximizing -`mean_completed_return`; selection can instead use `mean_completed_length` or -`success_rate` with an explicit direction. No completed episodes means the -return metric is unavailable. If no evaluation supplies the selection metric, -the latest actor is exported with `latest_actor_metric_unavailable` recorded; -it is not described as a best actor. Initial evaluation also participates in -best-actor selection. - -The default outputs are `models//.pt`, -`outputs/training//report.json`, and `checkpoints//ppo.pt`. -The portable model contains actor weights only and loads with `load_policy`, -`sequifier infer`, and `collect_policy_rollout`. The example includes ordinary -[inference](example_inputs/configs/infer-ppo-toy.yaml) and seeded -[rollout](example_inputs/configs/rollout-ppo-toy.yaml) configs: - -```sh -sequifier infer --config-path documentation/example_inputs/configs/infer-ppo-toy.yaml -sequifier infer --config-path documentation/example_inputs/configs/rollout-ppo-toy.yaml -``` - The learner checkpoint contains -actor/V weights and specs, both optimizers/schedulers and step clocks, the -frozen codec, config fingerprint, best actor, seed/RNG state, live history and -episode counters. Checkpoints commit atomically only at complete iteration -boundaries. A crash during collection or optimization restarts from the last -committed iteration; partial buffers are not resumed. - -Set the phase's `resume.policy` to `never`, `if_exists`, or `required` and choose -one of these declared `resume.mode` values: - -- `exact` requires `supports_exact_snapshot = True` and working - `snapshot_state()` / `restore_state(state)` methods. The snapshot must include - simulator and simulator RNG state using dictionaries, lists, tuples, finite - scalars, numeric NumPy arrays and torch tensors. Restore is checked against a fresh snapshot. - Merely exposing `state_dict` does not declare this capability. Exact mode - rejects unsupported adapters before collection. It preserves the live - history and episode counters; equivalence still depends on deterministic - adapter and device operations. -- `reset_environments` restores learner, action sampler and optimizer state, - then resets the environment and causal history. It discards incomplete - episode metrics and records a non-trajectory-equivalent continuation. Reset - counts continue forward, so the next reset uses a new derived seed. - -Resume validates the full run contract, including budget, model, codec, reward -scale, adapter identity/config and normalization. Resume policy and mode may -change; other changes require a new run. Evaluation seeds never affect training -reset counts. Optional `archive_directory` writes atomic, version-1 -`sequifier.ppo-collection` records, with pre-fragment context, identities, -policy generation, cutoff reason, raw/scaled rewards and true successors. `archive_scores: true` -also retains old log-probability, entropy and values. The independent -`load_collection_record` reader does not change existing v1 JSON episode -readers. Archives are never fed back into online updates. - -The report records return/evaluation curves, policy/value losses, entropy, clip -fraction, approximate KL, explained variance, gradient norms, ratio tails, -optimizer clocks, counts and separate collection/update/evaluation timing. -Invalid data fails immediately; successful reports therefore have zero invalid -episodes. A nonterminated final training episode is explicitly censored. - -This scope is one in-process adapter slot and one fixed categorical action on -CPU or a supported local CUDA/MPS device. Vector/Gym/remote adapters, -continuous/joint actions, media/depth, distributed learning, reward providers, -mixed phases, same-run phase transfer, generalized monitors/search and ONNX -policy execution remain deferred. This does not complete the broader rollout -and reinforcement-learning plan. +`PolicyRunConfig.model_validate` accepts the same mapping directly. The single +`PolicyRunResult` exposes `policy`, ordered `phases`, `representation` and output +paths. `rl_api` also exports the phase schemas, preference records, online +capabilities, `PolicyWindowView`, and model loading/execution helpers. + +Use `sequifier infer` with the final artifact for categorical sequence inference +or rollout. `PolicyExecutor` and `collect_policy_rollout` build the same causal +histories; return-conditioned actors require an explicit desired return. +See [policy inference](configs/infer.md), [rollout](rollout.md), and the +[Python PPO workflow](example_inputs/ppo_toy_control.py). + +CPU determinism is the reference execution contract. CUDA and MPS reproducibility +is limited by the selected PyTorch backend/operators and matching device/RNG +topology. This implementation was statically checked; no tests or training runs +were written/executed for this change, so numerical and end-to-end acceptance +verification remains outstanding. diff --git a/src/sequifier/docs.py b/src/sequifier/docs.py index 514d0dfd..562bc487 100644 --- a/src/sequifier/docs.py +++ b/src/sequifier/docs.py @@ -6,7 +6,7 @@ "preprocess": "# Preprocess Command Guide", "train": "# Train Command Guide", "infer": "# Infer Command Guide", - "policy": "# Policy Learning Command Guide", + "policy": "# Categorical policy learning", } diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index c9c7137d..90dadcc3 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -26,6 +26,7 @@ _finite_mean_and_std, create_id_map, get_combined_statistics, + normalize_values, ) from sequifier.io.sequence import SequenceEpisode from sequifier.io.values import numeric_scalar @@ -249,6 +250,36 @@ def use_config(self, config: CategoricalPolicyConfig) -> None: for column, value in episode.static_inputs.items(): encode_feature(value, column, config) + def validate_normalization(self, statistics): + """Check FP32 transforms episode-by-episode without constructing windows.""" + for index, (episode, length) in enumerate(zip(self.episodes, self.lengths)): + for column in self.config.observation_columns + self.config.static_columns: + if column in self.config.categorical_n_classes: + continue + values = ( + episode.observations[column][:length] + if column in episode.observations + else [episode.static_inputs[column]] + ) + tensor = torch.tensor(values, dtype=torch.float32) + transformed = normalize_values(tensor, statistics[column]) + if ( + not torch.isfinite(tensor).all() + or not torch.isfinite(transformed).all() + ): + raise ValueError( + f"episode {episode.episode_id}, {column}: non-finite FP32 input/normalization" + ) + if self.config.return_conditioned: + _, returns = self._episode_labels(index) + if not torch.isfinite( + torch.tensor(returns, dtype=torch.float32) + / self.config.returns.scale + ).all(): + raise ValueError( + f"episode {episode.episode_id}: non-finite conditioned return" + ) + def _episode_labels(self, index: int): if index not in self._labels: episode = self.episodes[index] diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py index 727274eb..6f3c7380 100644 --- a/src/sequifier/io/policy_window.py +++ b/src/sequifier/io/policy_window.py @@ -41,9 +41,7 @@ def action_index(value, config: CategoricalPolicyConfig) -> int: def action_global_ids(config: CategoricalPolicyConfig) -> list[int]: - offset = ( - SPECIAL_TOKEN_IDS.user_start if config.categorical_encoding == "global" else 0 - ) + offset = SPECIAL_TOKEN_IDS.user_start return list(range(offset, offset + len(config.action_values))) diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py new file mode 100644 index 00000000..b1d15509 --- /dev/null +++ b/src/sequifier/io/preference_dataset.py @@ -0,0 +1,277 @@ +"""Lazy preference alternatives, connected-component splits and causal scoring.""" + +import hashlib +import random +from dataclasses import dataclass +from typing import cast + +import torch +from torch.nn import functional as F + +from sequifier.io.batch import SequifierInputs, collate_model_inputs +from sequifier.io.policy_dataset import PolicySequenceDataset, episode_content_key +from sequifier.io.policy_window import PolicyWindowView, action_index +from sequifier.rl.run_state import fingerprint +from sequifier.trajectory.preferences import load_preferences + + +@dataclass(frozen=True) +class PreferenceData: + training: "PreferenceDataset" + validation: "PreferenceDataset" + + @property + def contract(self): + return [self.training.contract, self.validation.contract] + + +class PreferenceDataset: + def __init__(self, pairs, corpora, config): + self.pairs, self.corpora, self.config = tuple(pairs), corpora, config + self.lookup = { + name: {episode: i for i, episode in enumerate(data.episode_ids)} + for name, data in corpora.items() + } + self.contract = [] + for pair in self.pairs: + entries = [] + for span in (pair.chosen, pair.rejected): + episode = self.episode(span) + if span.end > len(episode.actions): + raise ValueError( + f"pair {pair.pair_id}: span exceeds episode {span.episode_id}" + ) + entries.append( + {"span": span.model_dump(), "content": episode_content_key(episode)} + ) + if self.alternative_key(pair.chosen) == self.alternative_key(pair.rejected): + raise ValueError( + f"pair {pair.pair_id}: chosen/rejected alternatives have identical decision content" + ) + if ( + pair.chosen.source == pair.rejected.source + and pair.chosen.episode_id == pair.rejected.episode_id + and max(pair.chosen.start, pair.rejected.start) + < min(pair.chosen.end, pair.rejected.end) + ): + raise ValueError(f"pair {pair.pair_id}: chosen/rejected spans overlap") + self.contract.append({"pair": pair.model_dump(), "alternatives": entries}) + + def __len__(self): + return len(self.pairs) + + def episode(self, span): + try: + return self.corpora[span.source].episodes[ + self.lookup[span.source][span.episode_id] + ] + except KeyError as exc: + raise ValueError( + f"preference references unknown source/episode {span.source}:{span.episode_id}" + ) from exc + + def alternative_key(self, span): + episode = self.episode(span) + start = max(0, span.start + 1 - self.config.context_length) + return fingerprint( + { + "observations": { + k: v[start : span.end] for k, v in episode.observations.items() + }, + "actions": episode.actions[max(0, start - 1) : span.end], + "static": episode.static_inputs, + "starts_episode": episode.starts_episode, + "previous_action": episode.previous_action, + } + ) + + def components(self, pair): + keys = [("context", pair.context_id)] + if pair.group_id is not None: + keys.append(("group", pair.group_id)) + for span in (pair.chosen, pair.rejected): + episode = self.episode(span) + keys += [ + ("episode", span.episode_id), + ("content", episode_content_key(episode)), + ("alternative", self.alternative_key(span)), + ] + if episode.group_id is not None: + keys.append(("group", episode.group_id)) + return keys + + def fitting_dataset(self, cache_size): + refs = {} + for pair in self.pairs: + for span in (pair.chosen, pair.rejected): + index = self.lookup[span.source][span.episode_id] + refs[(span.source, span.episode_id)] = self.corpora[ + span.source + ].episodes.sources[index] + return PolicySequenceDataset( + list(refs.values()), self.config, episode_cache_size=cache_size + ) + + def use_config(self, config): + self.config = config + view = PolicyWindowView(config) + contexts = {} + for pair in self.pairs: + initial = [] + alternatives = [] + for span in (pair.chosen, pair.rejected): + episode = self.episode(span) + digest = hashlib.sha256() + # Validate every selected decision and all its causal inputs before training. + for step in range(span.start, span.end): + inputs = view.episode_inputs(episode, step, None) + action = action_index(episode.actions[step], config) + digest.update( + fingerprint( + { + "features": { + k: v.tolist() for k, v in inputs.inputs.items() + }, + "masks": { + k: v.tolist() for k, v in inputs.metadata.items() + }, + "action": action, + } + ).encode() + ) + if step == span.start: + initial.append(inputs) + alternatives.append(digest.hexdigest()) + if alternatives[0] == alternatives[1]: + raise ValueError( + f"pair {pair.pair_id}: alternatives have identical canonical executable decisions" + ) + context_key = fingerprint( + { + "features": {k: v.tolist() for k, v in initial[0].inputs.items()}, + "masks": {k: v.tolist() for k, v in initial[0].metadata.items()}, + } + ) + if pair.context_id in contexts and contexts[pair.context_id] != context_key: + raise ValueError( + f"context {pair.context_id}: ID refers to different canonical initial histories" + ) + contexts[pair.context_id] = context_key + left, right = initial + for mapping in ("inputs", "metadata"): + lmap, rmap = getattr(left, mapping), getattr(right, mapping) + if lmap.keys() != rmap.keys() or any( + not torch.equal(lmap[k], rmap[k]) for k in lmap + ): + raise ValueError( + f"pair {pair.pair_id}: initial {mapping} differ under frozen codec" + ) + + +def prepare_preferences(run, phase, config): + pairs = load_preferences(run.preferences[phase.preferences]).pairs + validation = ( + load_preferences(run.preferences[phase.validation_preferences]).pairs + if phase.validation_preferences + else None + ) + all_pairs = pairs + (validation or []) + ids = [pair.pair_id for pair in all_pairs] + if len(ids) != len(set(ids)): + raise ValueError("preference pair IDs overlap across manifests") + names = {span.source for pair in all_pairs for span in (pair.chosen, pair.rejected)} + if names - run.sources.keys(): + raise ValueError( + f"preferences reference unknown sources: {sorted(names - run.sources.keys())}" + ) + corpora = { + name: PolicySequenceDataset( + [run.sources[name]], config, episode_cache_size=run.episode_cache_size + ) + for name in names + } + data = PreferenceDataset(all_pairs, corpora, config) + if validation is not None: + left = {key for pair in pairs for key in data.components(pair)} + right = {key for pair in validation for key in data.components(pair)} + if left & right: + raise ValueError( + f"preference train/validation components overlap: {next(iter(left & right))}" + ) + else: + parents = list(range(len(pairs))) + + def root(i): + while parents[i] != i: + parents[i] = parents[parents[i]] + i = parents[i] + return i + + seen = {} + for i, pair in enumerate(pairs): + for key in data.components(pair): + if key in seen: + parents[root(i)] = root(seen[key]) + seen[key] = i + groups = {} + for i, pair in enumerate(pairs): + groups.setdefault(root(i), []).append(pair) + components = list(groups.values()) + if len(components) < 2: + raise ValueError( + "DPO splitting requires at least two independent preference components" + ) + random.Random(run.seed).shuffle(components) + count = min( + len(components) - 1, max(1, int(len(components) * run.validation_fraction)) + ) + validation = [pair for group in components[:count] for pair in group] + pairs = [pair for group in components[count:] for pair in group] + return PreferenceData( + PreferenceDataset(pairs, corpora, config), + PreferenceDataset(validation, corpora, config), + ) + + +def sequence_log_probabilities(actor, episode, start, end, chunk_size): + """One FP32 score per executable decision; context/padding never contribute.""" + actor.eval() + view = PolicyWindowView(actor.config) + for offset in range(start, end, chunk_size): + positions = range(offset, min(offset + chunk_size, end)) + inputs = cast( + SequifierInputs, + collate_model_inputs( + [view.episode_inputs(episode, t, None) for t in positions] + ), + ).to(actor.device) + actions = torch.tensor( + [action_index(episode.actions[t], actor.config) for t in positions], + dtype=torch.long, + device=actor.device, + ) + scores = actor.evaluate_actions(inputs, actions).float() + if not torch.isfinite(scores).all(): + raise ValueError("non-finite sequence action log likelihood") + yield scores + + +def sequence_score(actor, episode, start, end, chunk_size): + return torch.stack( + [ + scores.sum() + for scores in sequence_log_probabilities( + actor, episode, start, end, chunk_size + ) + ] + ).sum() + + +def dpo_loss(chosen, rejected, reference_chosen, reference_rejected, beta): + margin = (chosen.float() - rejected.float()) - ( + reference_chosen.float() - reference_rejected.float() + ) + loss = -F.logsigmoid(beta * margin) + if not torch.isfinite(margin).all() or not torch.isfinite(loss).all(): + raise ValueError("non-finite DPO margin/loss") + return loss, margin diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 4b4a5307..7f3d7dbd 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -118,26 +118,25 @@ output_dim: 8 feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} decoder: {type: linear, prediction_length: 1, support: 1} -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: {paths: [PLEASE_FILL.csv]} -global_training: - training_objective: policy - context_length: 4 +policy_interface: actor +context_length: 4 +sources: + demonstrations.recorded: + paths: [PLEASE_FILL.csv] +validation_fraction: 0.2 +episode_cache_size: 8 +resume: {policy: never} +phases: +- algorithm: policy_imitation + epochs: 80 + sources: [demonstrations.recorded] + name: policy_imitation batch_size: 64 - learning_rate: 0.005 - validation_fraction: 0.2 - episode_cache_size: 8 - resume: {policy: never} -training_plan: - phases: - - {algorithm: policy_imitation, policy: actor, epochs: 80, sources: [{source: demonstrations.recorded}]} + actor_learning_rate: 0.005 """ policy_infer_config_string = """project_root: . -model_path: models/policy/policy.pt +model_path: models/policy/actor.pt data_path: PLEASE_FILL model_type: generative read_format: csv @@ -174,33 +173,25 @@ output_dim: 8 feature_embedding_dims: {OBSERVATION_COLUMN: 2, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} -dataset_training: - demonstrations: - model_interface: actor - parts: - recorded: - paths: [PLEASE_FILL.csv] - columns: - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN} -global_training: - training_objective: policy - context_length: 16 +policy_interface: actor +context_length: 16 +sources: + demonstrations.recorded: + paths: [PLEASE_FILL.csv] + columns: + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN} +validation_fraction: 0.2 +resume: {policy: if_exists} +phases: +- name: offline_improvement + algorithm: iql + sources: [demonstrations.recorded] + updates: 10000 batch_size: 64 - learning_rate: 0.005 - validation_fraction: 0.2 - resume: {policy: if_exists} -training_plan: - phases: - - name: offline_improvement - algorithm: iql - policy: actor - sources: [{source: demonstrations.recorded}] - updates: 10000 - batch_size: 64 """ ppo_train_config_string = """project_root: . @@ -231,38 +222,137 @@ output_dim: 8 feature_embedding_dims: {cue: 2, __policy_previous_action: 6} decoder: {type: linear, prediction_length: 1, support: 1} -training_plan: - phases: - - name: online_control - algorithm: ppo - policy: actor - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - normalization: {source: identity} - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - actor_learning_rate: 0.0003 - value_learning_rate: 0.0003 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 - resume: {policy: if_exists, mode: exact} +policy_interface: actor +normalization: {source: identity} +resume: {policy: if_exists, mode: exact} +phases: +- name: online_control + algorithm: ppo + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + actor_learning_rate: 0.0003 + value_learning_rate: 0.0003 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 """ +dpo_train_config_string = """project_root: . +model_name: dpo-control +device: cpu +seed: 1010 +context_length: 4 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 1 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 64} + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: [cue] + target_columns: [action] + policy: + observation_columns: [cue] + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +policy_interface: actor +normalization: {source: identity} +phases: +- {name: preferences, algorithm: dpo, preferences: rankings, updates: 100, beta: 0.1} +sources: + decisions: + paths: [data/control-decisions.csv] + columns: + episode_id: episodeId + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {cue: next_cue} + group: contextId +preferences: {rankings: data/control-preferences.json} +""" + +grpo_train_config_string = """project_root: . +model_name: grpo-control +device: cpu +seed: 1010 +context_length: 4 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 1 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 64} + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: [cue] + target_columns: [action] + policy: + observation_columns: [cue] + action_values: [left, right] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +policy_interface: actor +normalization: {source: identity} +phases: +- name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_update: 2 + group_size: 4 + max_steps: 8 + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 + metric: score_mean +""" + gitignore_string = """models/ logs/ checkpoints/ @@ -306,3 +396,9 @@ def make(args): with open(f"{project_name}/configs/train-ppo.yaml", "w") as f: f.write(ppo_train_config_string) + + with open(f"{project_name}/configs/train-dpo.yaml", "w") as f: + f.write(dpo_train_config_string) + + with open(f"{project_name}/configs/train-grpo.yaml", "w") as f: + f.write(grpo_train_config_string) diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 01a3a4b8..d4d5b454 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -7,7 +7,8 @@ from sequifier.artifacts.model_artifact import ( ModelArtifact, build_model_artifact, - load_execution_model, + load_model_artifact, + restore_model_artifact, save_model_artifact, ) from sequifier.rl.policy import CategoricalPolicy @@ -15,6 +16,8 @@ def validate_policy_weights(policy: CategoricalPolicy) -> None: for name, value in policy.network.state_dict().items(): + if value.is_floating_point() and value.dtype != torch.float32: + raise ValueError(f"policy tensor {name!r} requires FP32 precision") if not torch.isfinite(value).all(): raise ValueError(f"policy tensor {name!r} must be finite") statistics = policy.interface.selected_columns_statistics @@ -29,7 +32,10 @@ def save_policy( build_model_artifact( policy.network, policy.execution_config, - provenance=metadata, + provenance={ + **(metadata or {}), + "policy_units": dict(getattr(policy, "units", {})), + }, state_dict={ name: value.detach().cpu() for name, value in policy.network.state_dict().items() @@ -45,9 +51,14 @@ def load_policy( device: str = "cpu", interface_name: str | None = None, ) -> CategoricalPolicy: - network, config, _ = load_execution_model( - path, device=device, interface_name=interface_name - ) + if isinstance(path, ModelArtifact): + network, config, artifact = restore_model_artifact( + path, device=device, interface_name=interface_name + ) + else: + network, config, artifact = load_model_artifact( + path, device=device, interface_name=interface_name + ) if interface_name is None: candidates = [ name @@ -59,6 +70,7 @@ def load_policy( interface_name = candidates[0] assert interface_name is not None policy = CategoricalPolicy.from_network(network, config, interface_name) + policy.units = dict(artifact.metadata.provenance.get("policy_units", {})) validate_policy_weights(policy) policy.eval() return policy diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py new file mode 100644 index 00000000..2146173b --- /dev/null +++ b/src/sequifier/rl/collection.py @@ -0,0 +1,60 @@ +"""Independent seed streams and categorical online decision scoring.""" + +import hashlib +from dataclasses import dataclass + +import torch + +from sequifier.rl.distributions import CategoricalActionDistribution + + +def stream_seed(seed, stream, reset_count): + value = f"sequifier-policy-v1:{seed}:{stream}:{reset_count}".encode() + return int.from_bytes(hashlib.blake2b(value, digest_size=8).digest(), "big") % ( + 2**63 + ) + + +@dataclass(frozen=True) +class PolicyDecision: + domain_action: dict + action: int + log_probability: float + entropy: float + value: float + + +def validate_online_inputs(actor, inputs): + inputs = inputs.to(actor.device) + actor.validate_inputs(inputs) + features, _ = actor.model_inputs(inputs) + if any( + tensor.is_floating_point() and not torch.isfinite(tensor).all() + for tensor in features.values() + ): + raise ValueError("frozen input transforms produced non-finite values") + return inputs + + +@torch.no_grad() +def decide(actor, value, inputs, generator, *, deterministic=False): + """Score action and history-only V from exactly the same causal input.""" + actor.eval() + if value is not None: + value.eval() + inputs = validate_online_inputs(actor, inputs) + distribution = CategoricalActionDistribution(actor(inputs)[:, -1]) + action = distribution.select(deterministic, generator) + estimate = ( + value(inputs) if value is not None else torch.zeros(1, device=actor.device) + ) + if not torch.isfinite(estimate).all(): + raise ValueError("non-finite collection value") + index = int(action[0]) + return PolicyDecision( + {actor.config.action_column: actor.config.action_values[index]}, + index, + float(distribution.log_prob(action)[0]), + float(distribution.entropy()[0]), + float(estimate[0]), + ) diff --git a/src/sequifier/rl/control.py b/src/sequifier/rl/control.py index b75f6d83..d1402b7b 100644 --- a/src/sequifier/rl/control.py +++ b/src/sequifier/rl/control.py @@ -7,13 +7,23 @@ import random -from sequifier.rl.environment import OnlineResetResult, OnlineStepResult +from sequifier.rl.environment import ( + EnvironmentCapabilities, + OnlineResetResult, + OnlineStepResult, +) from sequifier.rollout.contracts import StepResult class CategoricalControlAdapter: - protocol_version = 2 - supports_exact_snapshot = True + protocol_version = 3 + implementation_id = "sequifier.categorical-control.v1" + capabilities = EnvironmentCapabilities( + exact_snapshot=True, + group_start="scenario", + reproducible_group_start=True, + group_start_randomness="independent", + ) def __init__(self, horizon=8): if not isinstance(horizon, int) or isinstance(horizon, bool) or horizon <= 0: @@ -30,6 +40,16 @@ def reset(self, seed): self.ended = False return OnlineResetResult({"cue": self.cue}) + def create_group_start(self, seed): + return {"cue": random.Random(seed).choice((-1.0, 1.0)), "horizon": self.horizon} + + def reset_group(self, token, rollout_seed): + if token.get("horizon") != self.horizon or token.get("cue") not in {-1.0, 1.0}: + raise ValueError("incompatible control group-start token") + self.step_id, self.correct, self.ended = 0, 0, False + self.cue = token["cue"] + return OnlineResetResult({"cue": self.cue}) + def step(self, action): if ( self.ended diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py new file mode 100644 index 00000000..0592db59 --- /dev/null +++ b/src/sequifier/rl/dpo.py @@ -0,0 +1,149 @@ +"""Canonical sigmoid DPO with bounded differentiable sequence scoring.""" + +import torch + +from sequifier.config.policy_plan import DPOPhaseConfig +from sequifier.io.preference_dataset import ( + PreferenceData, + dpo_loss, + sequence_log_probabilities, + sequence_score, +) +from sequifier.rl.run_state import ( + PhaseContext, + Selection, + actor_state, + apply_gradients, + make_optimization, + restore_actor_state, + validate_permutation, +) + + +def pair_scores(actor, reference, dataset, pair, chunk_size): + scores = [] + for policy in (actor, reference): + for span in (pair.chosen, pair.rejected): + scores.append( + sequence_score( + policy, dataset.episode(span), span.start, span.end, chunk_size + ) + ) + return scores[0], scores[1], scores[2], scores[3] + + +@torch.no_grad() +def evaluate_dpo(actor, reference, dataset, phase): + totals = { + key: 0.0 + for key in ( + "dpo_loss", + "margin", + "preference_accuracy", + "chosen_length", + "rejected_length", + ) + } + for pair in dataset.pairs: + loss, margin = dpo_loss( + *pair_scores(actor, reference, dataset, pair, phase.scoring_chunk_size), + phase.beta, + ) + totals["dpo_loss"] += float(loss) + totals["margin"] += float(margin) + totals["preference_accuracy"] += float(margin > 0) + totals["chosen_length"] += pair.chosen.end - pair.chosen.start + totals["rejected_length"] += pair.rejected.end - pair.rejected.start + return {key: value / len(dataset) for key, value in totals.items()} + + +def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): + actor, phase, data, reference = ( + context.actor.eval(), + context.phase, + context.data, + context.reference, + ) + if reference is None: + raise ValueError("DPO requires a frozen reference actor") + optimizer, scheduler = make_optimization(actor, phase) + sampler = torch.Generator().manual_seed(context.run.seed + context.index) + restore_actor_state(context, optimizer, scheduler, sampler) + state = context.resume_state or {} + transaction, metrics = state.get("transaction", 0), list(state.get("metrics", [])) + if not 0 <= transaction <= phase.updates: + raise ValueError("DPO checkpoint exceeds authored update budget") + selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) + order, position = state.get("order", []), state.get("position", 0) + if state: + validate_permutation(order, len(data.training), position) + size = phase.batch_size * phase.accumulation_steps + cycle = (len(data.training) + size - 1) // size + expected = min(((transaction - 1) % cycle + 1) * size, len(data.training)) + if position != expected: + raise ValueError( + "DPO sampler position disagrees with optimizer transaction" + ) + for update in range(transaction + 1, phase.updates + 1): + if position == len(order): + order, position = ( + torch.randperm(len(data.training), generator=sampler).tolist(), + 0, + ) + group = order[position : position + phase.batch_size * phase.accumulation_steps] + optimizer.zero_grad(set_to_none=True) + loss_total = 0.0 + for index in group: + pair = data.training.pairs[index] + # The derivative of the scalar DPO margin is known. First score without + # graphs, then backpropagate each causal chunk with that exact derivative. + # This bounds activation memory even for arbitrarily long alternatives. + with torch.no_grad(): + scores = pair_scores( + actor, reference, data.training, pair, phase.scoring_chunk_size + ) + loss, margin = dpo_loss(*scores, phase.beta) + coefficient = ( + -phase.beta * torch.sigmoid(-phase.beta * margin) / len(group) + ) + loss_total += float(loss) / len(group) + for span, sign in ((pair.chosen, 1), (pair.rejected, -1)): + episode = data.training.episode(span) + for chunk in sequence_log_probabilities( + actor, episode, span.start, span.end, phase.scoring_chunk_size + ): + (sign * coefficient * chunk.sum()).backward() + apply_gradients(actor, optimizer, scheduler, phase.gradient_clip) + transaction, position = update, position + len(group) + final = update == phase.updates + if update % phase.evaluation_interval == 0 or final: + held_out = evaluate_dpo(actor, reference, data.validation, phase) + selection.observe(actor, held_out["dpo_loss"], update) + metrics.append( + { + "transaction": update, + "training_dpo_loss": loss_total, + **{f"validation_{key}": value for key, value in held_out.items()}, + } + ) + if update % phase.checkpoint_interval == 0 or final: + context.commit( + actor_state( + context, + optimizer, + scheduler, + sampler, + selection, + update, + metrics, + order=order, + position=position, + ) + ) + return selection.finish( + context, + transaction, + {"optimizer_steps": transaction}, + metrics, + {"data": data.contract, "reference_weight_hash": context.reference_identity}, + ) diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py index e11c455a..1b6f7dc5 100644 --- a/src/sequifier/rl/environment.py +++ b/src/sequifier/rl/environment.py @@ -1,8 +1,9 @@ -"""Version-2 online local adapter contract, independent of seeded v1 rollouts.""" +"""Explicit capabilities for local PPO/GRPO environments and seeded rollout bridges.""" +import json from dataclasses import dataclass, field from importlib import import_module -from typing import Mapping, Protocol +from typing import Literal, Mapping, Protocol from sequifier.io.values import DomainValue, numeric_scalar from sequifier.rollout.contracts import ResetRequest, StepRequest @@ -26,28 +27,62 @@ class OnlineStepResult: success: bool | None = None +@dataclass(frozen=True) +class EnvironmentCapabilities: + exact_snapshot: bool = False + group_start: Literal["snapshot", "scenario"] | None = None + reproducible_group_start: bool = False + # captured: the start token fixes simulator RNG state; reset_group must still + # use its rollout seed for independent future environment randomness. + group_start_randomness: Literal["captured", "independent"] = "independent" + + def __post_init__(self): + if not isinstance(self.exact_snapshot, bool) or not isinstance( + self.reproducible_group_start, bool + ): + raise ValueError("environment capability flags must be booleans") + if self.group_start not in { + None, + "snapshot", + "scenario", + } or self.group_start_randomness not in {"captured", "independent"}: + raise ValueError("unsupported group-start capability") + if self.reproducible_group_start and self.group_start is None: + raise ValueError( + "reproducible_group_start requires a declared group_start capability" + ) + + class OnlineEnvironmentAdapter(Protocol): protocol_version: int - supports_exact_snapshot: bool + implementation_id: str + capabilities: EnvironmentCapabilities def reset(self, seed: int) -> OnlineResetResult: ... def step(self, action: Mapping[str, DomainValue]) -> OnlineStepResult: ... def close(self) -> None: ... def snapshot_state(self) -> dict: ... def restore_state(self, state: dict) -> None: ... + def create_group_start(self, seed: int) -> object: ... + def reset_group(self, token: object, rollout_seed: int) -> OnlineResetResult: ... def load_online_adapter(spec): module, attribute = spec.factory.split(":") adapter = getattr(import_module(module), attribute)(**spec.config) try: - if ( - not isinstance(getattr(adapter, "protocol_version", None), int) - or adapter.protocol_version != 2 - ): + if getattr(adapter, "protocol_version", None) != 3: + raise ValueError( + "online adapters must explicitly declare protocol_version = 3" + ) + if getattr(adapter, "implementation_id", None) != spec.implementation_id: raise ValueError( - "online adapters must explicitly declare protocol_version = 2" + "environment.implementation_id differs from adapter implementation" ) + if not isinstance( + getattr(adapter, "capabilities", None), EnvironmentCapabilities + ): + raise ValueError("online adapter requires explicit EnvironmentCapabilities") for method in ("reset", "step", "close"): if not callable(getattr(adapter, method, None)): raise ValueError(f"online adapter requires {method}") @@ -60,18 +95,41 @@ def load_online_adapter(spec): def require_exact_snapshot(adapter): - if getattr(adapter, "supports_exact_snapshot", None) is not True: + if adapter.capabilities.exact_snapshot is not True: + raise ValueError("exact PPO resume requires capabilities.exact_snapshot") + for method in ("snapshot_state", "restore_state"): + if not callable(getattr(adapter, method, None)): + raise ValueError(f"exact snapshot adapter requires {method}") + + +def require_group_start(adapter, *, exact=False): + if adapter.capabilities.group_start not in {"snapshot", "scenario"}: raise ValueError( - "exact resume requires an explicit supports_exact_snapshot capability" + "GRPO requires explicit group_start: snapshot or scenario capability; a repeated seed is insufficient" ) - if not callable(getattr(adapter, "snapshot_state", None)) or not callable( - getattr(adapter, "restore_state", None) - ): + for method in ("create_group_start", "reset_group"): + if not callable(getattr(adapter, method, None)): + raise ValueError(f"group-start adapter requires {method}") + if exact and adapter.capabilities.reproducible_group_start is not True: raise ValueError( - "exact snapshot adapters require snapshot_state and restore_state" + "exact GRPO resume requires documented reproducible_group_start capability" ) +def validate_start_token(token): + try: + # Tokens are opaque to Sequifier, but must survive portable JSON round trips. + encoded = json.dumps(token, allow_nan=False, sort_keys=True) + decoded = json.loads(encoded) + if decoded != token: + raise ValueError("group-start token changes after JSON serialization") + return decoded + except (TypeError, ValueError) as exc: + raise ValueError( + "group-start token must be finite JSON-serializable content" + ) from exc + + def validate_reset(result, units): if not isinstance(result, OnlineResetResult) or dict(result.units) != units: raise ValueError( @@ -109,8 +167,9 @@ class SeededAdapterBridge: support is deliberately not inferred from the wrapped object. """ - protocol_version = 2 - supports_exact_snapshot = False + protocol_version = 3 + implementation_id = "sequifier.seeded-bridge.v1" + capabilities = EnvironmentCapabilities() def __init__( self, adapter, seed_factory, action_column, context_length=1, units=None @@ -179,10 +238,12 @@ def close(self): def adapter_identity(adapter): + from dataclasses import asdict + return { "module": type(adapter).__module__, "class": type(adapter).__qualname__, "protocol_version": adapter.protocol_version, - "supports_exact_snapshot": getattr(adapter, "supports_exact_snapshot", False) - is True, + "implementation_id": adapter.implementation_id, + "capabilities": asdict(adapter.capabilities), } diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py new file mode 100644 index 00000000..51203af5 --- /dev/null +++ b/src/sequifier/rl/grpo.py @@ -0,0 +1,193 @@ +"""Actor-only GRPO runtime; checkpoints contain no live or reusable group.""" + +import statistics +from copy import deepcopy +from pathlib import Path + +import torch + +from sequifier.config.policy_plan import GRPOPhaseConfig +from sequifier.rl.environment import ( + adapter_identity, + load_online_adapter, + require_group_start, +) +from sequifier.rl.grpo_collector import collect_groups +from sequifier.rl.grpo_update import update_grpo +from sequifier.rl.ppo_evaluation import evaluate_policy +from sequifier.rl.run_state import ( + PhaseContext, + Selection, + actor_state, + make_optimization, + require_equal, + restore_actor_state, + restore_rng_state, + write_json_atomic, +) + + +def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): + actor, phase = context.actor.eval(), context.phase + optimizer, scheduler = make_optimization(actor, phase) + sampler = torch.Generator().manual_seed(context.run.seed + context.index) + restore_actor_state(context, optimizer, scheduler, sampler) + state = context.resume_state or {} + transaction, metrics = state.get("transaction", 0), list(state.get("metrics", [])) + groups, env_steps, optimizer_steps = ( + state.get("groups", 0), + state.get("env_steps", 0), + state.get("optimizer_steps", 0), + ) + evaluations, continuations = ( + list(state.get("evaluations", [])), + list(state.get("continuations", [])), + ) + restart_stream = state.get("restart_stream", 0) + if ( + not 0 <= groups <= phase.total_groups + or transaction + != (groups + phase.groups_per_update - 1) // phase.groups_per_update + ): + raise ValueError("GRPO checkpoint is not a complete update/group boundary") + if groups != min(transaction * phase.groups_per_update, phase.total_groups): + raise ValueError("GRPO checkpoint group counter is not a complete iteration") + if ( + not transaction <= optimizer_steps <= phase.optimizer_step_budget() + or not groups * phase.group_size + <= env_steps + <= groups * phase.group_size * phase.max_steps + ): + raise ValueError( + "GRPO checkpoint optimizer/environment clocks exceed authored bounds" + ) + selection = Selection(phase, "score_mean", "max", state.get("selection")) + actor.grpo_policy_version = transaction + adapter = load_online_adapter(phase.environment) + try: + require_group_start(adapter, exact=context.run.resume.mode == "exact") + identity = adapter_identity(adapter) + if state: + require_equal(state["adapter_identity"], identity, "grpo.adapter_identity") + if context.run.resume.mode != "exact" and groups < phase.total_groups: + restart_stream += 1 + continuations.append( + { + "transaction": transaction, + "mode": "new_groups", + "restart_stream": restart_stream, + "trajectory_equivalent": False, + } + ) + + if state: + restore_rng_state(actor.device, state["random"]) + + def evaluate(): + result = evaluate_policy(actor, phase.evaluation, forbidden_adapter=adapter) + score = float( + torch.tensor( + result["score_mean"] * phase.reward_scale, dtype=torch.float32 + ) + ) + if not torch.isfinite(torch.tensor(score)): + raise ValueError("GRPO evaluation score exceeds finite FP32 range") + selection.observe(actor, score, transaction) + evaluations.append( + {**result, "score_mean": score, "transaction": transaction} + ) + + while groups < phase.total_groups: + count = min(phase.groups_per_update, phase.total_groups - groups) + buffer, executed = collect_groups( + context, adapter, groups, count, transaction, env_steps, restart_stream + ) + record = update_grpo( + context, + buffer, + optimizer, + scheduler, + sampler, + transaction, + optimizer_steps, + ) + optimizer_steps = record["optimizer_steps"] + env_steps, groups, transaction = ( + env_steps + executed, + groups + count, + transaction + 1, + ) + if phase.archive_directory: + write_json_atomic( + Path(phase.archive_directory) / f"groups-{transaction:08d}.json", + { + "format": "sequifier.grpo-archive", + "version": 1, + "reference_identity": context.reference_identity, + "generation": buffer.generation, + "trajectories": [t.as_dict() for t in buffer.trajectories], + }, + ) + if ( + transaction % phase.evaluation_interval == 0 + or groups == phase.total_groups + ): + evaluate() + scores = [t.score for t in buffer.trajectories] + record.update( + { + "transaction": transaction, + "groups": groups, + "env_steps": env_steps, + "policy_version": actor.grpo_policy_version, + "collected_policy_version": buffer.generation, + "score_mean": statistics.fmean(scores), + "score_std": statistics.pstdev(scores), + "zero_variance_groups": sum( + len({t.score for t in group}) == 1 for group in buffer.groups + ), + "lengths": [len(t.actions) for t in buffer.trajectories], + "cutoffs": [t.cutoff_reason for t in buffer.trajectories], + "start_tokens": [ + deepcopy(group[0].start_token) for group in buffer.groups + ], + "buffer_bytes": buffer.estimated_bytes, + } + ) + metrics.append(record) + if ( + transaction % phase.checkpoint_interval == 0 + or groups == phase.total_groups + ): + context.commit( + actor_state( + context, + optimizer, + scheduler, + sampler, + selection, + transaction, + metrics, + groups=groups, + env_steps=env_steps, + optimizer_steps=optimizer_steps, + adapter_identity=identity, + evaluations=evaluations, + continuations=continuations, + restart_stream=restart_stream, + ) + ) + finally: + adapter.close() + return selection.finish( + context, + transaction, + {"optimizer_steps": optimizer_steps, "groups": groups, "env_steps": env_steps}, + metrics, + { + "reference_weight_hash": context.reference_identity, + "adapter": identity, + "evaluations": evaluations, + "continuations": continuations, + }, + ) diff --git a/src/sequifier/rl/grpo_buffer.py b/src/sequifier/rl/grpo_buffer.py new file mode 100644 index 00000000..af342914 --- /dev/null +++ b/src/sequifier/rl/grpo_buffer.py @@ -0,0 +1,112 @@ +"""Bounded complete groups with reconstructable causal frames and fixed scores.""" + +from dataclasses import dataclass +from typing import cast + +import torch + +from sequifier.io.batch import SequifierInputs, collate_model_inputs +from sequifier.rl.history import history_inputs + + +@dataclass(frozen=True) +class GRPOTrajectory: + group_index: int + sample_index: int + start_token: object + episode_id: str + frames: tuple + context_length: int + actions: tuple[int, ...] + rewards: tuple[float, ...] + old_log_probabilities: tuple[float, ...] + reference_log_probabilities: tuple[float, ...] + cutoff_reason: str + score: float + advantage: float + generation: int + + def inputs(self, spec, step): + return history_inputs( + spec, self.frames[max(0, step + 1 - spec.context_length) : step + 1] + ) + + def batches(self, actor, chunk_size): + for start in range(0, len(self.actions), chunk_size): + end = min(start + chunk_size, len(self.actions)) + yield ( + cast( + SequifierInputs, + collate_model_inputs( + [self.inputs(actor.config, t) for t in range(start, end)] + ), + ).to(actor.device), + torch.tensor( + self.actions[start:end], dtype=torch.long, device=actor.device + ), + torch.tensor( + self.old_log_probabilities[start:end], + dtype=torch.float32, + device=actor.device, + ), + torch.tensor( + self.reference_log_probabilities[start:end], + dtype=torch.float32, + device=actor.device, + ), + ) + + def as_dict(self): + return { + **self.__dict__, + "frames": [frame.state_dict() for frame in self.frames], + "decision_mask": [True] * len(self.actions), + "history_lengths": [ + min(i + 1, self.context_length) for i in range(len(self.actions)) + ], + } + + +class SealedGroupBuffer: + def __init__( + self, groups, generation, group_size, reference_identity, estimated_bytes + ): + if not groups or any(len(group) != group_size for group in groups): + raise ValueError("GRPO requires complete groups") + self.groups, self.generation = tuple(tuple(g) for g in groups), generation + self.reference_identity, self.estimated_bytes = ( + reference_identity, + estimated_bytes, + ) + self.trajectories = tuple( + trajectory for group in self.groups for trajectory in group + ) + self.consumed = False + if any(t.generation != generation or not t.actions for t in self.trajectories): + raise ValueError("GRPO group buffer has stale/empty trajectories") + + indices = [group[0].group_index for group in self.groups] + if len(set(indices)) != len(indices): + raise ValueError("GRPO buffer repeats a group identity") + for group in self.groups: + if {t.sample_index for t in group} != set(range(group_size)) or any( + t.group_index != group[0].group_index + or t.start_token != group[0].start_token + for t in group + ): + raise ValueError("GRPO group has inconsistent start/sample identities") + + def require_fresh(self, generation, reference_identity): + if ( + self.consumed + or generation != self.generation + or reference_identity != self.reference_identity + ): + raise ValueError( + "GRPO buffer is consumed, stale or has a different reference" + ) + + def consume(self): + if self.consumed: + raise ValueError("GRPO buffer has already been consumed") + self.consumed = True diff --git a/src/sequifier/rl/grpo_collector.py b/src/sequifier/rl/grpo_collector.py new file mode 100644 index 00000000..f1150996 --- /dev/null +++ b/src/sequifier/rl/grpo_collector.py @@ -0,0 +1,166 @@ +"""Sequential same-start groups with independent action/environment RNG streams.""" + +import json +from dataclasses import replace + +import torch + +from sequifier.rl.collection import decide, stream_seed, validate_online_inputs +from sequifier.rl.distributions import action_generator +from sequifier.rl.environment import ( + require_group_start, + validate_reset, + validate_start_token, + validate_step, +) +from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer +from sequifier.rl.history import PolicyHistory, frame_storage_bytes +from sequifier.rl.run_state import fingerprint + + +class GroupCollectionError(RuntimeError): + def __init__(self, message, executed_actions): + super().__init__( + f"{message}; discarded {executed_actions} uncommitted environment actions" + ) + self.executed_actions = executed_actions + + +@torch.no_grad() +def collect_groups( + context, adapter, start_group, count, generation, env_steps, restart_stream=0 +): + actor, reference, phase = context.actor.eval(), context.reference, context.phase + require_group_start(adapter) + bound = count * phase.group_size * phase.max_steps + if phase.max_env_steps is not None and env_steps + bound > phase.max_env_steps: + raise ValueError( + "GRPO cannot fit the requested complete groups within max_env_steps (conservative group_size * max_steps bound)" + ) + estimate = bound * (2048 + 128 * len(actor.interface.input_columns)) + if estimate > phase.max_buffer_bytes: + raise ValueError( + f"GRPO complete groups require approximately {estimate} buffer bytes" + ) + groups, executed, used = [], 0, 0 + seed = stream_seed( + context.run.seed, f"grpo-phase-{context.index}-restart-{restart_stream}", 0 + ) + try: + for group_index in range(start_group, start_group + count): + token = validate_start_token( + adapter.create_group_start( + stream_seed(seed, "group-start", group_index) + ) + ) + used += len(json.dumps(token).encode()) * phase.group_size + 1024 + trajectories, initial_key = [], None + for sample in range(phase.group_size): + # All samples restore the same opaque condition, then independently seed future randomness. + result = validate_reset( + adapter.reset_group( + validate_start_token(token), + stream_seed(seed, f"group-{group_index}-environment", sample), + ), + context.run.units, + ) + history = PolicyHistory(actor.config, strict_vocabulary=True) + episode_id = f"group-{group_index}-sample-{sample}" + history.reset(result.observation, result.static_inputs, episode_id) + inputs = validate_online_inputs(actor, history.inputs()) + key = fingerprint( + { + "features": {k: v.tolist() for k, v in inputs.inputs.items()}, + "metadata": {k: v.tolist() for k, v in inputs.metadata.items()}, + "units": dict(result.units), + } + ) + if initial_key is not None and key != initial_key: + raise ValueError( + "GRPO samples have different canonical initial histories or units" + ) + initial_key = key + generator = action_generator( + actor.device, + stream_seed(seed, f"group-{group_index}-actions", sample), + ) + frames, actions, rewards, old_scores, ref_scores = [], [], [], [], [] + reason = "max_steps" + for step in range(phase.max_steps): + frame = history.current + used += frame_storage_bytes(frame) + 1024 + if used > phase.max_buffer_bytes: + raise ValueError("GRPO group exceeded max_buffer_bytes") + inputs = validate_online_inputs(actor, history.inputs()) + decision = decide(actor, None, inputs, generator) + reference_score = float( + reference.evaluate_actions( + inputs, + torch.tensor( + [decision.action], dtype=torch.long, device=actor.device + ), + )[0] + ) + executed += 1 + raw = adapter.step(decision.domain_action) + step_result = validate_step(raw, context.run.units) + reward = float(step_result.reward) * phase.reward_scale + if not torch.isfinite( + torch.tensor( + [reward, reference_score, decision.log_probability], + dtype=torch.float32, + ) + ).all(): + raise ValueError("non-finite GRPO reward or fixed action score") + frames.append(frame) + actions.append(decision.action) + rewards.append(reward) + old_scores.append(decision.log_probability) + ref_scores.append(reference_score) + if step_result.observation is not None: + history.advance(decision.action, step_result.observation) + validate_online_inputs(actor, history.inputs()) + if step_result.terminated or step_result.truncated: + reason = "terminated" if step_result.terminated else "truncated" + break + score = float(torch.tensor(rewards, dtype=torch.float32).sum()) + if not torch.isfinite(torch.tensor(score)): + raise ValueError("non-finite GRPO trajectory score") + trajectories.append( + GRPOTrajectory( + group_index, + sample, + token, + episode_id, + tuple(frames), + actor.config.context_length, + tuple(actions), + tuple(rewards), + tuple(old_scores), + tuple(ref_scores), + reason, + score, + 0.0, + generation, + ) + ) + scores = torch.tensor([t.score for t in trajectories], dtype=torch.float32) + mean, spread = scores.mean(), scores.std(unbiased=False) + if not torch.isfinite(mean) or not torch.isfinite(spread): + raise ValueError( + "non-finite FP32 GRPO group mean or population standard deviation" + ) + advantages = (scores - mean) / (spread + phase.advantage_epsilon) + if not torch.isfinite(advantages).all(): + raise ValueError("non-finite GRPO group advantages") + groups.append( + tuple( + replace(t, advantage=float(a)) + for t, a in zip(trajectories, advantages) + ) + ) + return SealedGroupBuffer( + groups, generation, phase.group_size, context.reference_identity, used + ), executed + except Exception as exc: + raise GroupCollectionError(str(exc), executed) from exc diff --git a/src/sequifier/rl/grpo_update.py b/src/sequifier/rl/grpo_update.py new file mode 100644 index 00000000..e08b5ca1 --- /dev/null +++ b/src/sequifier/rl/grpo_update.py @@ -0,0 +1,126 @@ +"""Trajectory-weighted clipped GRPO with a sampled frozen-reference penalty.""" + +import torch + +from sequifier.rl.run_state import apply_gradients + + +def grpo_objective( + log_probability, old_log_probability, reference_log_probability, advantage, phase +): + log_ratio = log_probability.float() - old_log_probability.float() + ratio = log_ratio.exp() + reference_delta = reference_log_probability.float() - log_probability.float() + penalty = torch.expm1(reference_delta) - reference_delta + clipped = ratio.clamp(1 - phase.clip_epsilon, 1 + phase.clip_epsilon) + loss = ( + -torch.minimum(ratio * advantage, clipped * advantage) + + phase.kl_coefficient * penalty + ) + old_kl = ratio - 1 - log_ratio + if any(not torch.isfinite(t).all() for t in (ratio, penalty, loss, old_kl)): + raise ValueError( + "non-finite GRPO ratio, sampled reference penalty or objective" + ) + return loss, penalty, old_kl, ((ratio - 1).abs() > phase.clip_epsilon).float() + + +@torch.no_grad() +def validate_old_scores(actor, buffer, chunk_size): + for trajectory in buffer.trajectories: + for inputs, actions, old, _ in trajectory.batches(actor, chunk_size): + if not torch.allclose( + actor.evaluate_actions(inputs, actions).float(), + old, + atol=1e-5, + rtol=1e-4, + ): + raise ValueError( + "GRPO reconstructed old-policy scores differ before first update" + ) + + +def update_grpo( + context, buffer, optimizer, scheduler, sampler, generation, optimizer_steps +): + actor, phase = context.actor.eval(), context.phase + buffer.require_fresh(generation, context.reference_identity) + if actor.grpo_policy_version != generation: + raise ValueError("GRPO collection generation differs from actor") + validate_old_scores(actor, buffer, phase.scoring_chunk_size) + trajectories = buffer.trajectories + totals = { + key: 0.0 + for key in ( + "loss", + "sampled_reference_penalty", + "old_policy_approximate_kl", + "clip_fraction", + ) + } + scored, stopped = 0, False + size = phase.batch_size * phase.accumulation_steps + epoch = -1 + for epoch in range(phase.optimization_epochs): + order = torch.randperm(len(trajectories), generator=sampler).tolist() + for start in range(0, len(order), size): + indices = order[start : start + size] + if phase.old_policy_target_kl is not None: + old_kl = 0.0 + with torch.no_grad(): + for i in indices: + trajectory = trajectories[i] + for inputs, actions, old, ref in trajectory.batches( + actor, phase.scoring_chunk_size + ): + _, _, kl, _ = grpo_objective( + actor.evaluate_actions(inputs, actions), + old, + ref, + trajectory.advantage, + phase, + ) + old_kl += ( + float(kl.sum()) / len(trajectory.actions) / len(indices) + ) + if old_kl > phase.old_policy_target_kl: + stopped = True + break + optimizer.zero_grad(set_to_none=True) + for offset in range(0, len(indices), phase.batch_size): + # Whole trajectories belong to a minibatch; causal windows are + # chunked within each trajectory without changing its weight. + for i in indices[offset : offset + phase.batch_size]: + trajectory = trajectories[i] + for inputs, actions, old, ref in trajectory.batches( + actor, phase.scoring_chunk_size + ): + values = grpo_objective( + actor.evaluate_actions(inputs, actions), + old, + ref, + trajectory.advantage, + phase, + ) + ( + values[0].sum() / len(trajectory.actions) / len(indices) + ).backward() + for key, value in zip(totals, values): + totals[key] += float(value.detach().sum()) / len( + trajectory.actions + ) + scored += 1 + apply_gradients(actor, optimizer, scheduler, phase.gradient_clip) + optimizer_steps += 1 + if stopped: + break + if not scored: + raise ValueError("GRPO old-policy KL stopped all optimizer transactions") + buffer.consume() + actor.grpo_policy_version = generation + 1 + return { + **{key: value / scored for key, value in totals.items()}, + "optimizer_steps": optimizer_steps, + "optimization_epochs": epoch + 1, + "old_policy_kl_stopped": stopped, + } diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py index d9f3ae76..1076965e 100644 --- a/src/sequifier/rl/history.py +++ b/src/sequifier/rl/history.py @@ -1,5 +1,6 @@ """Bounded causal state shared by online collection and policy execution.""" +import json from collections import deque from dataclasses import dataclass @@ -38,6 +39,13 @@ def from_state_dict(cls, values): ) +def frame_storage_bytes(frame): + """Conservative host-object allowance plus the complete domain payload.""" + return 1024 + 4 * len( + json.dumps(frame.state_dict(), allow_nan=False).encode("utf-8") + ) + + def history_inputs(spec, frames): frames = list(frames) return PolicyWindowView(spec).inputs( diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 575a5318..4465675b 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -1,33 +1,25 @@ -"""Fixed-dataset categorical Implicit Q-Learning.""" +"""Fixed-dataset categorical IQL as a phase-local actor/value/twin-Q runtime.""" -import json import math -from collections.abc import Mapping -from dataclasses import dataclass -from pathlib import Path from typing import cast import torch -from sequifier.artifacts.run_checkpoint import checkpoint_path +from sequifier.config.policy_plan import IQLPhaseConfig from sequifier.io.iql_dataset import IQLBatch, IQLTransitionDataset -from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.io.tensors import collate_tensor_records -from sequifier.model.factory import build_transformer_network -from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class -from sequifier.rl.checkpoint import load_policy, save_policy from sequifier.rl.models import make_iql_models -from sequifier.rl.policy import CategoricalPolicy, policy_execution_config -from sequifier.runtime.random_state import RandomState, RandomStateManager - - -@dataclass(frozen=True) -class IQLTrainingResult: - policy: CategoricalPolicy - metrics: tuple[dict[str, float], ...] - updates: int - training_episode_ids: tuple[str, ...] - validation_episode_ids: tuple[str, ...] +from sequifier.rl.offline_data import OfflineData +from sequifier.rl.run_state import ( + PhaseContext, + Selection, + actor_state, + apply_gradients, + clone_weights, + make_optimization, + restore_actor_state, + validate_finite_state, +) def _select(inputs, mask): @@ -37,29 +29,6 @@ def _select(inputs, mask): ) -def _optimizer(module, spec, learning_rate): - if "lr" in spec.arguments: - raise ValueError("IQL optimizer learning rate belongs in the phase field") - return get_optimizer_class(spec.name)( - [parameter for parameter in module.parameters() if parameter.requires_grad], - lr=learning_rate, - **spec.arguments, - ) - - -def _step(loss, optimizer, parameters, clip): - if not torch.isfinite(loss): - raise ValueError("non-finite IQL loss") - optimizer.zero_grad(set_to_none=True) - loss.backward() - norm = torch.nn.utils.clip_grad_norm_(parameters, clip, error_if_nonfinite=True) - optimizer.step() - if any(not torch.isfinite(parameter).all() for parameter in parameters): - raise ValueError("IQL optimizer produced non-finite parameters") - optimizer.zero_grad(set_to_none=True) - return float(norm.detach()) - - def _polyak(target, source, rate): with torch.no_grad(): for target_parameter, parameter in zip( @@ -70,74 +39,6 @@ def _polyak(target, source, rate): target_buffer.copy_(buffer) -def _checkpoint_payload( - config, - data, - fitted, - statistics, - actor, - value, - q1, - q2, - targets, - optimizers, - schedulers, - sampler, - metrics, - update, - best_loss, - best_actor, - random, -): - return { - "artifact_type": "sequifier_iql_run", - "format_version": 1, - "config": _resume_contract(config), - "data_contract": [part.data_contract for part in data], - "fitted_policy": fitted.model_dump(mode="json"), - "statistics": statistics, - "update": update, - "best_loss": best_loss, - "best_actor": best_actor, - "metrics": metrics, - "networks": { - name: module.state_dict() - for name, module in ( - ("actor", actor.network), - ("value", value), - ("q1", q1), - ("q2", q2), - ("target_q1", targets[0]), - ("target_q2", targets[1]), - ) - }, - "optimizers": { - name: optimizer.state_dict() for name, optimizer in optimizers.items() - }, - "schedulers": { - name: scheduler.state_dict() for name, scheduler in schedulers.items() - }, - "sampler": sampler.get_state(), - "random": random.capture_local().state_dict(), - } - - -def _resume_contract(config): - values = config.model_dump(mode="json") - values["global_training"].pop("resume", None) - return values - - -def _save_checkpoint(path, payload): - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.tmp") - try: - torch.save(payload, temporary) - temporary.replace(path) - finally: - temporary.unlink(missing_ok=True) - - def _evaluate(actor, value, q1, q2, dataset, batch_size, gamma, device): totals = {"action_nll": 0.0, "bellman_residual": 0.0, "value": 0.0, "q_min": 0.0} with torch.no_grad(): @@ -166,221 +67,56 @@ def _evaluate(actor, value, q1, q2, dataset, batch_size, gamma, device): ) totals["value"] += float(v.sum()) totals["q_min"] += float(q.sum()) - return {key: value / len(dataset) for key, value in totals.items()} - - -def run_iql_training(config) -> IQLTrainingResult: - phase = config.training_plan.phases[0] - spec = config.policy - if spec.return_conditioned: - raise ValueError("IQL requires an unconditioned policy") - if config.global_training.distributed or config.global_training.data_parallelism: - raise ValueError("IQL currently requires local execution") - if config.global_training.layer_autocast or any( - dtype != "float32" - for dtype in (config.global_training.layer_type_dtypes or {}).values() - ): - raise ValueError("IQL requires float32 layers without autocast") - if config.global_training.torch_compile != "none": - raise ValueError("IQL requires torch_compile: none") - if config.global_training.accumulation_steps not in (None, 1): - raise ValueError("IQL does not support gradient accumulation") - if config.global_training.num_workers: - raise ValueError( - "IQL samples transitions in the main process; num_workers must be zero" - ) - for role in ("actor", "value", "critic"): - optimizer = getattr(phase, f"{role}_optimizer") - if optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"}: - raise ValueError( - f"IQL {role} optimizer must be Adam, AdamW, SGD, or RMSprop" - ) - if "lr" in optimizer.arguments: - raise ValueError("IQL optimizer learning rates belong in phase fields") - scheduler = getattr(phase, f"{role}_scheduler") - if scheduler.name == "ReduceLROnPlateau": - raise ValueError("IQL schedulers advance on optimizer updates, not metrics") - checkpoint = checkpoint_path(config) - resume = config.global_training.resume - if resume is not None and resume.policy == "required" and not checkpoint.is_file(): - raise FileNotFoundError(checkpoint) - will_resume = ( - resume is not None and resume.policy != "never" and checkpoint.is_file() - ) - if ( - config.initialize_from is not None - and not will_resume - and not Path(config.initialize_from).is_file() - ): - raise FileNotFoundError(config.initialize_from) - source_names = phase.trajectory_sources + (phase.validation_sources or []) - from sequifier.trajectory.importers import source_paths - - sources = {name: source_paths(config.sources[name]) for name in source_names} - paths = [path for values in sources.values() for path in values] - protected = set(paths) - for name in source_names: - source = config.sources[name] - protected.update( - Path(value).resolve() - for value in ( - source.manifest, - source.metadata_config_path, - source.static_data_path, - ) - if value is not None - ) - if config.initialize_from is not None: - protected.add(Path(config.initialize_from).resolve()) - outputs = {Path(config.model_path), Path(config.report_path), checkpoint} - if len(outputs) != 3 or outputs.intersection(protected): - raise ValueError("IQL outputs must be distinct from trajectory inputs") - if len(paths) != len(set(paths)): - raise ValueError("IQL trajectory sources must not overlap") + result = {key: value / len(dataset) for key, value in totals.items()} + if not all(math.isfinite(value) for value in result.values()): + raise ValueError("non-finite IQL held-out likelihood or critic diagnostics") + return result - def corpus(names): - return PolicySequenceDataset( - [config.sources[name] for name in names], - spec, - episode_cache_size=config.global_training.episode_cache_size, - ) - training = corpus(phase.trajectory_sources) - if phase.validation_sources is None: - training, validation = split_by_episode( - training, - config.global_training.validation_fraction, - config.seed, - group_identical=True, - ) - else: - validation = corpus(phase.validation_sources) - if not len(training) or not len(validation): - raise ValueError("IQL needs nonempty train and validation partitions") - for attribute in ("episode_ids", "group_ids", "demonstration_keys"): - if (set(getattr(training, attribute)) & set(getattr(validation, attribute))) - { - None - }: - raise ValueError(f"IQL {attribute} overlaps training and validation") - fitted = training.fit_categorical_config() - for part in (training, validation): - part.use_config(fitted) - data = (IQLTransitionDataset(training), IQLTransitionDataset(validation)) - statistics = training.normalization_statistics() - execution = policy_execution_config( - fitted, - config.model, - config.device, - global_training=config.global_training, - statistics=statistics, - project_root=config.project_root, - model_name=config.model_name, - seed=config.seed, - ) - device = torch.device(config.device) - torch.manual_seed(config.seed) - actor = CategoricalPolicy.from_network( - build_transformer_network(execution, device=device).network, - execution, - fitted.interface_name, +def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): + actor, phase = context.actor.eval(), context.phase + data = tuple( + IQLTransitionDataset(part) + for part in (context.data.training, context.data.validation) ) value, q1, q2, targets = make_iql_models( - execution, fitted.interface_name, len(fitted.action_values) + actor.execution_config, actor.interface_name, len(actor.config.action_values) ) - if config.initialize_from is not None and not will_resume: - source = load_policy(config.initialize_from, device=config.device) - if ( - source.config != fitted - or source.interface.selected_columns_statistics != statistics - or source.execution_config.model != execution.model - ): - raise ValueError( - "IQL warm-start policy schema, normalization, or model architecture differs" - ) - actor.network.load_state_dict(source.network.state_dict(), strict=True) - modules = { - "actor": actor.network, - "value": value, - "critic": torch.nn.ModuleList([q1, q2]), - } - trainable_ids = [ - id(parameter) - for module in modules.values() - for parameter in module.parameters() - if parameter.requires_grad - ] - if len(trainable_ids) != len(set(trainable_ids)): - raise ValueError("IQL optimizer parameter assignments overlap") - optimizers = { - role: _optimizer( - module, - getattr(phase, f"{role}_optimizer"), - getattr(phase, f"{role}_learning_rate"), - ) - for role, module in modules.items() - } - schedulers = { - role: get_scheduler_class(getattr(phase, f"{role}_scheduler").name)( - optimizer, **getattr(phase, f"{role}_scheduler").arguments - ) - for role, optimizer in optimizers.items() - } - for module in (actor, value, q1, q2, *targets): + for module in (value, q1, q2, *targets): module.eval() - sampler = torch.Generator(device="cpu").manual_seed(config.seed) - random = RandomStateManager(device) - metrics = [] - update_start = 0 - best_loss = math.inf - best_actor = None - if resume is not None and resume.policy != "never": - if checkpoint.exists(): - payload = torch.load(checkpoint, map_location="cpu", weights_only=False) - if not isinstance(payload, Mapping): - raise ValueError("IQL checkpoint must be a mapping") - if ( - payload.get("artifact_type") != "sequifier_iql_run" - or payload.get("format_version") != 1 - ): - raise ValueError("IQL resume requires an IQL learner checkpoint") - if ( - payload["config"] != _resume_contract(config) - or payload["data_contract"] != [part.data_contract for part in data] - or payload["fitted_policy"] != fitted.model_dump(mode="json") - or payload["statistics"] != statistics - ): - raise ValueError("IQL resume config, data, or codec differs") - for name, module in ( - ("actor", actor.network), - ("value", value), - ("q1", q1), - ("q2", q2), - ("target_q1", targets[0]), - ("target_q2", targets[1]), - ): - module.load_state_dict(payload["networks"][name], strict=True) - for name in optimizers: - optimizers[name].load_state_dict(payload["optimizers"][name]) - schedulers[name].load_state_dict(payload["schedulers"][name]) - sampler.set_state(payload["sampler"]) - random.restore(RandomState.from_state_dict(payload["random"])) - metrics = list(payload["metrics"]) - update_start = int(payload["update"]) - if not 0 <= update_start <= phase.updates: - raise ValueError( - "IQL checkpoint update is outside the configured budget" - ) - best_loss = float(payload["best_loss"]) - best_actor = payload["best_actor"] - if not math.isfinite(best_loss) and best_actor is not None: - raise ValueError("IQL checkpoint has an invalid best actor score") - - for update in range(update_start + 1, phase.updates + 1): + modules = {"actor": actor, "value": value, "critic": torch.nn.ModuleList([q1, q2])} + plans = { + role: make_optimization(module, phase, role) for role, module in modules.items() + } + optimizers = {role: plan[0] for role, plan in plans.items()} + schedulers = {role: plan[1] for role, plan in plans.items()} + sampler = torch.Generator().manual_seed(context.run.seed + context.index) + state = context.resume_state or {} + if state: + for key in ("networks", "optimizers", "schedulers"): + validate_finite_state(state[key], f"iql.{key}") + for name, module in ( + ("value", value), + ("q1", q1), + ("q2", q2), + ("target_q1", targets[0]), + ("target_q2", targets[1]), + ): + module.load_state_dict(state["networks"][name], strict=True) + for role in ("value", "critic"): + optimizers[role].load_state_dict(state["optimizers"][role]) + schedulers[role].load_state_dict(state["schedulers"][role]) + restore_actor_state(context, optimizers["actor"], schedulers["actor"], sampler) + selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) + metrics = list(state.get("metrics", [])) + transaction = state.get("transaction", 0) + if not 0 <= transaction <= phase.updates: + raise ValueError("IQL checkpoint exceeds phase update budget") + for update in range(transaction + 1, phase.updates + 1): indices = torch.randint(len(data[0]), (phase.batch_size,), generator=sampler) batch = cast( - IQLBatch, - collate_tensor_records([data[0][int(index)] for index in indices]), - ).to(device) + IQLBatch, collate_tensor_records([data[0][int(i)] for i in indices]) + ).to(actor.device) with torch.no_grad(): qbar = torch.minimum( targets[0](batch.current, batch.action), @@ -403,130 +139,93 @@ def corpus(names): v_loss = ( torch.where(delta < 0, 1 - phase.expectile, phase.expectile) * delta.square() - ) - v_loss = v_loss.mean() - v_norm = _step( - v_loss, optimizers["value"], list(value.parameters()), phase.gradient_clip - ) - schedulers["value"].step() - q1_value, q2_value = ( - q1(batch.current, batch.action), - q2(batch.current, batch.action), - ) - q_loss = ((q1_value - target).square() + (q2_value - target).square()).mean() - q_norm = _step( - q_loss, - optimizers["critic"], - list(q1.parameters()) + list(q2.parameters()), - phase.gradient_clip, - ) - schedulers["critic"].step() - _polyak(targets[0], q1, phase.target_update_rate) - _polyak(targets[1], q2, phase.target_update_rate) + ).mean() + q_loss = ( + (q1(batch.current, batch.action) - target).square() + + (q2(batch.current, batch.action) - target).square() + ).mean() actor_loss = -( weight * actor.evaluate_actions(batch.current, batch.action) ).mean() - actor_norm = _step( - actor_loss, - optimizers["actor"], - list(actor.parameters()), - phase.gradient_clip, - ) - schedulers["actor"].step() + losses = {"value": v_loss, "critic": q_loss, "actor": actor_loss} + # Validate every objective and every gradient before mutating any network. + for role, loss in losses.items(): + if not torch.isfinite(loss): + raise ValueError(f"non-finite IQL {role} objective") + optimizers[role].zero_grad(set_to_none=True) + loss.backward() + for module in modules.values(): + torch.nn.utils.clip_grad_norm_( + module.parameters(), phase.gradient_clip, error_if_nonfinite=True + ) + for role, module in modules.items(): + apply_gradients( + module, optimizers[role], schedulers[role], phase.gradient_clip + ) + _polyak(targets[0], q1, phase.target_update_rate) + _polyak(targets[1], q2, phase.target_update_rate) + transaction = update if update % phase.evaluation_interval == 0 or update == phase.updates: held_out = _evaluate( - actor, value, q1, q2, data[1], phase.batch_size, phase.gamma, device + actor, + value, + q1, + q2, + data[1], + phase.batch_size, + phase.gamma, + actor.device, ) - record = { - "update": update, - "value_loss": float(v_loss.detach()), - "critic_loss": float(q_loss.detach()), - "actor_loss": float(actor_loss.detach()), - "advantage_mean": float(advantage.mean()), - "weight_mean": float(weight.mean()), - "weight_cap_fraction": float( - (weight >= phase.max_weight).float().mean() - ), - "advantage_p10": float(torch.quantile(advantage, 0.1)), - "advantage_p50": float(torch.quantile(advantage, 0.5)), - "advantage_p90": float(torch.quantile(advantage, 0.9)), - "weight_p10": float(torch.quantile(weight, 0.1)), - "weight_p50": float(torch.quantile(weight, 0.5)), - "weight_p90": float(torch.quantile(weight, 0.9)), - "value_min": float(old_v.min()), - "value_max": float(old_v.max()), - "q_min": float(qbar.min()), - "q_max": float(qbar.max()), - "actor_optimizer_step": update, - "value_optimizer_step": update, - "critic_optimizer_step": update, - "value_grad_norm": v_norm, - "critic_grad_norm": q_norm, - "actor_grad_norm": actor_norm, - **{f"validation_{name}": result for name, result in held_out.items()}, - } - metrics.append(record) - if held_out["action_nll"] < best_loss: - best_loss = held_out["action_nll"] - best_actor = { - name: tensor.detach().cpu().clone() - for name, tensor in actor.network.state_dict().items() + selection.observe(actor, held_out["action_nll"], update) + metrics.append( + { + "transaction": update, + **{ + f"{role}_loss": float(loss.detach()) + for role, loss in losses.items() + }, + **{f"validation_{key}": val for key, val in held_out.items()}, + "advantage_mean": float(advantage.mean()), + "weight_mean": float(weight.mean()), + "weight_cap_fraction": float( + (weight >= phase.max_weight).float().mean() + ), } + ) if update % phase.checkpoint_interval == 0 or update == phase.updates: - _save_checkpoint( - checkpoint, - _checkpoint_payload( - config, - data, - fitted, - statistics, - actor, - value, - q1, - q2, - targets, - optimizers, - schedulers, + context.commit( + actor_state( + context, + optimizers["actor"], + schedulers["actor"], sampler, - metrics, + selection, update, - best_loss, - best_actor, - random, - ), + metrics, + networks={ + name: clone_weights(module) + for name, module in ( + ("value", value), + ("q1", q1), + ("q2", q2), + ("target_q1", targets[0]), + ("target_q2", targets[1]), + ) + }, + optimizers={ + role: optimizers[role].state_dict() + for role in ("value", "critic") + }, + schedulers={ + role: schedulers[role].state_dict() + for role in ("value", "critic") + }, + ) ) - if best_actor is None: - raise ValueError("IQL did not produce a finite held-out actor likelihood") - actor.network.load_state_dict(best_actor, strict=True) - report = { - "format": "sequifier.iql-training-report", - "version": 1, - "algorithm": "iql", - "config": config.model_dump(mode="json"), - "training_episode_ids": training.episode_ids, - "validation_episode_ids": validation.episode_ids, - "sources": { - name: [str(path) for path in values] for name, values in sources.items() - }, - "initialize_from": config.initialize_from, - "training_decisions": len(data[0]), - "validation_decisions": len(data[1]), - "best_validation_action_nll": best_loss, - "updates": phase.updates, - "metrics": metrics, - "model_path": config.model_path, - "checkpoint_path": str(checkpoint), - } - save_policy(actor, config.model_path, metadata=report) - destination = Path(config.report_path) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text( - json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8" - ) - return IQLTrainingResult( - actor, - tuple(metrics), - phase.updates, - tuple(training.episode_ids), - tuple(validation.episode_ids), + return selection.finish( + context, + transaction, + {f"{role}_optimizer_steps": transaction for role in modules}, + metrics, + {"data": context.data.contract}, ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 889bfbfc..6077ff66 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -1,146 +1,496 @@ -"""Algorithm boundary used by trajectory-backed train runs and Python callers.""" +"""Sequential policy coordinator and the sole versioned policy run checkpoint.""" -import json -import re +from copy import deepcopy +from dataclasses import replace from pathlib import Path -from typing import TYPE_CHECKING -from sequifier.artifacts.run_checkpoint import checkpoint_path -from sequifier.config.train_config import PolicyTrainConfig, PPOTrainConfig -from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode -from sequifier.logging_paths import dataset_artifact_prefix -from sequifier.rl.checkpoint import save_policy -from sequifier.rl.iql import IQLTrainingResult, run_iql_training -from sequifier.rl.training import PolicyTrainingResult, train_policy -from sequifier.trajectory.importers import source_paths +import torch -if TYPE_CHECKING: - from sequifier.rl.ppo import PPOTrainingResult +from sequifier.config.train_config import PolicyRunConfig +from sequifier.helpers import configure_determinism +from sequifier.io.preference_dataset import PreferenceData +from sequifier.rl.checkpoint import load_policy, save_policy +from sequifier.rl.offline_data import OfflineData +from sequifier.rl.preparation import ( + build_actor, + input_identity, + preflight_environments, + preflight_optimization, + prepare_data, + resolve_normalization, + validate_data, + validate_declared_policy, +) +from sequifier.rl.registry import BUILTIN_ALGORITHMS, validate_registry +from sequifier.rl.run_state import ( + ActorRepresentation, + PhaseContext, + PolicyPhaseResult, + PolicyRunResult, + atomic_save, + capture_rng_state, + clone_weights, + fingerprint, + require_equal, + restore_rng_state, + weight_fingerprint, + write_json_atomic, +) -def run_policy_training( - config: PolicyTrainConfig | PPOTrainConfig, -) -> "PolicyTrainingResult | IQLTrainingResult | PPOTrainingResult": - """Prepare episode partitions, train, and publish the selected policy and report.""" - if isinstance(config, PPOTrainConfig): - from sequifier.rl.ppo import run_ppo_training +def _plan_contract(config): + values = config.model_dump(mode="json") + values.pop("resume") + return values - return run_ppo_training(config) - phase = config.training_plan.phases[0] - if phase.algorithm == "iql": - return run_iql_training(config) - spec = config.policy - all_paths: dict[str, list[Path]] = {} - for name in phase.trajectory_sources + (phase.validation_sources or []): - all_paths[name] = source_paths(config.sources[name]) - flattened = [path for paths in all_paths.values() for path in paths] - protected_paths = set(flattened) - protected_paths.update( - Path(path).resolve() - for name in all_paths - for source in [config.sources[name]] - for path in ( - source.manifest, - source.metadata_config_path, - source.static_data_path, - ) - if path is not None - ) - output_paths = [ - Path(config.model_path), - Path(config.report_path), - checkpoint_path(config), - ] - checkpoint = output_paths[-1] - prefix = dataset_artifact_prefix(config.project_root, config.model_name) - output_paths += [checkpoint.parent / "manifest.json"] + [ - Path(f"{prefix}-{name}.csv") - for name in ( - "training", - "training-full", - "validation", - "validation-full", - "validation-class-shares", - ) - ] - for path in protected_paths: - snapshot = re.fullmatch( - re.escape(config.model_name) + r"-epoch-(\d+)(?:-batch-\d+)?\.pt", path.name - ) - if path.parent == checkpoint.parent and snapshot is not None: - epoch = int(snapshot[1]) - if 0 < epoch <= phase.epochs: + +def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: + """Validate the whole run, execute phase-local lifecycles, and export its final actor.""" + if not isinstance(config, PolicyRunConfig): + raise TypeError("policy training requires PolicyRunConfig") + validate_registry() + configure_determinism(config.seed, config.enforce_determinism) + plan = _plan_contract(config) + data_identity = input_identity(config) + checkpoint = Path(config.checkpoint_path) + if config.resume.policy == "required" and not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + payload = None + if config.resume.policy != "never" and checkpoint.is_file(): + payload = torch.load(checkpoint, map_location="cpu", weights_only=False) + if ( + not isinstance(payload, dict) + or payload.get("artifact_type") != "sequifier_policy_run" + or payload.get("format_version") != 1 + ): + raise ValueError( + "policy resume requires a version-1 multi-phase policy run checkpoint" + ) + require_equal(plan, payload["plan"], "resume.plan") + require_equal( + fingerprint(plan), payload["plan_fingerprint"], "resume.plan_fingerprint" + ) + require_equal(data_identity, payload["data_identity"], "resume.data") + adapter_contract = preflight_environments(config) + if payload: + require_equal( + adapter_contract, payload["adapter_contract"], "resume.environments" + ) + representation = ActorRepresentation(payload["representation"]) + require_equal( + representation.identity, + payload["representation_fingerprint"], + "resume.representation_fingerprint", + ) + validate_declared_policy(config.policy, representation.policy, "resume.policy") + data = prepare_data(config, representation.policy) + actor = build_actor( + config, representation.policy, representation.values["statistics"] + ) + representation.validate_actor(actor, config.units, "resume.representation") + require_equal( + weight_fingerprint(payload["run_actor"]), + payload["run_actor_identity"], + "resume.incoming_actor.weights", + ) + actor.network.load_state_dict(payload["run_actor"], strict=True) + else: + data = prepare_data(config, config.policy) + if config.initialize_from: + source = load_policy( + config.initialize_from, + device=config.device, + interface_name=config.policy_interface, + ) + validate_declared_policy( + config.policy, source.config, "initialize_from.policy" + ) + actor = build_actor( + config, source.config, source.interface.selected_columns_statistics + ) + ActorRepresentation.from_actor(actor, config.units).validate_actor( + source, config.units, "initialize_from" + ) + actor.network.load_state_dict(source.network.state_dict(), strict=True) + else: + first = data[0] + fitting = ( + first.training.fitting_dataset(config.episode_cache_size) + if isinstance(first, PreferenceData) + else first.training + if isinstance(first, OfflineData) + else None + ) + fitted = ( + fitting.fit_categorical_config() + if fitting is not None + else config.policy + ) + if set(fitted.categorical_columns) - fitted.categorical_n_classes.keys(): raise ValueError( - "training snapshots must not overwrite trajectory inputs" + "online-first categorical fields require frozen class counts and codecs" ) - if len(output_paths) != len(set(output_paths)): - raise ValueError("training outputs must use distinct paths") - if set(output_paths) & protected_paths: - raise ValueError("training outputs must not overwrite trajectory inputs") - if len(flattened) != len(set(flattened)): - raise ValueError("trajectory files cannot appear in multiple selected sources") + if config.normalization is not None or fitting is None: + statistics = resolve_normalization(fitted, config.normalization) + else: + fitting.use_config(fitted) + statistics = fitting.normalization_statistics() + actor = build_actor(config, fitted, statistics) + representation = ActorRepresentation.from_actor(actor, config.units) + if config.normalization is not None: + require_equal( + resolve_normalization(actor.config, config.normalization), + representation.values["statistics"], + "run.normalization", + ) + validate_data(config, data, actor) + data_contract = [item.contract if item is not None else None for item in data] + if payload: + require_equal(data_contract, payload["data_contract"], "resume.partitions") + for phase in config.phases: + if phase.representation is not None: + require_equal( + representation.values["policy"], + phase.representation.model_dump(mode="json"), + f"phases.{phase.name}.representation", + ) + normalization = getattr(phase, "normalization", None) + if normalization is not None: + require_equal( + representation.values["statistics"], + resolve_normalization(actor.config, normalization), + f"phases.{phase.name}.normalization", + ) + phase_index = payload["phase_index"] if payload else 0 + results = ( + [ + PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}) + for result in payload["completed_results"] + ] + if payload + else [] + ) + selected_actors = payload["selected_actors"] if payload else [] + current_state = payload["phase_state"] if payload else None + if ( + not isinstance(phase_index, int) + or not 0 <= phase_index <= len(config.phases) + or len(results) != phase_index + or len(selected_actors) != phase_index + ): + raise ValueError("checkpoint phase index/completed handoffs disagree") + if payload and payload["status"] not in {"ready", "running", "handoff", "complete"}: + raise ValueError("invalid policy checkpoint status") + if payload: + status = payload["status"] + if ( + (status == "running") != (current_state is not None) + or status == "complete" + and phase_index != len(config.phases) + or status == "ready" + and phase_index != 0 + or status == "handoff" + and phase_index == 0 + ): + raise ValueError( + "checkpoint status disagrees with the committed phase boundary" + ) + require_equal( + { + "phase_index": phase_index, + "transaction": current_state["transaction"] if current_state else None, + }, + payload["last_committed_transaction"], + "resume.last_committed_transaction", + ) + if current_state is not None: + if phase_index == len(config.phases): + raise ValueError("completed run cannot have a live phase state") + phase = config.phases[phase_index] + entry = BUILTIN_ALGORITHMS[phase.algorithm] + require_equal( + {"algorithm": phase.algorithm, "version": entry.state_version}, + payload["phase_runtime"] if payload else None, + "resume.phase_runtime", + ) + if current_state.get("version") != entry.state_version: + raise ValueError("unsupported phase checkpoint state version") + if current_state is not None: + restored_actor = deepcopy(actor) + restored_actor.network.load_state_dict(current_state["actor"], strict=True) + if any( + not torch.isfinite(t).all() + for t in restored_actor.network.state_dict().values() + ): + raise ValueError("non-finite checkpoint actor") + if getattr(config.phases[phase_index], "reference", None) is not None: + reference_weights = current_state["reference"] + restored_actor.network.load_state_dict(reference_weights, strict=True) + require_equal( + weight_fingerprint(reference_weights), + current_state["reference_identity"], + "resume.embedded_reference", + ) + if any(not torch.isfinite(t).all() for t in reference_weights.values()): + raise ValueError("non-finite checkpoint reference") + initialization = ( + payload["initialization"] + if payload + else { + "path": config.initialize_from, + "weight_hash": weight_fingerprint(actor.network.state_dict()), + } + ) + if payload and config.initialize_from and Path(config.initialize_from).is_file(): + initial = load_policy( + config.initialize_from, + device=config.device, + interface_name=config.policy_interface, + ) + representation.validate_actor(initial, config.units, "resume.initialize_from") + require_equal( + initialization["weight_hash"], + weight_fingerprint(initial.network.state_dict()), + "resume.initialize_from.weights", + ) + if ( + current_state is not None + and getattr(config.phases[phase_index], "reference", None) is not None + ): + declaration = getattr(config.phases[phase_index], "reference") + if declaration.source == "phase_start": + require_equal( + weight_fingerprint(current_state["reference"]), + weight_fingerprint(actor.network.state_dict()), + "resume.phase_start_reference.weights", + ) + reference_contract = dict(payload["reference_contract"]) if payload else {} + for index, phase in enumerate(config.phases): + reference = getattr(phase, "reference", None) + if reference is None or reference.source != "artifact": + continue + path = Path(reference.path) + if path.is_file(): + ref = load_policy( + path, device=config.device, interface_name=config.policy_interface + ) + representation.validate_actor( + ref, config.units, f"phases.{phase.name}.reference" + ) + identity = weight_fingerprint(ref.network.state_dict()) + if str(index) in reference_contract: + require_equal( + reference_contract[str(index)], + identity, + f"phases.{phase.name}.reference.weights", + ) + reference_contract[str(index)] = identity + elif ( + not payload + or index > phase_index + or index == phase_index + and current_state is None + ): + raise FileNotFoundError( + f"reference artifact required before phase start: {path}" + ) + for index, (result, weights) in enumerate(zip(results, selected_actors)): + require_equal(index, result.index, "resume.completed_phase.index") + require_equal( + config.phases[index].name, result.name, "resume.completed_phase.name" + ) + require_equal( + config.phases[index].algorithm, + result.algorithm, + "resume.completed_phase.algorithm", + ) + require_equal( + result.selected_weight_hash, + weight_fingerprint(weights), + "resume.selected_actor.weights", + ) + if any(not torch.isfinite(t).all() for t in weights.values()): + raise ValueError("non-finite completed phase actor") + if selected_actors: + require_equal( + weight_fingerprint(actor.network.state_dict()), + weight_fingerprint(selected_actors[-1]), + "resume.incoming_selected_actor", + ) + saved_random = capture_rng_state(actor.device) + preflight_optimization(config, actor, data) + restore_rng_state(actor.device, payload["random"] if payload else saved_random) - def dataset(names: list[str]) -> PolicySequenceDataset: - return PolicySequenceDataset( - [config.sources[name] for name in names], - spec, - episode_cache_size=config.global_training.episode_cache_size, + def envelope(status, state=None): + runtime = ( + None + if phase_index == len(config.phases) + else { + "algorithm": config.phases[phase_index].algorithm, + "version": BUILTIN_ALGORITHMS[ + config.phases[phase_index].algorithm + ].state_version, + } ) + return { + "artifact_type": "sequifier_policy_run", + "format_version": 1, + "plan": plan, + "plan_fingerprint": fingerprint(plan), + "representation": representation.canonical, + "representation_fingerprint": representation.identity, + "data_identity": data_identity, + "data_contract": data_contract, + "adapter_contract": adapter_contract, + "reference_contract": reference_contract, + "random": capture_rng_state(actor.device), + "phase_index": phase_index, + "status": status, + "completed_results": [r.as_dict() for r in results], + "selected_actors": selected_actors, + "run_actor": clone_weights(actor.network), + "run_actor_identity": weight_fingerprint(actor.network.state_dict()), + "initialization": initialization, + "resume_request": config.resume.model_dump(mode="json"), + "phase_runtime": runtime, + "phase_state": state, + "last_committed_transaction": { + "phase_index": phase_index, + "transaction": state["transaction"] if state else None, + }, + } - corpus = dataset(phase.trajectory_sources) - if phase.validation_sources is None: - training, validation = split_by_episode( - corpus, - config.global_training.validation_fraction, - config.seed, - group_identical=True, + published = set() + + def publish_completed(): + for result, weights in zip(results, selected_actors): + if result.index in published: + continue + portable = deepcopy(actor).eval() + portable.network.load_state_dict(weights, strict=True) + directory = ( + Path(config.phase_directory) / f"{result.index:03d}-{result.name}" + ) + save_policy(portable, directory / "actor.pt", metadata=result.as_dict()) + write_json_atomic(directory / "report.json", result.as_dict()) + published.add(result.index) + + if payload is None: + # Persist the complete fitted representation before the first update. + atomic_save(checkpoint, envelope("ready")) + while phase_index < len(config.phases): + phase = config.phases[phase_index] + require_equal( + data_identity, + input_identity(config), + f"phases.{phase.name}.source_identity", ) - else: - training = corpus - validation = dataset(phase.validation_sources) - if set(corpus.all_episode_ids) & set(validation.all_episode_ids): - raise ValueError("duplicate episode IDs across training/validation sources") - if not len(training) or not len(validation): - raise ValueError( - f"policy training needs nonempty eligible partitions; exclusions: {corpus.exclusions}" - ) - for attribute in ("group_ids", "demonstration_keys"): - if (set(getattr(training, attribute)) & set(getattr(validation, attribute))) - { - None - }: - raise ValueError(f"{attribute} cannot cross training/validation partitions") - exclusions = {**corpus.exclusions, **validation.exclusions} - sources = {name: [str(path) for path in paths] for name, paths in all_paths.items()} + incoming_weight_hash = weight_fingerprint(actor.network.state_dict()) + phase_actor = deepcopy(actor).eval() + reference, reference_identity = None, None + declaration = getattr(phase, "reference", None) + if declaration is not None: + if current_state is not None: + reference = deepcopy(phase_actor) + reference.network.load_state_dict( + current_state["reference"], strict=True + ) + elif declaration.source == "artifact": + reference = load_policy( + declaration.path, + device=config.device, + interface_name=config.policy_interface, + ) + else: + reference = deepcopy(phase_actor) + reference.eval().requires_grad_(False) + representation.validate_actor( + reference, config.units, f"phases.{phase.name}.reference" + ) + reference_identity = weight_fingerprint(reference.network.state_dict()) + if current_state is not None: + require_equal( + reference_identity, + current_state["reference_identity"], + "resume.reference.weights", + ) + if declaration.source == "artifact": + require_equal( + reference_contract[str(phase_index)], + reference_identity, + "reference.artifact.weights", + ) - result = train_policy(training, validation, config) + def commit(state): + representation.validate_actor( + phase_actor, config.units, f"phases.{phase.name}.actor" + ) + atomic_save(checkpoint, envelope("running", state)) + + context = PhaseContext( + config, + phase_index, + phase, + phase_actor, + representation, + data[phase_index], + reference, + reference_identity, + current_state, + commit, + ) + actor, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) + representation.validate_actor( + actor, config.units, f"phases.{phase.name}.selected_actor" + ) + result = replace( + result, + provenance={ + **result.provenance, + "config_fingerprint": fingerprint(phase.model_dump(mode="json")), + "representation_fingerprint": representation.identity, + "incoming_weight_hash": incoming_weight_hash, + "actor_path": str( + Path(config.phase_directory) + / f"{phase_index:03d}-{phase.name}" + / "actor.pt" + ), + "report_path": str( + Path(config.phase_directory) + / f"{phase_index:03d}-{phase.name}" + / "report.json" + ), + }, + ) + results.append(result) + selected_actors.append(clone_weights(actor.network)) + phase_index += 1 + current_state = None + atomic_save(checkpoint, envelope("handoff")) + publish_completed() + publish_completed() report = { - "format": "sequifier.policy-training-report", + "format": "sequifier.policy-run-report", "version": 1, - "algorithm": "policy_imitation", - "config": config.model_dump(mode="json"), - "sources": sources, - "policy": result.policy.config.model_dump(mode="json"), - "training_episode_ids": list(result.training_episode_ids), - "validation_episode_ids": list(result.validation_episode_ids), - "excluded_episode_ids": list(exclusions), - "exclusions": exclusions, - "training_decisions": len(training), - "validation_decisions": len(validation), - "best_epoch": result.best_epoch, - "optimizer_steps": result.optimizer_steps, - "epochs": list(result.metrics), - "monitor": { - "metric": "action_nll", - "direction": "min", - "value": result.metrics[result.best_epoch]["validation_nll"], - }, - "artifact_kind": "sequifier_model", + "plan": plan, + "plan_fingerprint": fingerprint(plan), + "representation_fingerprint": representation.identity, + "source_provenance": data_identity, + "reference_provenance": reference_contract, + "initialization": initialization, + "resume": config.resume.model_dump(mode="json"), + "phases": [r.as_dict() for r in results], "model_path": config.model_path, + "checkpoint_path": config.checkpoint_path, } - save_policy(result.policy, config.model_path, metadata=report) - destination = Path(config.report_path) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text( - json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8" + save_policy(actor.eval(), config.model_path, metadata=report) + write_json_atomic(config.report_path, report) + atomic_save(checkpoint, envelope("complete")) + return PolicyRunResult( + actor, + tuple(results), + representation, + config.model_path, + config.report_path, + config.checkpoint_path, ) - return result diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py new file mode 100644 index 00000000..820a42c7 --- /dev/null +++ b/src/sequifier/rl/offline_data.py @@ -0,0 +1,63 @@ +"""Offline phase partitioning and frozen-codec validation.""" + +from dataclasses import dataclass + +from sequifier.io.iql_dataset import IQLTransitionDataset +from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode +from sequifier.rl.run_state import fingerprint + + +@dataclass(frozen=True) +class OfflineData: + training: PolicySequenceDataset + validation: PolicySequenceDataset + + @property + def contract(self): + return [ + { + "episodes": [ + fingerprint(episode.model_dump(mode="json")) + for episode in part.episodes + ], + "episode_ids": part.episode_ids, + "decisions": len(part), + } + for part in (self.training, self.validation) + ] + + def use_config(self, config): + for part in (self.training, self.validation): + part.use_config(config) + + +def prepare_offline(run, phase, config): + def corpus(names): + return PolicySequenceDataset( + [run.sources[name] for name in names], + config, + episode_cache_size=run.episode_cache_size, + ) + + training = corpus(phase.sources) + if phase.validation_sources: + validation = corpus(phase.validation_sources) + else: + training, validation = split_by_episode( + training, run.validation_fraction, run.seed, group_identical=True + ) + if not len(training) or not len(validation): + raise ValueError( + f"phases.{phase.name}: nonempty train and validation partitions required" + ) + for attribute in ("all_episode_ids", "group_ids", "demonstration_keys"): + if (set(getattr(training, attribute)) & set(getattr(validation, attribute))) - { + None + }: + raise ValueError( + f"phases.{phase.name}: {attribute} crosses train/validation partitions" + ) + if phase.algorithm == "iql": + for part in (training, validation): + IQLTransitionDataset(part) + return OfflineData(training, validation) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index f2801759..184777bb 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -1,6 +1,9 @@ """Policy alignment adapter over Sequifier's configurable model system.""" +from __future__ import annotations + from collections.abc import Mapping +from typing import TYPE_CHECKING import torch from torch import Tensor, nn @@ -8,7 +11,11 @@ from sequifier.artifacts.model_config import execution_config from sequifier.config.metadata import DatasetMetadata from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig -from sequifier.config.train_config import GlobalTrainingSpecModel, _resolve_interface +from sequifier.config.train_config import ( + GlobalTrainingSpecModel, + ResolvedSequifierConfig, + _resolve_interface, +) from sequifier.io.batch import SequifierInputs from sequifier.io.policy_window import ( DESIRED_RETURN, @@ -20,7 +27,9 @@ from sequifier.io.preprocessing import normalization_statistics from sequifier.model.execution_schema import ExecutionSchema from sequifier.rl.distributions import CategoricalActionDistribution -from sequifier.special_tokens import SPECIAL_TOKEN_IDS, SPECIAL_TOKEN_NAMES + +if TYPE_CHECKING: + from sequifier.model.network import ComposableTransformerNetwork def policy_execution_config( @@ -85,21 +94,12 @@ def policy_execution_config( if column != DESIRED_RETURN }, ) - # Legacy action IDs can occupy reserved-token slots. Resolve the same IDs - # through the ordinary codec, then retain the persisted policy contract. - spec = interface.model_copy( - update={ - "categorical_decoder_special_tokens": { - target: [ - name - for name in SPECIAL_TOKEN_NAMES - if getattr(SPECIAL_TOKEN_IDS, name) in decoder_ids - ] - } - } - ) resolved = _resolve_interface( - config.interface_name, spec, metadata, global_training, metadata.storage_layout + config.interface_name, + interface, + metadata, + global_training, + metadata.storage_layout, ) resolved.policy = config resolved.categorical_columns = categorical @@ -117,6 +117,14 @@ def policy_execution_config( class CategoricalPolicy(nn.Module): """A convenience view; the network owns all trainable weights and execution.""" + config: CategoricalPolicyConfig + execution_config: ResolvedSequifierConfig + interface_name: str + network: ComposableTransformerNetwork + units: dict[str, str] + ppo_policy_version: int + grpo_policy_version: int + @classmethod def from_network(cls, network, execution_config, interface_name: str): interface = execution_config.interface(interface_name) diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index a5cd6401..37178caf 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -1,499 +1,196 @@ -"""Synchronous single-slot categorical PPO, with portable actor-only export.""" +"""Phase-local PPO lifecycle over the shared collector and clipped actor/V update.""" -import hashlib -import platform -import statistics -import sys -import time from copy import deepcopy -from dataclasses import dataclass from pathlib import Path import torch -from loguru import logger -from sequifier.model.factory import build_transformer_network -from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class -from sequifier.rl.checkpoint import load_policy, save_policy +from sequifier.config.policy_plan import PPOPhaseConfig from sequifier.rl.environment import ( adapter_identity, load_online_adapter, require_exact_snapshot, ) from sequifier.rl.models import HistoryCritic -from sequifier.rl.policy import CategoricalPolicy, policy_execution_config -from sequifier.rl.ppo_buffer import write_json_atomic -from sequifier.rl.ppo_checkpoint import ( - capture_rng_state, +from sequifier.rl.ppo_collector import PPOCollector +from sequifier.rl.ppo_evaluation import evaluate_policy +from sequifier.rl.ppo_update import update_ppo +from sequifier.rl.run_state import ( + PhaseContext, + Selection, + actor_state, clone_weights, - contract_fingerprint, - load_ppo_checkpoint, + make_optimization, + require_equal, + restore_actor_state, restore_rng_state, - resume_contract, - save_ppo_checkpoint, - snapshots_equal, + validate_finite_state, ) -from sequifier.rl.ppo_collector import PPOCollector, stream_seed -from sequifier.rl.ppo_evaluation import evaluate_policy, require_evaluation_timer -from sequifier.rl.ppo_update import update_ppo - - -@dataclass(frozen=True) -class PPOTrainingResult: - policy: CategoricalPolicy - metrics: tuple[dict, ...] - evaluations: tuple[dict, ...] - env_steps: int - iterations: int - best_metric: float | None - report: dict - - -def _make_optimizer(module, spec, learning_rate): - return get_optimizer_class(spec.name)( - [parameter for parameter in module.parameters() if parameter.requires_grad], - lr=learning_rate, - **spec.arguments, - ) - - -def _initial_actor(config, execution, will_resume): - actor = CategoricalPolicy.from_network( - build_transformer_network( - execution, device=torch.device(config.device) - ).network, - execution, - config.policy.interface_name, - ) - provenance = None - if config.initialize_from is not None and not will_resume: - source = load_policy(config.initialize_from, device=config.device) - if ( - source.config != config.policy - or source.interface.selected_columns_statistics - != config.normalization_statistics - or source.execution_config.model != execution.model - ): - raise ValueError( - "PPO initialize_from must exactly match policy, normalization, model architecture and context" - ) - actor.network.load_state_dict(source.network.state_dict(), strict=True) - provenance = { - "path": config.initialize_from, - "sha256": hashlib.sha256( - Path(config.initialize_from).read_bytes() - ).hexdigest(), - } - return actor.eval(), provenance - - -def run_ppo_training(config) -> PPOTrainingResult: - """Train one authored PPO phase; commits occur only after complete updates.""" - from sequifier.config.train_config import PPOTrainConfig - from sequifier.helpers import configure_determinism +from sequifier.rl.snapshots import snapshots_equal - if not isinstance(config, PPOTrainConfig): - raise TypeError("run_ppo_training requires PPOTrainConfig") - require_evaluation_timer() - import resource - configure_determinism(config.seed, config.enforce_determinism) - phase = config.training_plan.phases[0] - device = torch.device(config.device) - if device.type == "cuda" and not torch.cuda.is_available(): - raise ValueError("requested CUDA device is unavailable") - if device.type == "mps" and not torch.backends.mps.is_available(): - raise ValueError("requested MPS device is unavailable") - checkpoint = Path(config.checkpoint_path) - if phase.resume.policy == "required" and not checkpoint.is_file(): - raise FileNotFoundError(checkpoint) - will_resume = phase.resume.policy != "never" and checkpoint.is_file() - payload = load_ppo_checkpoint(checkpoint, config) if will_resume else None - if ( - config.initialize_from is not None - and not will_resume - and not Path(config.initialize_from).is_file() - ): - raise FileNotFoundError(config.initialize_from) - execution = policy_execution_config( - config.policy, - config.model, - config.device, - statistics=config.normalization_statistics, - project_root=config.project_root, - model_name=config.model_name, - seed=config.seed, +def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): + actor, phase = context.actor.eval(), context.phase + value = HistoryCritic( + deepcopy(actor.execution_config), actor.interface_name, None + ).eval() + optimizer, scheduler = make_optimization(actor, phase) + value_optimizer, value_scheduler = make_optimization(value, phase, "value") + sampler = torch.Generator().manual_seed(context.run.seed + context.index) + state = context.resume_state or {} + transaction, env_steps = state.get("transaction", 0), state.get("env_steps", 0) + metrics, evaluations = ( + list(state.get("metrics", [])), + list(state.get("evaluations", [])), ) - actor, provenance = _initial_actor(config, execution, will_resume) - actor.ppo_policy_version = 0 - actor.set_normalization_statistics(config.normalization_statistics) - value = HistoryCritic(execution, config.policy.interface_name, None).eval() - modules = {"actor": actor, "value": value} - if any( - not torch.isfinite(tensor).all() - for module in modules.values() - for tensor in module.state_dict().values() - ): - raise ValueError("PPO actor and V initialization must be finite") - ids = [ - id(parameter) - for module in modules.values() - for parameter in module.parameters() - if parameter.requires_grad - ] - if len(ids) != len(set(ids)): - raise ValueError("PPO actor and V must own disjoint trainable parameters") + continuations = list(state.get("continuations", [])) + clocks = dict(state.get("clocks", {"actor": 0, "value": 0})) if ( - value.interface.selected_columns_statistics - != actor.interface.selected_columns_statistics + env_steps != min(transaction * phase.rollout_length, phase.total_env_steps) + or not 0 <= env_steps <= phase.total_env_steps ): - raise ValueError("PPO actor and V normalization differs") - plans = { - "actor": ( - actor, - phase.actor_optimizer, - phase.actor_learning_rate, - phase.actor_scheduler, - ), - "value": ( - value, - phase.value_optimizer, - phase.value_learning_rate, - phase.value_scheduler, - ), - } - optimizers = { - role: _make_optimizer(module, optimizer_spec, learning_rate) - for role, (module, optimizer_spec, learning_rate, _) in plans.items() - } - schedulers = { - role: get_scheduler_class(scheduler_spec.name)( - optimizers[role], **scheduler_spec.arguments - ) - for role, (_, _, _, scheduler_spec) in plans.items() - } - sampler = torch.Generator(device="cpu").manual_seed( - stream_seed(config.seed, "minibatches", 0) + raise ValueError("PPO checkpoint is not a complete collection/update boundary") + if state: + for key in ("value", "value_optimizer", "value_scheduler"): + validate_finite_state(state[key], f"ppo.{key}") + if any( + not isinstance(clock, int) + or isinstance(clock, bool) + or not transaction <= clock <= phase.optimizer_step_budget() + for clock in clocks.values() + ) or set(clocks) != {"actor", "value"}: + raise ValueError("PPO optimizer clocks disagree with committed progress") + value.load_state_dict(state["value"], strict=True) + value_optimizer.load_state_dict(state["value_optimizer"]) + value_scheduler.load_state_dict(state["value_scheduler"]) + restore_actor_state(context, optimizer, scheduler, sampler) + selection = Selection( + phase, + phase.evaluation.metric, + phase.evaluation.direction, + state.get("selection"), ) - clocks = {"actor": 0, "value": 0} - metrics, evaluations, continuations = [], [], [] - env_steps, iteration, policy_version = 0, 0, 0 - best_actor, best_metric, best_iteration = None, None, None - trajectory_equivalent = True + actor.ppo_policy_version = transaction adapter = load_online_adapter(phase.environment) try: - if phase.resume.mode == "exact": + if context.run.resume.mode == "exact": require_exact_snapshot(adapter) - collector = PPOCollector(actor, value, adapter, phase, config.seed) - if payload is not None: - if payload["adapter_identity"] != adapter_identity(adapter): - raise ValueError( - "PPO adapter implementation identity or snapshot capability differs" - ) - actor.network.load_state_dict(payload["actor"], strict=True) - value.load_state_dict(payload["value"], strict=True) - for role in optimizers: - optimizers[role].load_state_dict(payload["optimizers"][role]) - schedulers[role].load_state_dict(payload["schedulers"][role]) - sampler.set_state(payload["sampler"]) - clocks = dict(payload["optimizer_steps"]) - env_steps, iteration, policy_version = ( - payload["env_steps"], - payload["iteration"], - payload["policy_version"], - ) - best_actor, best_metric, best_iteration = ( - payload["best_actor"], - payload["best_metric"], - payload["best_iteration"], - ) - metrics, evaluations = ( - list(payload["metrics"]), - list(payload["evaluations"]), - ) - continuations = list(payload["continuations"]) - provenance = payload["initialization"] - trajectory_equivalent = payload["trajectory_equivalent"] - exact = phase.resume.mode == "exact" + identity = adapter_identity(adapter) + collector = PPOCollector( + actor, value, adapter, phase, context.run.seed + context.index + ) + if state: + require_equal(state["adapter_identity"], identity, "ppo.adapter_identity") + exact = context.run.resume.mode == "exact" if exact: - if payload["environment_state"] is None: + if state["environment_state"] is None: raise ValueError( - "checkpoint has no declared exact environment snapshot" + "exact PPO resume requires an embedded environment snapshot" ) - adapter.restore_state(deepcopy(payload["environment_state"])) + adapter.restore_state(deepcopy(state["environment_state"])) if not snapshots_equal( - adapter.snapshot_state(), payload["environment_state"] + adapter.snapshot_state(), state["environment_state"] ): - raise ValueError( - "adapter did not restore its simulator and RNG snapshot exactly" - ) - else: - trajectory_equivalent = False - collector.load_state_dict(payload["collector"], exact=exact) + raise ValueError("PPO adapter did not restore its snapshot exactly") + collector.load_state_dict(state["collector"], exact=exact) continuations.append( { - "iteration": iteration, - "mode": phase.resume.mode, - "trajectory_equivalent": trajectory_equivalent, - "dropped_incomplete_episode": not exact - and not payload["collector"]["ended"], - } - ) - actor.ppo_policy_version = policy_version - restore_rng_state(device, payload["random"]) - - def evaluate(): - nonlocal best_actor, best_metric, best_iteration - result = evaluate_policy(actor, phase.evaluation, forbidden_adapter=adapter) - result.update( - { - "env_steps": env_steps, - "iteration": iteration, - "policy_version": policy_version, + "transaction": transaction, + "mode": context.run.resume.mode, + "trajectory_equivalent": exact, } ) - evaluations.append(result) - score = result["selection_value"] - better = score is not None and ( - best_metric is None - or ( - score > best_metric - if phase.evaluation.direction == "max" - else score < best_metric - ) - ) - if better: - best_metric, best_iteration = score, iteration - best_actor = clone_weights(actor.network) - return result["seconds"] - - if not evaluations: - evaluate() + if state: + restore_rng_state(actor.device, state["random"]) while env_steps < phase.total_env_steps: horizon = min(phase.rollout_length, phase.total_env_steps - env_steps) - started = time.monotonic() - buffer, episodes = collector.collect(horizon, policy_version) - collection_seconds = time.monotonic() - started - if phase.archive_directory is not None: + buffer, episodes = collector.collect(horizon, transaction) + if phase.archive_directory: buffer.archive( Path(phase.archive_directory) - / f"collection-{iteration + 1:08d}.json", + / f"collection-{transaction+1:08d}.json", include_scores=phase.archive_scores, ) - started = time.monotonic() record = update_ppo( actor, value, buffer, phase, - optimizers, - schedulers, + {"actor": optimizer, "value": value_optimizer}, + {"actor": scheduler, "value": value_scheduler}, sampler, - policy_version, + transaction, clocks, ) - update_seconds = time.monotonic() - started - env_steps += horizon - iteration += 1 - policy_version += 1 - evaluation_seconds = 0.0 + env_steps, transaction = env_steps + horizon, transaction + 1 if ( - iteration % phase.evaluation_interval == 0 + transaction % phase.evaluation_interval == 0 or env_steps == phase.total_env_steps ): - evaluation_seconds = evaluate() - completed_returns = [ - episode["return"] for episode in episodes if episode["terminated"] - ] - completed_lengths = [ - episode["length"] for episode in episodes if episode["terminated"] - ] - record.update( + result = evaluate_policy( + actor, phase.evaluation, forbidden_adapter=adapter + ) + selection.observe(actor, result["selection_value"], transaction) + evaluations.append( + {**result, "transaction": transaction, "env_steps": env_steps} + ) + metrics.append( { - "iteration": iteration, - "policy_version": policy_version, - "collected_policy_version": buffer.policy_version, + **record, + "transaction": transaction, "env_steps": env_steps, - "collected_decisions": horizon, - "collection_seconds": collection_seconds, - "update_seconds": update_seconds, - "evaluation_seconds": evaluation_seconds, - "collection_steps_per_second": horizon - / max(collection_seconds, 1e-9), - "finished_episodes": episodes, - "training_mean_completed_return": statistics.fmean( - completed_returns - ) - if completed_returns - else None, - "training_return_std": statistics.pstdev(completed_returns) - if completed_returns - else None, - "training_mean_completed_length": statistics.fmean( - completed_lengths - ) - if completed_lengths - else None, - "completed_episode_count": sum( - episode["terminated"] for episode in episodes - ), - "censored_episode_count": sum( - episode["truncated"] and not episode["terminated"] - for episode in episodes - ), - "invalid_episode_count": 0, - "live_episode_length": 0 - if collector.ended - else collector.episode_length, - "buffer_estimated_bytes": buffer.estimated_bytes, - "actor_learning_rate": optimizers["actor"].param_groups[0]["lr"], - "value_learning_rate": optimizers["value"].param_groups[0]["lr"], + "policy_version": actor.ppo_policy_version, + "episodes": episodes, } ) - metrics.append(record) - logger.info( - "PPO iteration {}: {}/{} actions, policy loss {:.5f}, V loss {:.5f}", - iteration, - env_steps, - phase.total_env_steps, - record["policy_loss"], - record["value_loss"], - ) if ( - iteration % phase.checkpoint_interval == 0 + transaction % phase.checkpoint_interval == 0 or env_steps == phase.total_env_steps ): environment_state = None - if getattr(adapter, "supports_exact_snapshot", None) is True: + if adapter.capabilities.exact_snapshot: require_exact_snapshot(adapter) environment_state = adapter.snapshot_state() if not isinstance(environment_state, dict) or not snapshots_equal( environment_state, environment_state ): raise ValueError( - "adapter exact snapshot must be a finite, portable state dictionary" + "PPO snapshot must be a finite portable state dictionary" ) - save_ppo_checkpoint( - checkpoint, - { - "artifact_type": "sequifier_ppo_run", - "format_version": 1, - "config": resume_contract(config), - "config_fingerprint": contract_fingerprint(config), - "policy": config.policy.model_dump(mode="json"), - "statistics": config.normalization_statistics, - "actor_spec": config.model.model_dump(mode="json"), - "value_spec": { - "type": "HistoryCritic", - "action_count": None, - "model": config.model.model_dump(mode="json"), - }, - "actor": clone_weights(actor.network), - "value": clone_weights(value), - "optimizers": { - role: optimizer.state_dict() - for role, optimizer in optimizers.items() - }, - "schedulers": { - role: scheduler.state_dict() - for role, scheduler in schedulers.items() - }, - "optimizer_steps": clocks, - "sampler": sampler.get_state(), - "random": capture_rng_state(device), - "seed_streams": { - "version": 1, - "run_seed": config.seed, - "evaluation_seeds": phase.evaluation.seeds, - }, - "collector": collector.state_dict(), - "environment_state": environment_state, - "adapter_identity": adapter_identity(adapter), - "env_steps": env_steps, - "iteration": iteration, - "policy_version": policy_version, - "best_actor": best_actor, - "best_metric": best_metric, - "best_iteration": best_iteration, - "initialization": provenance, - "trajectory_equivalent": trajectory_equivalent, - "continuations": continuations, - "metrics": metrics, - "evaluations": evaluations, - }, + context.commit( + actor_state( + context, + optimizer, + scheduler, + sampler, + selection, + transaction, + metrics, + value=clone_weights(value), + value_optimizer=value_optimizer.state_dict(), + value_scheduler=value_scheduler.state_dict(), + clocks=clocks, + env_steps=env_steps, + evaluations=evaluations, + continuations=continuations, + collector=collector.state_dict(), + environment_state=environment_state, + adapter_identity=identity, + ) ) finally: adapter.close() - if best_actor is not None: - actor.network.load_state_dict(best_actor, strict=True) - elapsed = sum( - record["collection_seconds"] + record["update_seconds"] for record in metrics - ) - peak_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - report = { - "format": "sequifier.ppo-training-report", - "version": 1, - "algorithm": "ppo", - "config": config.model_dump(mode="json"), - "config_fingerprint": contract_fingerprint(config), - "env_steps": env_steps, - "iterations": iteration, - "policy_version": policy_version, - "optimizer_steps": clocks, - "metrics": metrics, - "evaluations": evaluations, - "monitor": { - "metric": phase.evaluation.metric, - "direction": phase.evaluation.direction, - "value": best_metric, - "available": best_metric is not None, - "best_iteration": best_iteration, + return selection.finish( + context, + transaction, + { + "env_steps": env_steps, + **{f"{key}_optimizer_steps": val for key, val in clocks.items()}, }, - "export_selection": "best_evaluated_actor" - if best_actor is not None - else "latest_actor_metric_unavailable", - "initialization": provenance, - "trajectory_equivalent": trajectory_equivalent, - "continuations": continuations, - "invalid_episode_count": 0, - "evaluation_censored_count": sum( - result["censored_count"] for result in evaluations - ), - "training_completed_count": sum( - record["completed_episode_count"] for record in metrics - ), - "training_censored_count": sum( - record["censored_episode_count"] for record in metrics - ), - "final_live_episode_censored": not collector.ended, - "hardware": { - "device": str(device), - "platform": platform.platform(), - "processor": platform.processor(), - "torch_version": str(torch.__version__), - "torch_threads": torch.get_num_threads(), + metrics, + { + "adapter": identity, + "evaluations": evaluations, + "continuations": continuations, }, - "context_length": config.context_length, - "actor_parameters": sum(parameter.numel() for parameter in actor.parameters()), - "value_parameters": sum(parameter.numel() for parameter in value.parameters()), - "steps_per_second": env_steps / max(elapsed, 1e-9), - "peak_process_memory_bytes": peak_rss - if sys.platform == "darwin" - else peak_rss * 1024, - "peak_cuda_memory_bytes": torch.cuda.max_memory_allocated(device) - if device.type == "cuda" - else None, - "model_path": config.model_path, - "checkpoint_path": config.checkpoint_path, - } - save_policy(actor, config.model_path, metadata=report) - write_json_atomic(config.report_path, report) - return PPOTrainingResult( - actor, - tuple(metrics), - tuple(evaluations), - env_steps, - iteration, - best_metric, - report, ) diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py index 67079962..78ea1a83 100644 --- a/src/sequifier/rl/ppo_buffer.py +++ b/src/sequifier/rl/ppo_buffer.py @@ -10,7 +10,8 @@ from sequifier.config.policy_config import CategoricalPolicyConfig from sequifier.io.batch import SequifierInputs from sequifier.io.tensors import collate_tensor_records -from sequifier.rl.history import history_inputs +from sequifier.rl.history import frame_storage_bytes, history_inputs +from sequifier.rl.run_state import write_json_atomic @dataclass(frozen=True) @@ -143,19 +144,6 @@ def archive(self, path, *, include_scores): write_json_atomic(path, payload) -def write_json_atomic(path, payload): - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.tmp") - try: - temporary.write_text( - json.dumps(payload, indent=2, allow_nan=False) + "\n", encoding="utf-8" - ) - temporary.replace(path) - finally: - temporary.unlink(missing_ok=True) - - def load_collection_record(path): """Read the independent archival format without changing v1 episode readers.""" payload = json.loads(Path(path).read_text(encoding="utf-8")) @@ -242,7 +230,7 @@ def __init__(self, spec, version, horizon, initial_frames, max_bytes): self.decisions = [] self.max_bytes = max_bytes self.bytes_per_frame = max( - [self.frame_bytes(frame) for frame in self.frames] + [1024] + [frame_storage_bytes(frame) for frame in self.frames] + [1024] ) self.estimated_bytes = ( 2 * horizon + spec.context_length @@ -251,20 +239,13 @@ def __init__(self, spec, version, horizon, initial_frames, max_bytes): raise ValueError( f"PPO buffer estimate {self.estimated_bytes} exceeds max_buffer_bytes={max_bytes}" ) - self.used_bytes = sum(self.frame_bytes(frame) for frame in self.frames) + self.used_bytes = sum(frame_storage_bytes(frame) for frame in self.frames) self.sealed = False - @staticmethod - def frame_bytes(frame): - # Conservative Python-object allowance plus the complete domain payload. - return 1024 + 4 * len( - json.dumps(frame.state_dict(), allow_nan=False).encode("utf-8") - ) - def append_frame(self, frame): if self.sealed: raise ValueError("PPO buffer is sealed") - cost = self.frame_bytes(frame) + cost = frame_storage_bytes(frame) if self.used_bytes + cost + len(self.decisions) * 1024 > self.max_bytes: raise ValueError("PPO history exceeded its declared memory budget") if len(self.frames) >= 2 * self.horizon + self.spec.context_length: diff --git a/src/sequifier/rl/ppo_checkpoint.py b/src/sequifier/rl/ppo_checkpoint.py deleted file mode 100644 index b95cee97..00000000 --- a/src/sequifier/rl/ppo_checkpoint.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Atomic, algorithm-owned PPO iteration checkpoints.""" - -import hashlib -import json -import math -from collections.abc import Mapping -from pathlib import Path - -import numpy as np -import torch - -from sequifier.runtime.random_state import RandomState, RandomStateManager - - -def resume_contract(config): - values = config.model_dump(mode="json") - values["training_plan"]["phases"][0].pop("resume") - return values - - -def contract_fingerprint(config): - return hashlib.sha256( - json.dumps(resume_contract(config), sort_keys=True, allow_nan=False).encode() - ).hexdigest() - - -def clone_weights(module): - return { - name: tensor.detach().cpu().clone() - for name, tensor in module.state_dict().items() - } - - -def save_ppo_checkpoint(path, payload): - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.tmp") - try: - torch.save(payload, temporary) - temporary.replace(path) - finally: - temporary.unlink(missing_ok=True) - - -def load_ppo_checkpoint(path, config): - payload = torch.load(path, map_location="cpu", weights_only=False) - if ( - not isinstance(payload, Mapping) - or payload.get("artifact_type") != "sequifier_ppo_run" - or payload.get("format_version") != 1 - ): - raise ValueError("PPO resume requires a version-1 PPO learner checkpoint") - if payload.get("config_fingerprint") != contract_fingerprint(config) or payload.get( - "config" - ) != resume_contract(config): - raise ValueError( - "PPO resume config, adapter identity, codec, model, or reward scale differs" - ) - if ( - payload.get("policy") != config.policy.model_dump(mode="json") - or payload.get("statistics") != config.normalization_statistics - ): - raise ValueError("PPO resume frozen representation differs") - phase = config.training_plan.phases[0] - steps = payload["env_steps"] - iteration = payload["iteration"] - expected_steps = min(iteration * phase.rollout_length, phase.total_env_steps) - if ( - not isinstance(steps, int) - or isinstance(steps, bool) - or not isinstance(iteration, int) - or isinstance(iteration, bool) - or iteration < 1 - or not 0 < steps <= phase.total_env_steps - or steps != expected_steps - or iteration != (steps + phase.rollout_length - 1) // phase.rollout_length - or payload["policy_version"] != iteration - or len(payload["metrics"]) != iteration - ): - raise ValueError("PPO checkpoint is not a complete iteration boundary") - if payload["best_metric"] is not None and not math.isfinite(payload["best_metric"]): - raise ValueError("PPO checkpoint best metric must be finite or unavailable") - if (payload["best_actor"] is None) != (payload["best_metric"] is None): - raise ValueError( - "PPO checkpoint best actor and metric must be recorded together" - ) - if set(payload["optimizer_steps"]) != {"actor", "value"} or any( - not isinstance(clock, int) or isinstance(clock, bool) or clock < iteration - for clock in payload["optimizer_steps"].values() - ): - raise ValueError("PPO checkpoint optimizer clocks are invalid") - for state in (payload["actor"], payload["value"], payload["best_actor"] or {}): - if any(not torch.isfinite(tensor).all() for tensor in state.values()): - raise ValueError("PPO checkpoint contains non-finite weights") - return payload - - -def snapshots_equal(left, right): - """Exact adapters use torch-serializable scalar, tensor and container state.""" - if type(left) is not type(right): - return False - if isinstance(left, torch.Tensor): - return ( - left.dtype == right.dtype - and bool(torch.isfinite(left).all()) - and torch.equal(left.cpu(), right.cpu()) - ) - if isinstance(left, np.ndarray): - finite = ( - np.isfinite(left).all() - if left.dtype.kind in "biufc" - else left.dtype.kind in "US" - ) - return ( - left.dtype == right.dtype and bool(finite) and np.array_equal(left, right) - ) - if isinstance(left, np.generic): - return snapshots_equal(left.item(), right.item()) - if isinstance(left, dict): - return left.keys() == right.keys() and all( - snapshots_equal(left[key], right[key]) for key in left - ) - if isinstance(left, (list, tuple)): - return len(left) == len(right) and all( - snapshots_equal(a, b) for a, b in zip(left, right) - ) - if isinstance(left, float): - return math.isfinite(left) and left == right - if isinstance(left, (str, int, bool, type(None))): - return left == right - raise ValueError( - "exact adapter snapshots must use scalar, tensor, NumPy array, dict, list, or tuple values" - ) - - -def capture_rng_state(device): - return { - "local": RandomStateManager(device).capture_local().state_dict(), - "cuda_all": torch.cuda.get_rng_state_all() - if torch.cuda.is_available() - else None, - "mps": torch.mps.get_rng_state() if torch.backends.mps.is_available() else None, - } - - -def restore_rng_state(device, state): - RandomStateManager(device).restore(RandomState.from_state_dict(state["local"])) - if state["cuda_all"] is not None: - if ( - not torch.cuda.is_available() - or len(state["cuda_all"]) != torch.cuda.device_count() - ): - raise ValueError("PPO checkpoint CUDA RNG topology differs") - torch.cuda.set_rng_state_all(state["cuda_all"]) - if state["mps"] is not None: - if not torch.backends.mps.is_available(): - raise ValueError("PPO checkpoint requires the saved MPS RNG") - torch.mps.set_rng_state(state["mps"]) diff --git a/src/sequifier/rl/ppo_collector.py b/src/sequifier/rl/ppo_collector.py index 8e552ce5..e66ce17a 100644 --- a/src/sequifier/rl/ppo_collector.py +++ b/src/sequifier/rl/ppo_collector.py @@ -1,69 +1,16 @@ """One persistent environment slot; horizons cut history without resetting it.""" -import hashlib import math -from dataclasses import dataclass import torch -from sequifier.rl.distributions import CategoricalActionDistribution, action_generator +from sequifier.rl.collection import decide, stream_seed, validate_online_inputs +from sequifier.rl.distributions import action_generator from sequifier.rl.environment import validate_reset, validate_step from sequifier.rl.history import PolicyHistory from sequifier.rl.ppo_buffer import PPOBufferBuilder, PPODecision -def stream_seed(seed, stream, reset_count): - value = f"sequifier-ppo-v1:{seed}:{stream}:{reset_count}".encode() - return int.from_bytes(hashlib.blake2b(value, digest_size=8).digest(), "big") % ( - 2**63 - ) - - -@dataclass(frozen=True) -class PolicyDecision: - domain_action: dict - action: int - log_probability: float - entropy: float - value: float - - -def validate_online_inputs(actor, inputs): - inputs = inputs.to(actor.device) - actor.validate_inputs(inputs) - features, _ = actor.model_inputs(inputs) - if any( - tensor.is_floating_point() and not torch.isfinite(tensor).all() - for tensor in features.values() - ): - raise ValueError("frozen input transforms produced non-finite values") - return inputs - - -@torch.no_grad() -def decide(actor, value, inputs, generator, *, deterministic=False): - """Score action and history-only V from exactly the same causal input.""" - actor.eval() - if value is not None: - value.eval() - inputs = validate_online_inputs(actor, inputs) - distribution = CategoricalActionDistribution(actor(inputs)[:, -1]) - action = distribution.select(deterministic, generator) - estimate = ( - value(inputs) if value is not None else torch.zeros(1, device=actor.device) - ) - if not torch.isfinite(estimate).all(): - raise ValueError("non-finite PPO collection value") - index = int(action[0]) - return PolicyDecision( - {actor.config.action_column: actor.config.action_values[index]}, - index, - float(distribution.log_prob(action)[0]), - float(distribution.entropy()[0]), - float(estimate[0]), - ) - - class PPOCollector: def __init__(self, actor, value, adapter, phase, seed): self.actor = actor diff --git a/src/sequifier/rl/ppo_evaluation.py b/src/sequifier/rl/ppo_evaluation.py index a831e6d2..c9f3ce26 100644 --- a/src/sequifier/rl/ppo_evaluation.py +++ b/src/sequifier/rl/ppo_evaluation.py @@ -8,11 +8,11 @@ from contextlib import contextmanager from copy import deepcopy +from sequifier.rl.collection import decide, stream_seed, validate_online_inputs from sequifier.rl.distributions import action_generator from sequifier.rl.environment import load_online_adapter, validate_reset, validate_step from sequifier.rl.history import PolicyHistory -from sequifier.rl.ppo_checkpoint import capture_rng_state, restore_rng_state -from sequifier.rl.ppo_collector import decide, stream_seed, validate_online_inputs +from sequifier.rl.run_state import capture_rng_state, restore_rng_state class EvaluationTimeout(TimeoutError): @@ -60,6 +60,10 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): Adapters must release resources promptly from close(). """ require_evaluation_timer() + if evaluation.environment.units != getattr(actor, "units", {}): + raise ValueError( + "evaluation environment units differ from the portable actor representation" + ) saved_random = capture_rng_state(actor.device) started = time.monotonic() adapter = None @@ -171,6 +175,12 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): } if successes and all(success is not None for success in successes): result["success_rate"] = statistics.fmean(successes) + if evaluation.metric == "score_mean": + if timed_out or attempted != len(evaluation.seeds): + raise ValueError("GRPO evaluation requires every finite-horizon score") + result["score_mean"] = statistics.fmean( + episode["return"] for episode in episodes + ) result["selection_metric"] = evaluation.metric result["selection_value"] = result.get(evaluation.metric) result["selection_available"] = result["selection_value"] is not None diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py new file mode 100644 index 00000000..0e0c95a3 --- /dev/null +++ b/src/sequifier/rl/preparation.py @@ -0,0 +1,252 @@ +"""Read-only policy run preparation and whole-plan preflight validation.""" + +import hashlib +from pathlib import Path + +import torch + +from sequifier.io.iql_dataset import IQLTransitionDataset +from sequifier.io.policy_window import PolicyWindowView +from sequifier.io.preference_dataset import PreferenceData +from sequifier.model.factory import build_transformer_network +from sequifier.rl.environment import ( + adapter_identity, + load_online_adapter, + require_exact_snapshot, + require_group_start, +) +from sequifier.rl.offline_data import OfflineData +from sequifier.rl.policy import CategoricalPolicy, policy_execution_config +from sequifier.rl.ppo_evaluation import require_evaluation_timer +from sequifier.rl.registry import BUILTIN_ALGORITHMS +from sequifier.rl.run_state import ( + capture_rng_state, + make_optimization, + require_equal, + restore_rng_state, +) +from sequifier.trajectory.importers import source_paths + + +def file_hash(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def input_identity(config): + protected = set() + for source in config.sources.values(): + protected.update(source_paths(source)) + protected.update( + Path(path).resolve() + for path in ( + source.manifest, + source.metadata_config_path, + source.static_data_path, + ) + if path + ) + protected.update(Path(path) for path in config.preferences.values()) + data_identity = {str(path): file_hash(path) for path in sorted(protected)} + if config.initialize_from: + protected.add(Path(config.initialize_from)) + protected.update( + Path(phase.reference.path) + for phase in config.phases + if getattr(phase, "reference", None) is not None and phase.reference.path + ) + outputs = [ + Path(config.model_path), + Path(config.report_path), + Path(config.checkpoint_path), + ] + for i, phase in enumerate(config.phases): + directory = Path(config.phase_directory) / f"{i:03d}-{phase.name}" + outputs += [directory / "actor.pt", directory / "report.json"] + archives = [ + Path(phase.archive_directory) + for phase in config.phases + if getattr(phase, "archive_directory", None) + ] + if len(set(outputs)) != len(outputs) or len(set(archives)) != len(archives): + raise ValueError("policy outputs and phase archives must have distinct paths") + for path in outputs: + if path.is_dir() or any(p.exists() and not p.is_dir() for p in path.parents): + raise ValueError(f"output requires a file destination: {path}") + for other in protected | set(outputs): + if ( + path == other + and other in protected + or path != other + and (path in other.parents or other in path.parents) + ): + raise ValueError( + f"policy output {path} conflicts with input/output {other}" + ) + for directory in archives: + if directory.exists() and not directory.is_dir(): + raise ValueError(f"archive requires a directory: {directory}") + for other in protected | set(outputs) | (set(archives) - {directory}): + if ( + directory == other + or directory in other.parents + or other in directory.parents + ): + raise ValueError( + f"archive {directory} conflicts with input/output {other}" + ) + return data_identity + + +def resolve_normalization(policy, declaration): + real = ( + set(policy.observation_columns + policy.static_columns) + - policy.categorical_n_classes.keys() + - set(policy.categorical_columns) + ) + if declaration is None: + if real: + raise ValueError( + "online-first policy runs require frozen normalization or initialize_from" + ) + return {} + if declaration.source == "identity": + return {column: {"mean": 0.0, "std": 1.0} for column in sorted(real)} + require_equal( + sorted(real), sorted(declaration.statistics), "normalization.statistics fields" + ) + return declaration.statistics + + +def validate_declared_policy(declared, fitted, field): + expected, actual = declared.model_dump(mode="json"), fitted.model_dump(mode="json") + # Unspecified categorical codecs may be fitted once, or supplied by the incoming artifact. + for key in ("id_maps", "observation_n_classes", "static_n_classes"): + for column, value in expected.pop(key).items(): + require_equal(value, actual[key].get(column), f"{field}.{key}.{column}") + actual.pop(key) + require_equal(expected, actual, field) + + +def build_actor(config, fitted, statistics): + execution = policy_execution_config( + fitted, + config.model, + config.device, + statistics=statistics, + project_root=config.project_root, + model_name=config.model_name, + seed=config.seed, + ) + actor = CategoricalPolicy.from_network( + build_transformer_network( + execution, device=torch.device(config.device) + ).network, + execution, + config.policy_interface, + ).eval() + actor.units = dict(config.units) + actor.set_normalization_statistics(statistics) + if any( + (tensor.is_floating_point() and tensor.dtype != torch.float32) + or not torch.isfinite(tensor).all() + for tensor in actor.network.state_dict().values() + ): + raise ValueError("policy initialization requires finite FP32 network weights") + return actor + + +def prepare_data(config, policy): + return [ + entry.prepare_data(config, phase, policy) if entry.prepare_data else None + for phase in config.phases + for entry in [BUILTIN_ALGORITHMS[phase.algorithm]] + ] + + +def validate_data(config, data, actor): + policy = actor.config + for phase, item in zip(config.phases, data): + if isinstance(item, OfflineData): + item.use_config(policy) + for part in (item.training, item.validation): + part.validate_normalization(actor.interface.selected_columns_statistics) + if phase.algorithm == "iql": + transitions = IQLTransitionDataset(part) + for episode in part.episodes: + step = len(episode.actions) - 1 + successor = transitions._successor(episode, step) + if successor is not None: + inputs = PolicyWindowView(policy).successor_inputs( + episode, step, successor + ) + features, _ = actor.model_inputs(inputs.to(actor.device)) + if any( + t.is_floating_point() and not torch.isfinite(t).all() + for t in features.values() + ): + raise ValueError( + f"episode {episode.episode_id}: non-finite normalized IQL successor" + ) + elif isinstance(item, PreferenceData): + item.training.use_config(policy) + item.validation.use_config(policy) + for partition in (item.training, item.validation): + fitting = partition.fitting_dataset(config.episode_cache_size) + fitting.use_config(policy) + fitting.validate_normalization( + actor.interface.selected_columns_statistics + ) + + +def preflight_environments(config): + # Adapter probes must not change the seeded initialization or phase RNG. + device = torch.device(config.device) + saved_random = capture_rng_state(device) + try: + identities = {} + for index, phase in enumerate(config.phases): + if not hasattr(phase, "environment"): + continue + require_evaluation_timer() + for role, spec in ( + ("training", phase.environment), + ("evaluation", phase.evaluation.environment), + ): + adapter = load_online_adapter(spec) + try: + if role == "training": + if phase.algorithm == "grpo": + require_group_start( + adapter, exact=config.resume.mode == "exact" + ) + elif config.resume.mode == "exact": + require_exact_snapshot(adapter) + identities[f"{index}.{role}"] = adapter_identity(adapter) + finally: + adapter.close() + return identities + finally: + restore_rng_state(device, saved_random) + + +def preflight_optimization(config, actor, data): + """Resolve constructors and authored scheduler budgets before creating outputs.""" + for phase, part in zip(config.phases, data): + size = len(part.training) if part is not None else None + maximum = phase.optimizer_step_budget(size) + for role in ("actor", "value", "critic"): + if not hasattr(phase, f"{role}_optimizer"): + continue + scheduler = getattr(phase, f"{role}_scheduler") + if ( + scheduler.name == "OneCycleLR" + and scheduler.arguments.get("total_steps") != maximum + ): + raise ValueError( + f"phases.{phase.name}.{role}_scheduler.total_steps must equal {maximum}" + ) + make_optimization(actor, phase, role) diff --git a/src/sequifier/rl/registry.py b/src/sequifier/rl/registry.py new file mode 100644 index 00000000..e869c1a6 --- /dev/null +++ b/src/sequifier/rl/registry.py @@ -0,0 +1,51 @@ +"""One explicit built-in inventory for phase configuration and execution.""" + +from dataclasses import dataclass +from typing import Callable, get_args + +from sequifier.config.policy_plan import ( + DPOPhaseConfig, + GRPOPhaseConfig, + IQLPhaseConfig, + PolicyImitationPhaseConfig, + PolicyPhase, + PPOPhaseConfig, +) +from sequifier.io.preference_dataset import prepare_preferences +from sequifier.rl.dpo import run_dpo_phase +from sequifier.rl.grpo import run_grpo_phase +from sequifier.rl.iql import run_iql_phase +from sequifier.rl.offline_data import prepare_offline +from sequifier.rl.ppo import run_ppo_phase +from sequifier.rl.training import run_imitation_phase + + +@dataclass(frozen=True) +class BuiltinAlgorithm: + config_type: type + runtime: Callable + prepare_data: Callable | None = None + state_version: int = 1 + + +BUILTIN_ALGORITHMS = { + "policy_imitation": BuiltinAlgorithm( + PolicyImitationPhaseConfig, run_imitation_phase, prepare_offline + ), + "iql": BuiltinAlgorithm(IQLPhaseConfig, run_iql_phase, prepare_offline), + "ppo": BuiltinAlgorithm(PPOPhaseConfig, run_ppo_phase), + "dpo": BuiltinAlgorithm(DPOPhaseConfig, run_dpo_phase, prepare_preferences), + "grpo": BuiltinAlgorithm(GRPOPhaseConfig, run_grpo_phase), +} + + +def validate_registry(): + union_types = set(get_args(get_args(PolicyPhase)[0])) + registered = {entry.config_type for entry in BUILTIN_ALGORITHMS.values()} + if union_types != registered or any( + get_args(entry.config_type.model_fields["algorithm"].annotation) != (name,) + for name, entry in BUILTIN_ALGORITHMS.items() + ): + raise RuntimeError( + "built-in policy algorithm registry and discriminated union disagree" + ) diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py new file mode 100644 index 00000000..2d01cd27 --- /dev/null +++ b/src/sequifier/rl/run_state.py @@ -0,0 +1,375 @@ +"""Shared policy transactions, representation checks and phase result contracts.""" + +import hashlib +import json +import math +import uuid +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Callable, Generic, TypeVar + +import torch + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_plan import PhaseConfig +from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class +from sequifier.rl.policy import CategoricalPolicy +from sequifier.runtime.random_state import RandomState, RandomStateManager + +if TYPE_CHECKING: + from sequifier.config.train_config import PolicyRunConfig + +PhaseType = TypeVar("PhaseType", bound=PhaseConfig) +DataType = TypeVar("DataType") + + +def fingerprint(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, allow_nan=False).encode() + ).hexdigest() + + +def clone_weights(module): + return { + name: tensor.detach().cpu().clone() + for name, tensor in module.state_dict().items() + } + + +def weight_fingerprint(weights): + digest = hashlib.sha256() + for name, tensor in sorted(weights.items()): + value = tensor.detach().cpu().contiguous() + digest.update(name.encode()) + digest.update(str((value.dtype, tuple(value.shape))).encode()) + digest.update(value.reshape(-1).view(torch.uint8).numpy().tobytes()) + return digest.hexdigest() + + +def atomic_save(path, payload): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + torch.save(payload, temporary) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def write_json_atomic(path, payload): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + temporary.write_text( + json.dumps(payload, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def capture_rng_state(device): + return { + "local": RandomStateManager(device).capture_local().state_dict(), + "cuda_all": torch.cuda.get_rng_state_all() + if torch.cuda.is_available() + else None, + "mps": torch.mps.get_rng_state() if torch.backends.mps.is_available() else None, + } + + +def restore_rng_state(device, state): + RandomStateManager(device).restore(RandomState.from_state_dict(state["local"])) + if state["cuda_all"] is not None: + if ( + not torch.cuda.is_available() + or len(state["cuda_all"]) != torch.cuda.device_count() + ): + raise ValueError("policy checkpoint CUDA RNG topology differs") + torch.cuda.set_rng_state_all(state["cuda_all"]) + if state["mps"] is not None: + if not torch.backends.mps.is_available(): + raise ValueError("policy checkpoint requires the saved MPS RNG") + torch.mps.set_rng_state(state["mps"]) + + +def require_equal(expected, actual, field): + """Locate the first mismatching contract field rather than a generic failure.""" + if isinstance(expected, dict) and isinstance(actual, dict): + if expected.keys() != actual.keys(): + raise ValueError( + f"{field}: expected fields {sorted(expected)}, received {sorted(actual)}" + ) + for key in expected: + require_equal(expected[key], actual[key], f"{field}.{key}") + elif expected != actual: + raise ValueError(f"{field}: expected {expected!r}, received {actual!r}") + + +@dataclass(frozen=True) +class ActorRepresentation: + # Canonical JSON provides an immutable owned value, even for nested schemas. + canonical: str + + @classmethod + def from_actor(cls, actor, units): + return cls( + json.dumps( + { + "policy": actor.config.model_dump(mode="json"), + "model": actor.execution_config.model.model_dump(mode="json"), + "statistics": actor.interface.selected_columns_statistics, + "units": dict(getattr(actor, "units", units)), + }, + sort_keys=True, + allow_nan=False, + ) + ) + + @property + def values(self): + return json.loads(self.canonical) + + @property + def policy(self): + return CategoricalPolicyConfig.model_validate(self.values["policy"]) + + @property + def identity(self): + return hashlib.sha256(self.canonical.encode()).hexdigest() + + def validate_actor(self, actor, units, field="actor"): + require_equal(self.values, type(self).from_actor(actor, units).values, field) + + +@dataclass(frozen=True) +class PolicyPhaseResult: + index: int + name: str + algorithm: str + monitor: str + direction: str + handoff: str + selected_metric: float | None + selected_transaction: int + selected_weight_hash: str + clocks: dict + metrics: tuple[dict, ...] + provenance: dict + + def as_dict(self): + from dataclasses import asdict + + return asdict(self) + + +@dataclass(frozen=True) +class PolicyRunResult: + policy: CategoricalPolicy + phases: tuple[PolicyPhaseResult, ...] + representation: ActorRepresentation + model_path: str + report_path: str + checkpoint_path: str + + +@dataclass(frozen=True) +class PhaseContext(Generic[PhaseType, DataType]): + run: "PolicyRunConfig" + index: int + phase: PhaseType + actor: CategoricalPolicy + representation: ActorRepresentation + data: DataType + reference: CategoricalPolicy | None + reference_identity: str | None + resume_state: dict | None + commit: Callable[[dict], None] + + +class Selection: + """Strict comparisons retain the earliest finite best on ties.""" + + def __init__(self, phase, monitor, direction, state=None): + self.phase, self.monitor, self.direction = phase, monitor, direction + self.best_metric, self.best_weights, self.best_transaction = None, None, None + self.last_metric = None + if state: + self.best_metric = state["best_metric"] + self.best_weights = state["best_weights"] + self.best_transaction = state["best_transaction"] + self.last_metric = state["last_metric"] + if self.best_metric is not None and not math.isfinite(self.best_metric): + raise ValueError("checkpoint selection metric must be finite") + if (self.best_metric is None) != (self.best_weights is None): + raise ValueError( + "checkpoint best actor and metric must be recorded together" + ) + + def observe(self, actor, metric, transaction): + if metric is not None and not math.isfinite(metric): + raise ValueError(f"non-finite selection metric {self.monitor}") + self.last_metric = metric + if metric is None: + return + if self.best_metric is None or ( + metric < self.best_metric + if self.direction == "min" + else metric > self.best_metric + ): + self.best_metric = float(metric) + self.best_weights = clone_weights(actor.network) + self.best_transaction = transaction + + def state_dict(self): + return { + key: getattr(self, key) + for key in ( + "best_metric", + "best_weights", + "best_transaction", + "last_metric", + ) + } + + def finish(self, context, transaction, clocks, metrics, provenance): + actor = deepcopy(context.actor).eval() + if self.phase.handoff == "best": + if self.best_weights is None: + raise ValueError( + f"phases.{self.phase.name}: handoff best requires a finite {self.monitor}" + ) + actor.network.load_state_dict(self.best_weights, strict=True) + if self.best_transaction is None: + raise ValueError("best actor has no selected transaction") + selected, metric = self.best_transaction, self.best_metric + else: + selected, metric = transaction, self.last_metric + result = PolicyPhaseResult( + context.index, + self.phase.name, + self.phase.algorithm, + self.monitor, + self.direction, + self.phase.handoff, + metric, + selected, + weight_fingerprint(actor.network.state_dict()), + clocks, + tuple(metrics), + provenance, + ) + return actor, result + + +def make_optimization(module, phase, role="actor"): + optimizer_spec = getattr(phase, f"{role}_optimizer") + scheduler_spec = getattr(phase, f"{role}_scheduler") + optimizer = get_optimizer_class(optimizer_spec.name)( + [p for p in module.parameters() if p.requires_grad], + **{"lr": getattr(phase, f"{role}_learning_rate"), **optimizer_spec.arguments}, + ) + scheduler = get_scheduler_class(scheduler_spec.name)( + optimizer, **scheduler_spec.arguments + ) + return optimizer, scheduler + + +def apply_gradients(module, optimizer, scheduler, clip): + torch.nn.utils.clip_grad_norm_(module.parameters(), clip, error_if_nonfinite=True) + optimizer.step() + if any(not torch.isfinite(p).all() for p in module.parameters()): + raise ValueError( + "policy optimizer produced non-finite parameters; transaction was not committed" + ) + scheduler.step() + optimizer.zero_grad(set_to_none=True) + + +def actor_state( + context, optimizer, scheduler, sampler, selection, transaction, metrics, **extra +): + return { + "version": 1, + "actor": clone_weights(context.actor.network), + "actor_identity": weight_fingerprint(context.actor.network.state_dict()), + "optimizer": optimizer.state_dict(), + "scheduler": scheduler.state_dict(), + "sampler": sampler.get_state(), + "selection": selection.state_dict(), + "transaction": transaction, + "metrics": metrics, + "reference": clone_weights(context.reference.network) + if context.reference is not None + else None, + "reference_identity": context.reference_identity, + "random": capture_rng_state(context.actor.device), + **extra, + } + + +def restore_actor_state(context, optimizer, scheduler, sampler): + state = context.resume_state + if state is None: + return + if state.get("version") != 1 or state["transaction"] < 0: + raise ValueError(f"invalid {context.phase.algorithm} phase checkpoint") + if ( + not isinstance(state["transaction"], int) + or isinstance(state["transaction"], bool) + or state["transaction"] < 1 + ): + raise ValueError( + "phase checkpoint must follow a completed optimizer transaction" + ) + require_equal( + weight_fingerprint(state["actor"]), + state["actor_identity"], + "resume.phase_actor.weights", + ) + selection = state["selection"] + if ( + selection["best_transaction"] is not None + and not 1 <= selection["best_transaction"] <= state["transaction"] + ): + raise ValueError( + "checkpoint selected transaction is outside committed progress" + ) + validate_finite_state(state["optimizer"], "optimizer") + validate_finite_state(state["scheduler"], "scheduler") + if selection["best_weights"] is not None: + validate_finite_state(selection["best_weights"], "selection.best_weights") + context.actor.network.load_state_dict(state["actor"], strict=True) + optimizer.load_state_dict(state["optimizer"]) + scheduler.load_state_dict(state["scheduler"]) + sampler.set_state(state["sampler"]) + restore_rng_state(context.actor.device, state["random"]) + + +def validate_finite_state(value, field): + """Check algorithm-owned optimizer/network trees before restoring them.""" + if isinstance(value, torch.Tensor): + if not torch.isfinite(value).all(): + raise ValueError(f"non-finite checkpoint tensor: {field}") + elif isinstance(value, dict): + for key, item in value.items(): + validate_finite_state(item, f"{field}.{key}") + elif isinstance(value, (tuple, list)): + for index, item in enumerate(value): + validate_finite_state(item, f"{field}[{index}]") + elif isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"non-finite checkpoint value: {field}") + + +def validate_permutation(order, size, position): + if ( + not isinstance(position, int) + or isinstance(position, bool) + or not 0 <= position <= size + ): + raise ValueError("checkpoint sampler position is invalid") + if sorted(order) != list(range(size)): + raise ValueError("checkpoint sampler order is not the dataset permutation") diff --git a/src/sequifier/rl/snapshots.py b/src/sequifier/rl/snapshots.py new file mode 100644 index 00000000..d85d97a2 --- /dev/null +++ b/src/sequifier/rl/snapshots.py @@ -0,0 +1,44 @@ +"""Portable environment snapshot comparison.""" + +import math + +import numpy as np +import torch + + +def snapshots_equal(left, right): + """Exact adapters use torch-serializable scalar, tensor and container state.""" + if type(left) is not type(right): + return False + if isinstance(left, torch.Tensor): + return ( + left.dtype == right.dtype + and bool(torch.isfinite(left).all()) + and torch.equal(left.cpu(), right.cpu()) + ) + if isinstance(left, np.ndarray): + finite = ( + np.isfinite(left).all() + if left.dtype.kind in "biufc" + else left.dtype.kind in "US" + ) + return ( + left.dtype == right.dtype and bool(finite) and np.array_equal(left, right) + ) + if isinstance(left, np.generic): + return snapshots_equal(left.item(), right.item()) + if isinstance(left, dict): + return left.keys() == right.keys() and all( + snapshots_equal(left[key], right[key]) for key in left + ) + if isinstance(left, (list, tuple)): + return len(left) == len(right) and all( + snapshots_equal(a, b) for a, b in zip(left, right) + ) + if isinstance(left, float): + return math.isfinite(left) and left == right + if isinstance(left, (str, int, bool, type(None))): + return left == right + raise ValueError( + "exact adapter snapshots must use scalar, tensor, NumPy array, dict, list, or tuple values" + ) diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index d08dfb72..a33cec54 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -1,149 +1,122 @@ -"""Policy datasets adapted to Sequifier's ordinary TrainingRun and TrainingEngine.""" - -from dataclasses import dataclass +"""Phase-local categorical behavior cloning, including return conditioning.""" import torch -from sequifier.artifacts.model_artifact import load_execution_model -from sequifier.config.train_config import ( - EvaluationMonitorSpecModel, - PolicyTrainConfig, - ResolvedTrainingPhase, - ResolvedTrainingSource, +from sequifier.config.policy_plan import PolicyImitationPhaseConfig +from sequifier.io.batch import collate_sequence_batches +from sequifier.io.policy_window import action_global_ids, action_target +from sequifier.rl.offline_data import OfflineData +from sequifier.rl.run_state import ( + PhaseContext, + Selection, + actor_state, + apply_gradients, + make_optimization, + restore_actor_state, + validate_permutation, ) -from sequifier.integration.callbacks import IntegrationManager -from sequifier.io.policy_dataset import PolicySequenceDataset -from sequifier.rl.policy import CategoricalPolicy, policy_execution_config -from sequifier.runtime.builder import RunBuilder, TrainingRun -from sequifier.runtime.context import ExecutionEnvironment -from sequifier.training.engine import TrainingEngine -from sequifier.training.runtime import DatasetPartRuntime, PartLoaderFactory -@dataclass(frozen=True) -class PolicyTrainingResult: - policy: CategoricalPolicy - metrics: tuple[dict[str, float], ...] - best_epoch: int - optimizer_steps: int - training_episode_ids: tuple[str, ...] - validation_episode_ids: tuple[str, ...] +def action_nll(actor, batch): + batch = batch.to(actor.device) + labels = batch.targets[action_target(actor.config)][:, -1] + ids = torch.tensor(action_global_ids(actor.config), device=actor.device) + actions = (labels[:, None] == ids[None, :]).long().argmax(dim=-1) + return -actor.evaluate_actions(batch.model_inputs, actions).float() -def train_policy( - training: PolicySequenceDataset, - validation: PolicySequenceDataset, - run_config: PolicyTrainConfig, - *, - integrations: IntegrationManager | None = None, -) -> PolicyTrainingResult: - phase = run_config.training_plan.phases[0] - if phase.algorithm != "policy_imitation": - raise ValueError("train_policy requires a policy_imitation phase") - if not len(training) or not len(validation): - raise ValueError("policy training and validation partitions must be nonempty") - if training.config != validation.config or training.config != run_config.policy: - raise ValueError( - "dataset policies must match the resolved training configuration" - ) - if set(training.episode_ids) & set(validation.episode_ids): - raise ValueError( - "training and validation partitions must not share episode IDs" +@torch.no_grad() +def evaluate_imitation(actor, dataset, batch_size): + actor.eval() + total = 0.0 + for start in range(0, len(dataset), batch_size): + batch = collate_sequence_batches( + [dataset[i] for i in range(start, min(start + batch_size, len(dataset)))] ) - fitted = training.fit_categorical_config() - for dataset in (training, validation): - dataset.use_config(fitted) - name, part_name = fitted.interface_name, "episodes" - source = ResolvedTrainingSource( - source=f"{name}.{part_name}", dataset=name, part=part_name - ) - config = policy_execution_config( - fitted, - run_config.model, - run_config.device, - global_training=run_config.global_training, - statistics=training.normalization_statistics(), - project_root=run_config.project_root, - model_name=run_config.model_name, - seed=run_config.seed, - training_plan=[ - ResolvedTrainingPhase( - name=phase.name, - algorithm="policy_imitation", - epochs=phase.epochs, - mode="sequential", - sources=[source], - ) - ], - evaluation_sources=[source], - evaluation_monitor=EvaluationMonitorSpecModel( - source=source.source, metric="loss", mode="min" - ), - ) + total += float(action_nll(actor, batch).sum()) + return total / len(dataset) - def configure(run: TrainingRun, resuming: bool) -> None: - run.datasets.resolve(name).parts[part_name] = DatasetPartRuntime( - part_name, - PartLoaderFactory( - config, - name, - part_name, - {"training": training, "validation": validation}, - ), - ) - if not resuming and run_config.initialize_from is not None: - source_network, source_config, _ = load_execution_model( - run_config.initialize_from, device=config.device - ) - if ( - source_config.model.backbone.architecture - != config.model.backbone.architecture - ): - raise ValueError( - "initialize_from requires the same backbone architecture" - ) - run.network.backbone.load_state_dict( - source_network.backbone.state_dict(), strict=True - ) - device = torch.device(config.device) - cuda_devices = ( - [device.index if device.index is not None else torch.cuda.current_device()] - if device.type == "cuda" - else [] - ) - with torch.random.fork_rng(devices=cuda_devices): - torch.manual_seed(config.seed) - run = RunBuilder().build( - config, - ExecutionEnvironment(0, 0, 1, device, False), - integrations or IntegrationManager(), - configure=configure, - ) - try: - TrainingEngine().run(run) - if run.state.best_model_state_dict is None: - raise ValueError( - "policy training did not produce finite validation likelihood" - ) - run.network.load_state_dict(run.state.best_model_state_dict, strict=True) - policy = CategoricalPolicy.from_network(run.network, config, name) - policy.eval() - metrics = tuple( +def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, OfflineData]): + actor, phase, data = context.actor.eval(), context.phase, context.data + optimizer, scheduler = make_optimization(actor, phase) + sampler = torch.Generator().manual_seed(context.run.seed + context.index) + restore_actor_state(context, optimizer, scheduler, sampler) + state = context.resume_state or {} + selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) + transaction = state.get("transaction", 0) + epoch, position = state.get("epoch", 0), state.get("position", 0) + order, metrics = state.get("order"), list(state.get("metrics", [])) + training = data.training + size = phase.batch_size * phase.accumulation_steps + per_epoch = (len(training) + size - 1) // size + if ( + not isinstance(epoch, int) + or not 0 <= epoch <= phase.epochs + or transaction != epoch * per_epoch + (position + size - 1) // size + ): + raise ValueError("imitation checkpoint epoch/transaction clocks disagree") + if order is not None: + validate_permutation(order, len(training), position) + if position >= len(training) or position % size: + raise ValueError( + "imitation checkpoint is not a complete accumulation boundary" + ) + elif position != 0: + raise ValueError("imitation checkpoint has a sampler position without an order") + if epoch == phase.epochs and (order is not None or position): + raise ValueError("completed imitation checkpoint has a live sampler order") + while epoch < phase.epochs: + if order is None: + order = torch.randperm(len(training), generator=sampler).tolist() + group = order[position : position + size] + optimizer.zero_grad(set_to_none=True) + total = 0.0 + for start in range(0, len(group), phase.batch_size): + indices = group[start : start + phase.batch_size] + batch = collate_sequence_batches([training[i] for i in indices]) + loss = action_nll(actor, batch).sum() / len(group) + if not torch.isfinite(loss): + raise ValueError("non-finite imitation action likelihood") + loss.backward() + total += float(loss.detach()) + apply_gradients(actor, optimizer, scheduler, phase.gradient_clip) + transaction += 1 + position += len(group) + if position == len(training): + epoch += 1 + position, order = 0, None + final = epoch == phase.epochs + if transaction % phase.evaluation_interval == 0 or final: + score = evaluate_imitation(actor, data.validation, phase.batch_size) + selection.observe(actor, score, transaction) + metrics.append( { - "epoch": float(item["epoch"] - 1), - "training_nll": item["training"][name], - "validation_nll": item["validation"][source.source], + "transaction": transaction, + "epochs_completed": epoch, + "training_action_nll": total, + phase.monitor: score, } - for item in run.state.epoch_metrics ) - return PolicyTrainingResult( - policy, - metrics, - run.state.best_epoch - 1, - run.optimization.optimizer_step, - tuple(training.episode_ids), - tuple(validation.episode_ids), + if transaction % phase.checkpoint_interval == 0 or final: + context.commit( + actor_state( + context, + optimizer, + scheduler, + sampler, + selection, + transaction, + metrics, + epoch=epoch, + position=position, + order=order, + ) ) - finally: - run.distributed.finalize() + return selection.finish( + context, + transaction, + {"optimizer_steps": transaction, "epochs": epoch}, + metrics, + {"data": data.contract}, + ) diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index 01eb6297..2c6808d8 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -1,4 +1,4 @@ -"""Local categorical policy learning, online collection and execution.""" +"""Local categorical policy learning, inference, and environment contracts.""" from sequifier.config.policy_config import ( CategoricalPolicyConfig, @@ -6,21 +6,25 @@ ReturnDefinition, ) from sequifier.config.policy_infer_config import PolicyInferenceConfig -from sequifier.config.policy_train_config import ( - PolicyTableColumns, - SequenceSourceConfig, -) -from sequifier.config.sequence_import_config import SequenceImportConfig -from sequifier.config.train_config import PolicyTrainConfig, PPOTrainConfig -from sequifier.config.training_plan import ( +from sequifier.config.policy_plan import ( + DPOPhaseConfig, + FrozenNormalizationConfig, + GRPOEvaluationConfig, + GRPOPhaseConfig, IQLPhaseConfig, OnlineEnvironmentConfig, + OnlineEvaluationConfig, PolicyImitationPhaseConfig, - PPOEvaluationConfig, - PPONormalizationConfig, + PolicyResumeConfig, PPOPhaseConfig, - PPOResumeConfig, + ReferenceConfig, +) +from sequifier.config.policy_train_config import ( + PolicyTableColumns, + SequenceSourceConfig, ) +from sequifier.config.sequence_import_config import SequenceImportConfig +from sequifier.config.train_config import PolicyRunConfig from sequifier.io.batch import SequifierBatch, SequifierInputs, collate_sequence_batches from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.io.policy_window import PolicyWindowView, action_index @@ -31,70 +35,84 @@ masked_action_nll_loss, ) from sequifier.rl.environment import ( + EnvironmentCapabilities, OnlineEnvironmentAdapter, OnlineResetResult, OnlineStepResult, SeededAdapterBridge, ) -from sequifier.rl.iql import IQLTrainingResult, run_iql_training from sequifier.rl.learner import run_policy_training from sequifier.rl.policy import CategoricalPolicy -from sequifier.rl.ppo import PPOTrainingResult, run_ppo_training from sequifier.rl.ppo_buffer import load_collection_record from sequifier.rl.ppo_evaluation import evaluate_policy +from sequifier.rl.run_state import ( + ActorRepresentation, + PolicyPhaseResult, + PolicyRunResult, +) from sequifier.rl.runtime import PolicyExecutor, collect_policy_rollout -from sequifier.rl.training import PolicyTrainingResult, train_policy from sequifier.trajectory.importers import import_sequences, source_episodes +from sequifier.trajectory.preferences import ( + PreferenceManifest, + PreferencePair, + PreferenceSpan, + load_preferences, +) from sequifier.trajectory.returns import remaining_return, return_to_go __all__ = [ - "SequifierInputs", - "SequifierBatch", - "SequenceEpisode", - "TransitionColumns", - "PolicyWindowView", - "PolicySchemaConfig", - "PolicySequenceDataset", - "SequenceSourceConfig", - "PolicyTableColumns", - "SequenceImportConfig", - "source_episodes", - "import_sequences", - "PolicyInferenceConfig", - "PolicyTrainConfig", - "PPOTrainConfig", - "PPOPhaseConfig", - "OnlineEnvironmentConfig", - "PPOEvaluationConfig", - "PPONormalizationConfig", - "PPOResumeConfig", + "ActorRepresentation", + "CategoricalActionDistribution", + "CategoricalPolicy", + "CategoricalPolicyConfig", + "DPOPhaseConfig", + "EnvironmentCapabilities", + "FrozenNormalizationConfig", + "GRPOPhaseConfig", + "GRPOEvaluationConfig", + "IQLPhaseConfig", "OnlineEnvironmentAdapter", + "OnlineEnvironmentConfig", + "OnlineEvaluationConfig", "OnlineResetResult", "OnlineStepResult", - "SeededAdapterBridge", - "PPOTrainingResult", - "run_ppo_training", - "evaluate_policy", - "load_collection_record", + "PolicyExecutor", "PolicyImitationPhaseConfig", - "IQLPhaseConfig", - "run_policy_training", - "run_iql_training", - "IQLTrainingResult", - "CategoricalPolicyConfig", + "PolicyInferenceConfig", + "PolicyPhaseResult", + "PolicyResumeConfig", + "PolicyRunConfig", + "PolicyRunResult", + "PolicySchemaConfig", + "PolicySequenceDataset", + "PolicyTableColumns", + "PolicyWindowView", + "PPOPhaseConfig", + "PreferenceManifest", + "PreferencePair", + "PreferenceSpan", + "ReferenceConfig", "ReturnDefinition", - "CategoricalActionDistribution", - "CategoricalPolicy", - "PolicyExecutor", - "PolicyTrainingResult", + "SeededAdapterBridge", + "SequenceEpisode", + "SequenceImportConfig", + "SequenceSourceConfig", + "SequifierBatch", + "SequifierInputs", + "TransitionColumns", "action_index", + "collate_sequence_batches", "collect_policy_rollout", + "evaluate_policy", + "import_sequences", + "load_collection_record", "load_policy", + "load_preferences", "masked_action_nll_loss", "remaining_return", "return_to_go", + "run_policy_training", "save_policy", + "source_episodes", "split_by_episode", - "collate_sequence_batches", - "train_policy", ] diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index b54aa322..6a8cc732 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -142,11 +142,7 @@ def _construct_loaders(self, run: TrainingRun) -> None: for dataset in run.datasets.values(): for part_name, part in dataset.parts.items(): part.loader("training") - configured = dataset.config.parts.get(part_name) - if "validation" in part.loaders or ( - configured is not None - and configured.validation_data_path is not None - ): + if dataset.config.parts[part_name].validation_data_path is not None: part.loader("validation") def build( @@ -154,8 +150,6 @@ def build( config: Any, execution: ExecutionEnvironment, integrations: IntegrationManager, - *, - configure: Callable[[TrainingRun, bool], None] | None = None, ) -> TrainingRun: strategy = self._strategy(config, execution) random_manager = RandomStateManager(execution.device) @@ -291,8 +285,6 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: loader_state=loader_state, context=context, ) - if configure is not None: - configure(run, loaded is not None) self._construct_loaders(run) if loaded is not None: restorer.restore_runtime(loaded, run) diff --git a/src/sequifier/sequifier.py b/src/sequifier/sequifier.py index 7f4f058f..c75539e7 100644 --- a/src/sequifier/sequifier.py +++ b/src/sequifier/sequifier.py @@ -64,7 +64,10 @@ def setup_parser() -> ArgumentParser: parser_preprocess = subparsers.add_parser( "preprocess", help="Run the preprocessing step" ) - parser_train = subparsers.add_parser("train", help="Run the training step") + parser_train = subparsers.add_parser( + "train", + help="Train supervised models or ordered imitation/IQL/PPO/DPO/GRPO policy phases", + ) parser_infer = subparsers.add_parser("infer", help="Run the inference step") parser_hyperparameter_search = subparsers.add_parser( "hyperparameter-search", help="Run hyperparamter search" diff --git a/src/sequifier/train.py b/src/sequifier/train.py index a2b9c2b8..713c892d 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -19,11 +19,8 @@ ModelArtifact, load_execution_model, ) +from sequifier.config.train_config import PolicyRunConfig # noqa: E402 from sequifier.config.train_config import load_train_config # noqa: E402 -from sequifier.config.train_config import ( # noqa: E402 - PolicyTrainConfig, - PPOTrainConfig, -) from sequifier.config.train_config import ( # noqa: E402 ResolvedSequifierConfig as TrainModel, ) @@ -166,7 +163,7 @@ def _mp_train_worker_wrapper( @beartype def run_training( - config: TrainModel | PolicyTrainConfig | PPOTrainConfig, + config: TrainModel | PolicyRunConfig, *, integration_specs: tuple[IntegrationSpec, ...] = (), integration_instances: tuple[Any, ...] = (), @@ -174,7 +171,7 @@ def run_training( ) -> None: """Launch canonical service-composed training locally or across workers.""" - if isinstance(config, (PolicyTrainConfig, PPOTrainConfig)): + if isinstance(config, PolicyRunConfig): if integration_specs or integration_instances or semantic_optimizer_grouping: raise ValueError( "policy training does not yet support custom integrations or parameter grouping" @@ -184,9 +181,7 @@ def run_training( configure_logger(config.project_root, config.model_name, 0) configure_determinism( config.seed, - config.enforce_determinism - if isinstance(config, PPOTrainConfig) - else config.global_training.enforce_determinism, + config.enforce_determinism, ) run_policy_training(config) return diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index d595ed11..ef706314 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -46,11 +46,6 @@ def validate_for_restore( model_config=checkpoint.model.model_config.values, ) current = self._resume_settings(current_config) - if current_config.global_training.training_objective == "policy": - for contract in (saved, current): - for phase in contract["training_plan"]: - phase.pop("epochs") - contract["resume_training"].pop("num_workers") mismatches = self._mismatches(saved, current) if mismatches: raise ValueError( diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index b88fb9bc..252b2db6 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -174,7 +174,6 @@ def _update_monitor(self, run: Any, result: EvaluationResult) -> tuple[float, bo ) if improved: run.state.best_validation_loss = value - run.state.best_epoch = run.state.epoch run.state.epochs_without_improvement = 0 state = run.distributed.capture_model_state(run.network) if run.distributed.rank == 0: @@ -294,7 +293,7 @@ def flush_metrics( if window is None: return dataset = run.datasets.resolve(dataset_name) - total, targets = run.loss.finalize_reduced_accounting( + total, targets = run.loss.finalize_accounting( window["sums"], window["count"], dataset, allow_empty=True ) run.metrics.record_training( @@ -502,22 +501,6 @@ def flush_metrics( dataset.parts[runtime_batch.part].iterator_positions[ "training" ] = state.iterator_positions[position_key] - sums, count_tensor = run.loss.reduce_accounting( - loss.accounting_sums, - loss.accounting_count, - dataset.config.interface.target_columns, - ) - total, _ = run.loss.finalize_reduced_accounting( - sums, count_tensor, dataset, allow_empty=True - ) - count = float(count_tensor.item()) - state.training_sums[dataset.name] = ( - state.training_sums.get(dataset.name, 0.0) - + float(total.item()) * count - ) - state.training_counts[dataset.name] = ( - state.training_counts.get(dataset.name, 0.0) + count - ) batch_committed = True batch_in_progress = False window = metric_windows.setdefault( @@ -525,17 +508,21 @@ def flush_metrics( { "sums": { name: value.new_zeros(()) - for name, value in sums.items() + for name, value in loss.accounting_sums.items() }, - "count": count_tensor.new_zeros(()), + "count": loss.accounting_count.new_zeros(()).to( + next(iter(loss.accounting_sums.values())).dtype + ), "batches": 0, "seconds": 0.0, "part": runtime_batch.part, }, ) - for name, value in sums.items(): + for name, value in loss.accounting_sums.items(): window["sums"][name] += value - window["count"] += count_tensor + window["count"] += loss.accounting_count.to( + window["count"].dtype + ) window["batches"] += 1 window["seconds"] += time.perf_counter() - started if window["part"] != runtime_batch.part: @@ -658,19 +645,6 @@ def flush_metrics( state.iterator_positions = {} state.source_scheduler_state = scheduler.state_dict() self._update_monitor(run, evaluation) - state.epoch_metrics.append( - { - "epoch": current_epoch, - "training": { - name: total / state.training_counts[name] - for name, total in state.training_sums.items() - if state.training_counts[name] - }, - "validation": evaluation.total_losses, - } - ) - state.training_sums.clear() - state.training_counts.clear() boundary = capture_runtime_boundary(run) boundary_identity = last_identity if current_epoch % config.global_training.save_interval_epochs == 0: diff --git a/src/sequifier/training/loss.py b/src/sequifier/training/loss.py index 4aac18e1..9be8562a 100644 --- a/src/sequifier/training/loss.py +++ b/src/sequifier/training/loss.py @@ -83,10 +83,6 @@ def prepare_batch( key: value.to(device, non_blocking=True) for key, value in batch.metadata.items() } - if interface.policy is not None: - from sequifier.io.policy_window import normalize_policy_features - - features = normalize_policy_features(features, metadata, interface) features, targets, metadata = dataset.objective.prepare_batch( features, targets, metadata, eval_seed=eval_seed ) @@ -284,8 +280,6 @@ def calculate( backward_loss: Tensor = total + network.regularization_loss( dataset.interface_name ) - if interface.policy is not None and not torch.isfinite(backward_loss): - raise ValueError("policy action likelihood became non-finite") accounting_dtype = ( torch.float32 if backward_loss.device.type == "mps" else torch.float64 ) diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index ec11891e..40aa3c1d 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -22,7 +22,7 @@ dataset_part_view, ) from sequifier.helpers import normalize_path -from sequifier.io.batch import SequifierBatch, collate_sequence_batches +from sequifier.io.batch import SequifierBatch from sequifier.io.sequifier_dataset_from_file import SequifierDatasetFromFile from sequifier.io.sequifier_dataset_from_folder_parquet import ( SequifierDatasetFromFolderParquet, @@ -66,7 +66,6 @@ class PartLoaderFactory: config: ResolvedSequifierConfig dataset_name: str part_name: str - dataset_splits: dict[str, Any] = field(default_factory=dict) loaders: dict[Literal["training", "validation"], DataLoader] = field( default_factory=dict, init=False, @@ -82,75 +81,67 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: if split in self.loaders: return self.loaders[split] - sequence_dataset = bool(self.dataset_splits) - if sequence_dataset: - dataset = self.dataset_splits[split] - else: - dataset_config: Any = dataset_part_view( - self.config, self.dataset_name, self.part_name + dataset_config: Any = dataset_part_view( + self.config, self.dataset_name, self.part_name + ) + if split != "training" or not _training_plan_uses_part( + self.config, self.dataset_name, self.part_name + ): + dataset_config = replace( + dataset_config, + curriculum_training=False, + curriculum_column=None, ) - if split != "training" or not _training_plan_uses_part( - self.config, self.dataset_name, self.part_name - ): - dataset_config = replace( - dataset_config, - curriculum_training=False, - curriculum_column=None, - ) - part = self.config.dataset_training[self.dataset_name].parts[self.part_name] - path = ( - part.training_data_path - if split == "training" - else part.validation_data_path + part = self.config.dataset_training[self.dataset_name].parts[self.part_name] + path = ( + part.training_data_path + if split == "training" + else part.validation_data_path + ) + if path is None: + raise ValueError( + f"Part {self.dataset_name}.{self.part_name} has no {split} path" ) - if path is None: + shuffle = split == "training" + if part.storage_form == "file": + if global_spec.distributed: raise ValueError( - f"Part {self.dataset_name}.{self.part_name} has no {split} path" - ) - shuffle = split == "training" - if part.storage_form == "file": - if global_spec.distributed: - raise ValueError( - "Distributed training is not supported with single-file parts" - ) - split_paths = [ - normalize_path(split_path, self.config.project_root) - for split_path in part.metadata.split_paths - ] - if path in split_paths: - dataset_config = replace( - dataset_config, - storage_layout=part.metadata.layout_for_split( - split_paths.index(path) - ), - ) - dataset = SequifierDatasetFromFile( - path, dataset_config, shuffle=shuffle - ) - elif global_spec.read_format == "pt": - cls = ( - SequifierDatasetFromFolderPt - if global_spec.load_full_data_to_ram - else SequifierDatasetFromFolderPtLazy - ) - dataset = cls(path, dataset_config, shuffle=shuffle) - elif global_spec.read_format == "parquet": - cls = ( - SequifierDatasetFromFolderParquet - if global_spec.load_full_data_to_ram - else SequifierDatasetFromFolderParquetLazy + "Distributed training is not supported with single-file parts" ) - dataset = cls(path, dataset_config, shuffle=shuffle) - else: - raise ValueError( - f"Folder parts do not support read_format={global_spec.read_format!r}" + split_paths = [ + normalize_path(split_path, self.config.project_root) + for split_path in part.metadata.split_paths + ] + if path in split_paths: + dataset_config = replace( + dataset_config, + storage_layout=part.metadata.layout_for_split( + split_paths.index(path) + ), ) + dataset = SequifierDatasetFromFile(path, dataset_config, shuffle=shuffle) + elif global_spec.read_format == "pt": + cls = ( + SequifierDatasetFromFolderPt + if global_spec.load_full_data_to_ram + else SequifierDatasetFromFolderPtLazy + ) + dataset = cls(path, dataset_config, shuffle=shuffle) + elif global_spec.read_format == "parquet": + cls = ( + SequifierDatasetFromFolderParquet + if global_spec.load_full_data_to_ram + else SequifierDatasetFromFolderParquetLazy + ) + dataset = cls(path, dataset_config, shuffle=shuffle) + else: + raise ValueError( + f"Folder parts do not support read_format={global_spec.read_format!r}" + ) generator = self.generators.get(split) if generator is None: generator = torch.Generator().manual_seed( self.config.seed - if sequence_dataset - else self.config.seed + (10_001 if split == "training" else 10_002) + list(self.config.dataset_training).index(self.dataset_name) * 101 + list(self.config.dataset_training[self.dataset_name].parts).index( @@ -160,18 +151,12 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: self.generators[split] = generator loader = DataLoader( dataset, - batch_size=global_spec.batch_size if sequence_dataset else None, - sampler=dataset.sampler(generator) - if sequence_dataset and split == "training" - else None, - collate_fn=collate_sequence_batches if sequence_dataset else None, + batch_size=None, + sampler=None, num_workers=global_spec.num_workers, - pin_memory=not sequence_dataset - and self.config.device not in {"mps", "cpu"}, - prefetch_factor=4 - if global_spec.num_workers > 0 and not sequence_dataset - else None, - persistent_workers=global_spec.num_workers > 0 and not sequence_dataset, + pin_memory=self.config.device not in {"mps", "cpu"}, + prefetch_factor=4 if global_spec.num_workers > 0 else None, + persistent_workers=global_spec.num_workers > 0, generator=generator, ) self.loaders[split] = loader @@ -185,7 +170,6 @@ class DatasetPartRuntime: iterator_positions: dict[str, int] = field(default_factory=dict) iterator_start_states: dict[str, torch.Tensor] = field(default_factory=dict) _restore_from_iterator_start: set[str] = field(default_factory=set) - data_contract: dict[str, Any] | None = None @property def loaders(self) -> dict[Literal["training", "validation"], DataLoader]: @@ -199,13 +183,6 @@ def loader_generators( @property def source_metadata(self) -> dict[str, Any]: - if self.data_contract is not None: - return self.data_contract - if self.factory.dataset_splits: - return { - split: dataset.data_contract - for split, dataset in self.factory.dataset_splits.items() - } part = self.factory.config.dataset_training[self.factory.dataset_name].parts[ self.name ] @@ -230,7 +207,6 @@ def iter_loader(self, split: Literal["training", "validation"]) -> Iterator[Any] def loader_state_dict(self) -> dict[str, Any]: return { - "data_contract": self.data_contract, "generators": { split: generator.get_state() for split, generator in self.factory.generators.items() @@ -245,13 +221,6 @@ def load_loader_state_dict( *, resume_from_iterator_start: bool, ) -> None: - if ( - self.data_contract is not None - and state.get("data_contract") != self.data_contract - ): - raise ValueError( - "Dataset content or episode order changed since the checkpoint" - ) for split, generator_state in state.get("generators", {}).items(): generator = self.factory.generators.get(split) if generator is None: diff --git a/src/sequifier/training/state.py b/src/sequifier/training/state.py index 340f1950..ea0c9893 100644 --- a/src/sequifier/training/state.py +++ b/src/sequifier/training/state.py @@ -33,10 +33,6 @@ class RunState: best_validation_loss: float = float("inf") epochs_without_improvement: int = 0 best_model_state_dict: dict[str, Tensor] | None = None - best_epoch: int = 0 - epoch_metrics: list[dict[str, Any]] = field(default_factory=list) - training_sums: dict[str, float] = field(default_factory=dict) - training_counts: dict[str, float] = field(default_factory=dict) backbone_parent_revision_id: str | None = None def snapshot(self) -> RunStateSnapshot: diff --git a/src/sequifier/trajectory/preferences.py b/src/sequifier/trajectory/preferences.py new file mode 100644 index 00000000..aa97c2cb --- /dev/null +++ b/src/sequifier/trajectory/preferences.py @@ -0,0 +1,52 @@ +"""Versioned reward-free preference manifests over contiguous decision spans.""" + +import json +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class PreferenceSpan(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + source: str = Field(min_length=1) + episode_id: str = Field(min_length=1) + start: int = Field(default=0, ge=0, strict=True) + end: int = Field(gt=0, strict=True) + + @model_validator(mode="after") + def bounds(self): + if self.end <= self.start: + raise ValueError( + "preference spans require start < end (zero-based, end-exclusive)" + ) + return self + + +class PreferencePair(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + pair_id: str = Field(min_length=1) + context_id: str = Field(min_length=1) + group_id: str | None = None + chosen: PreferenceSpan + rejected: PreferenceSpan + + +class PreferenceManifest(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + format: Literal["sequifier.preference-pairs"] + version: Literal[1] + pairs: list[PreferencePair] = Field(min_length=1) + + @model_validator(mode="after") + def unique_ids(self): + ids = [pair.pair_id for pair in self.pairs] + if len(ids) != len(set(ids)): + raise ValueError("preference pair IDs must be unique") + return self + + +def load_preferences(path): + return PreferenceManifest.model_validate( + json.loads(Path(path).read_text(encoding="utf-8")) + ) From 1137e732eb8317299ef621790e0bb5db6fa7547d Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 18:19:40 +0100 Subject: [PATCH 18/71] cleanup --- documentation/policy-learning.md | 18 +- src/sequifier/config/policy_infer_config.py | 4 +- src/sequifier/config/policy_optimization.py | 58 ++ src/sequifier/config/policy_plan.py | 156 +++--- src/sequifier/config/train_config.py | 51 +- src/sequifier/config/training_plan.py | 31 +- src/sequifier/consolidated-docs.md | 18 +- src/sequifier/fingerprints.py | 10 + src/sequifier/io/episode_corpus.py | 154 ++++++ src/sequifier/io/iql_dataset.py | 10 +- src/sequifier/io/partitioning.py | 35 ++ src/sequifier/io/policy_dataset.py | 249 ++++----- src/sequifier/io/preference_dataset.py | 127 ++--- src/sequifier/rl/collection.py | 19 +- src/sequifier/rl/dpo.py | 26 +- src/sequifier/rl/grpo.py | 3 +- src/sequifier/rl/grpo_collector.py | 4 +- src/sequifier/rl/grpo_update.py | 4 +- src/sequifier/rl/iql.py | 25 +- src/sequifier/rl/learner.py | 257 ++------- src/sequifier/rl/models.py | 6 +- src/sequifier/rl/offline_data.py | 37 +- src/sequifier/rl/policy.py | 28 +- src/sequifier/rl/ppo.py | 9 +- src/sequifier/rl/ppo_collector.py | 9 +- src/sequifier/rl/ppo_update.py | 15 +- src/sequifier/rl/preparation.py | 72 +-- src/sequifier/rl/run_checkpoint.py | 300 +++++++++++ src/sequifier/rl/run_state.py | 70 +-- src/sequifier/rl/scoring.py | 42 ++ src/sequifier/rl/training.py | 9 +- src/sequifier/trajectory/importers.py | 130 +++-- tests/conftest.py | 6 + tests/unit/test_policy_architecture.py | 560 ++++++++++++++++++++ 34 files changed, 1707 insertions(+), 845 deletions(-) create mode 100644 src/sequifier/config/policy_optimization.py create mode 100644 src/sequifier/fingerprints.py create mode 100644 src/sequifier/io/episode_corpus.py create mode 100644 src/sequifier/io/partitioning.py create mode 100644 src/sequifier/rl/run_checkpoint.py create mode 100644 src/sequifier/rl/scoring.py create mode 100644 tests/conftest.py create mode 100644 tests/unit/test_policy_architecture.py diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 80d6c1e7..ebb782fb 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -366,11 +366,19 @@ environment actions. Sealed buffers are consumed once; stale generations fail. ## Checkpoints, resume, and artifacts -The only policy-training checkpoint is `sequifier_policy_run`, format version 1. -It contains the resolved plan/fingerprint, immutable representation, data and -adapter identities, run RNG, current phase/index/status, completed results and -selected actors, algorithm/version and phase-owned state, and the last committed -transaction. DPO/GRPO reference weights are embedded. Optimizer, scheduler, +Named sources share indexed episode storage across phases and partitions. Table +sources are parsed once into temporary sorted Parquet storage with direct episode +row-group access. The episode cache stays bounded; partition views retain metadata +and validated transition indices. Imitation shuffles episodes and then decisions +within each episode to preserve storage locality. + +The only policy-training checkpoint is `sequifier_policy_run`, format version 2. +Completed phase results determine the phase boundary; an optional live phase +state records the latest committed transaction. Actor snapshots are stored once +per weight fingerprint and remain embedded for self-contained recovery. +It contains the resolved plan, immutable representation and its fingerprint, data +and adapter identities, run RNG, completed results, actor snapshots, and optional +versioned phase state. DPO/GRPO reference weights are embedded. Optimizer, scheduler, sampler, selection, evaluation and phase counters remain phase-local. Writes use a temporary file and atomic replacement at completed updates or diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 99da3bb0..53a7e017 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -1,6 +1,5 @@ """Inference options for policy interfaces in ordinary portable model artifacts.""" -from pathlib import Path from typing import Literal from pydantic import Field, PrivateAttr, model_validator @@ -8,6 +7,7 @@ from sequifier.artifacts.model_artifact import ModelArtifact, read_execution_artifact from sequifier.config.policy_config import PolicyConfigBase from sequifier.config.rollout_config import RolloutConfig +from sequifier.helpers import normalize_path class PolicyInferenceConfig(PolicyConfigBase): @@ -67,7 +67,7 @@ def policy_inference_config( continue if path not in artifacts: artifacts[path] = read_execution_artifact( - Path(values.get("project_root", ".")) / path + normalize_path(path, values.get("project_root", ".")) ) artifact = artifacts[path] for name, interface in artifact.model_config.values["interfaces"].items(): diff --git a/src/sequifier/config/policy_optimization.py b/src/sequifier/config/policy_optimization.py new file mode 100644 index 00000000..73bee2ce --- /dev/null +++ b/src/sequifier/config/policy_optimization.py @@ -0,0 +1,58 @@ +"""Concrete optimizer ownership and scheduler budgets for policy phases.""" + +from dataclasses import dataclass + +from sequifier.config.components import ComponentSpec +from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class + + +def default_optimizer(): + return ComponentSpec(name="Adam") + + +def default_scheduler(): + return ComponentSpec(name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1}) + + +@dataclass(frozen=True) +class OptimizationPlan: + optimizer: ComponentSpec + scheduler: ComponentSpec + learning_rate: float + gradient_clip: float + step_budget: int | None + + def __post_init__(self): + if self.optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"} or { + "lr", + "params", + }.intersection(self.optimizer.arguments): + raise ValueError( + "policy optimizers require a supported optimizer and phase learning rate" + ) + if ( + self.scheduler.name + in {"ReduceLROnPlateau", "SequentialLR", "ChainedScheduler"} + or "optimizer" in self.scheduler.arguments + ): + raise ValueError( + "policy schedulers must advance independently on optimizer steps" + ) + if ( + self.step_budget is not None + and self.scheduler.name == "OneCycleLR" + and self.scheduler.arguments.get("total_steps") != self.step_budget + ): + raise ValueError( + f"OneCycleLR.total_steps must equal the maximum authored optimizer-step budget ({self.step_budget})" + ) + + def build(self, module): + optimizer = get_optimizer_class(self.optimizer.name)( + [p for p in module.parameters() if p.requires_grad], + lr=self.learning_rate, + **self.optimizer.arguments, + ) + return optimizer, get_scheduler_class(self.scheduler.name)( + optimizer, **self.scheduler.arguments + ) diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 76d27c85..fb55c16c 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -8,6 +8,11 @@ from sequifier.config.components import ComponentSpec from sequifier.config.identifiers import identifier from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_optimization import ( + OptimizationPlan, + default_optimizer, + default_scheduler, +) class PhaseConfig(BaseModel): @@ -19,6 +24,14 @@ class PhaseConfig(BaseModel): def optimizer_step_budget(self, training_size=None): raise NotImplementedError("built-in phase must declare its optimizer budget") + def optimization_plans(self, training_size=None): + raise NotImplementedError("built-in phase must declare optimization roles") + + @model_validator(mode="after") + def validate_optimization(self): + self.optimization_plans() + return self + @field_validator("name") @classmethod def phase_name(cls, value): @@ -29,18 +42,24 @@ class ActorOptimizationConfig(PhaseConfig): batch_size: int = Field(default=32, gt=0) accumulation_steps: int = Field(default=1, gt=0) actor_learning_rate: float = Field(default=3e-4, gt=0) - actor_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - actor_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) + actor_optimizer: ComponentSpec = Field(default_factory=default_optimizer) + actor_scheduler: ComponentSpec = Field(default_factory=default_scheduler) gradient_clip: float = Field(default=1.0, gt=0) evaluation_interval: int = Field(default=1, gt=0) checkpoint_interval: int = Field(default=1, gt=0) + def optimization_plans(self, training_size=None): + budget = self.optimizer_step_budget(training_size) + return { + "actor": OptimizationPlan( + self.actor_optimizer, + self.actor_scheduler, + self.actor_learning_rate, + self.gradient_clip, + budget, + ) + } + class OfflinePhaseConfig(PhaseConfig): sources: list[str] = Field(min_length=1) @@ -55,7 +74,7 @@ class PolicyImitationPhaseConfig(ActorOptimizationConfig, OfflinePhaseConfig): def optimizer_step_budget(self, training_size=None): if training_size is None: - raise ValueError("imitation optimizer budget requires training decisions") + return None size = self.batch_size * self.accumulation_steps return self.epochs * ((training_size + size - 1) // size) @@ -76,30 +95,12 @@ class IQLPhaseConfig(OfflinePhaseConfig): actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) critic_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - actor_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - value_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - critic_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - actor_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - value_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - critic_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) + actor_optimizer: ComponentSpec = Field(default_factory=default_optimizer) + value_optimizer: ComponentSpec = Field(default_factory=default_optimizer) + critic_optimizer: ComponentSpec = Field(default_factory=default_optimizer) + actor_scheduler: ComponentSpec = Field(default_factory=default_scheduler) + value_scheduler: ComponentSpec = Field(default_factory=default_scheduler) + critic_scheduler: ComponentSpec = Field(default_factory=default_scheduler) gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) evaluation_interval: int = Field(default=100, gt=0) checkpoint_interval: int = Field(default=100, gt=0) @@ -107,6 +108,32 @@ class IQLPhaseConfig(OfflinePhaseConfig): def optimizer_step_budget(self, training_size=None): return self.updates + def optimization_plans(self, training_size=None): + budget = self.optimizer_step_budget(training_size) + return { + "actor": OptimizationPlan( + self.actor_optimizer, + self.actor_scheduler, + self.actor_learning_rate, + self.gradient_clip, + budget, + ), + "value": OptimizationPlan( + self.value_optimizer, + self.value_scheduler, + self.value_learning_rate, + self.gradient_clip, + budget, + ), + "critic": OptimizationPlan( + self.critic_optimizer, + self.critic_scheduler, + self.critic_learning_rate, + self.gradient_clip, + budget, + ), + } + class OnlineEnvironmentConfig(BaseModel): model_config = ConfigDict(extra="forbid") @@ -201,22 +228,10 @@ class PPOPhaseConfig(PhaseConfig): reward_scale: float = Field(default=1.0, gt=0, allow_inf_nan=False) actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - actor_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - value_optimizer: ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam") - ) - actor_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) - value_scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1} - ) - ) + actor_optimizer: ComponentSpec = Field(default_factory=default_optimizer) + value_optimizer: ComponentSpec = Field(default_factory=default_optimizer) + actor_scheduler: ComponentSpec = Field(default_factory=default_scheduler) + value_scheduler: ComponentSpec = Field(default_factory=default_scheduler) actor_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) value_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) evaluation_interval: int = Field(default=1, gt=0) @@ -242,32 +257,27 @@ def validate_plans(self): raise ValueError( "rollout_length must divide total_env_steps unless allow_partial_rollout" ) - for optimizer in (self.actor_optimizer, self.value_optimizer): - if optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"}: - raise ValueError("PPO optimizers must be Adam, AdamW, SGD, or RMSprop") - if "lr" in optimizer.arguments or "params" in optimizer.arguments: - raise ValueError( - "PPO learning rates and parameter ownership belong to the phase" - ) - max_steps = self.optimizer_step_budget() - for scheduler in (self.actor_scheduler, self.value_scheduler): - if ( - scheduler.name == "OneCycleLR" - and scheduler.arguments.get("total_steps") != max_steps - ): - raise ValueError( - "PPO OneCycleLR.total_steps must equal the maximum authored optimizer-step budget" - ) - if ( - scheduler.name - in {"ReduceLROnPlateau", "ChainedScheduler", "SequentialLR"} - or "optimizer" in scheduler.arguments - ): - raise ValueError( - "PPO schedulers must advance independently on optimizer steps" - ) return self + def optimization_plans(self, training_size=None): + budget = self.optimizer_step_budget(training_size) + return { + "actor": OptimizationPlan( + self.actor_optimizer, + self.actor_scheduler, + self.actor_learning_rate, + self.actor_gradient_clip, + budget, + ), + "value": OptimizationPlan( + self.value_optimizer, + self.value_scheduler, + self.value_learning_rate, + self.value_gradient_clip, + budget, + ), + } + class ReferenceConfig(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index e228d6d3..12f43db3 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -75,13 +75,10 @@ SequenceSourceConfig, resolve_sequence_source, ) -from sequifier.config.training_plan import ( - SupervisedTrainingPlanModel as TrainingPlanModel, -) from sequifier.config.training_plan import ( TrainingPhaseSpecModel as TrainingPhaseSpecModel, ) -from sequifier.config.training_plan import TrainingSourceSpecModel +from sequifier.config.training_plan import TrainingPlanModel, TrainingSourceSpecModel from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -411,8 +408,6 @@ class GlobalTrainingSpecModel(BaseModel): max_ram_gb: int | float = 16 world_size: int = Field(default=1, gt=0) num_workers: int = Field(default=0, ge=0) - episode_cache_size: int = Field(default=8, gt=0) - validation_fraction: float = Field(default=0.2, gt=0, lt=1) backend: str = "nccl" layer_type_dtypes: Optional[dict[str, str]] = None layer_autocast: bool = False @@ -872,25 +867,23 @@ def resolve(self): isinstance(phase, (PPOPhaseConfig, GRPOPhaseConfig)) and archive is not None ): - phase.archive_directory = str((root / archive).resolve()) - for role in ("actor", "value", "critic"): - optimizer = getattr(phase, f"{role}_optimizer", None) - scheduler = getattr(phase, f"{role}_scheduler", None) - if optimizer is not None and ( - optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"} - or {"lr", "params"}.intersection(optimizer.arguments) - ): - raise ValueError( - f"phases.{phase.name}.{role}_optimizer: use a supported optimizer and phase learning rate" - ) - if scheduler is not None and ( - scheduler.name - in {"ReduceLROnPlateau", "SequentialLR", "ChainedScheduler"} - or "optimizer" in scheduler.arguments + archive_path = (root / archive).resolve() + for artifact in ( + self.model_path, + self.report_path, + self.checkpoint_path, + self.phase_directory, ): - raise ValueError( - f"phases.{phase.name}.{role}_scheduler must advance on optimizer steps" - ) + artifact_path = Path(artifact) + if ( + archive_path == artifact_path + or archive_path in artifact_path.parents + or artifact_path in archive_path.parents + ): + raise ValueError( + "archive_directory must be separate from artifacts" + ) + phase.archive_directory = str(archive_path) return self @property @@ -1797,6 +1790,15 @@ def _resolve_interface( resolve_window_view(storage_layout, window_view) return ResolvedModelInterface( name=name, + policy=( + CategoricalPolicyConfig( + **spec.policy.model_dump(), + interface_name=name, + context_length=global_spec.context_length, + ) + if spec.policy is not None + else None + ), input_columns=spec.input_columns, static_conditioning=spec.static_conditioning, static_features_path=metadata.static_features_path, @@ -1852,6 +1854,7 @@ def _resolve_interface( if target_types[target] == "real" ] ) + if spec.policy is None or column in metadata.selected_columns_statistics }, normalize_real_columns=metadata.normalize_real_columns, target_decoder_ids=target_decoder_ids, diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py index 215dacfd..858a2f83 100644 --- a/src/sequifier/config/training_plan.py +++ b/src/sequifier/config/training_plan.py @@ -1,6 +1,6 @@ -"""Shared authored phase contracts; algorithm-specific budgets remain explicit.""" +"""Authored supervised training phases and curriculum schedules.""" -from typing import Generic, Literal, Optional, TypeVar +from typing import Literal, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -65,26 +65,12 @@ def validate_mode_fields(self): return self -Phase = TypeVar( - "Phase", - bound=TrainingPhaseSpecModel, -) - - -class TrainingPlanModel(BaseModel, Generic[Phase]): +class TrainingPlanModel(BaseModel): model_config = ConfigDict(extra="forbid") curriculum_training: bool = False curriculum_column: Optional[str] = Field(default=None, min_length=1) - phases: list[Phase] = Field(min_length=1) - - @field_validator("phases", mode="before") - @classmethod - def default_algorithm(cls, values): - return [ - {"algorithm": "supervised", **value} if isinstance(value, dict) else value - for value in values - ] + phases: list[TrainingPhaseSpecModel] = Field(min_length=1) @model_validator(mode="after") @beartype @@ -97,16 +83,7 @@ def validate_unique_names(self): raise ValueError( "curriculum_column must be set when curriculum_training is true" ) - if self.curriculum_training and any( - phase.algorithm != "supervised" for phase in self.phases - ): - raise ValueError( - "curriculum scheduling currently requires supervised phases" - ) names = [phase.name for phase in self.phases] if len(names) != len(set(names)): raise ValueError("Training phase names must be unique") return self - - -SupervisedTrainingPlanModel = TrainingPlanModel[TrainingPhaseSpecModel] diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 64c7be85..76e4a265 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -2713,11 +2713,19 @@ environment actions. Sealed buffers are consumed once; stale generations fail. ## Checkpoints, resume, and artifacts -The only policy-training checkpoint is `sequifier_policy_run`, format version 1. -It contains the resolved plan/fingerprint, immutable representation, data and -adapter identities, run RNG, current phase/index/status, completed results and -selected actors, algorithm/version and phase-owned state, and the last committed -transaction. DPO/GRPO reference weights are embedded. Optimizer, scheduler, +Named sources share indexed episode storage across phases and partitions. Table +sources are parsed once into temporary sorted Parquet storage with direct episode +row-group access. The episode cache stays bounded; partition views retain metadata +and validated transition indices. Imitation shuffles episodes and then decisions +within each episode to preserve storage locality. + +The only policy-training checkpoint is `sequifier_policy_run`, format version 2. +Completed phase results determine the phase boundary; an optional live phase +state records the latest committed transaction. Actor snapshots are stored once +per weight fingerprint and remain embedded for self-contained recovery. +It contains the resolved plan, immutable representation and its fingerprint, data +and adapter identities, run RNG, completed results, actor snapshots, and optional +versioned phase state. DPO/GRPO reference weights are embedded. Optimizer, scheduler, sampler, selection, evaluation and phase counters remain phase-local. Writes use a temporary file and atomic replacement at completed updates or diff --git a/src/sequifier/fingerprints.py b/src/sequifier/fingerprints.py new file mode 100644 index 00000000..2cec39ea --- /dev/null +++ b/src/sequifier/fingerprints.py @@ -0,0 +1,10 @@ +"""Deterministic identities for JSON-compatible content.""" + +import hashlib +import json + + +def fingerprint(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, allow_nan=False).encode() + ).hexdigest() diff --git a/src/sequifier/io/episode_corpus.py b/src/sequifier/io/episode_corpus.py new file mode 100644 index 00000000..1ebfea0b --- /dev/null +++ b/src/sequifier/io/episode_corpus.py @@ -0,0 +1,154 @@ +"""Indexed episode storage and immutable metadata shared by learning views.""" + +import hashlib +import json +from collections import OrderedDict +from dataclasses import dataclass +from decimal import Decimal + +from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.fingerprints import fingerprint +from sequifier.io.policy_window import action_index +from sequifier.io.sequence import SequenceEpisode +from sequifier.io.values import numeric_scalar +from sequifier.trajectory.importers import source_episodes + + +def _load(source) -> SequenceEpisode: + if isinstance(source, SequenceEpisode): + return SequenceEpisode.model_validate(source.model_dump()) + return source.load() + + +@dataclass(frozen=True) +class EpisodeMetadata: + episode_id: str + group_id: str | None + length: int + content_key: str + fingerprint: str + + +class EpisodeCorpus: + """Own validated episode metadata and one bounded storage cache.""" + + def __init__(self, sources, config, cache_size=8): + if cache_size < 1: + raise ValueError("episode_cache_size must be positive") + self.sources = [ + episode + for source in sources + for episode in ( + source_episodes(source, config) + if isinstance(source, SequenceSourceConfig) + else [source] + ) + ] + self.cache_size = cache_size + self.cache = OrderedDict() + self.metadata = [] + categorical = ( + set(config.categorical_columns) | config.categorical_n_classes.keys() + ) + seen = set() + for index in range(len(self.sources)): + episode = self[index] + if episode.episode_id in seen: + raise ValueError(f"duplicate episode ID {episode.episode_id!r}") + seen.add(episode.episode_id) + if set(episode.observations) != set(config.observation_columns) or set( + episode.static_inputs + ) != set(config.static_columns): + raise ValueError( + "sequence observations/static inputs do not match the policy schema" + ) + for value in episode.actions: + action_index(value, config) + if episode.previous_action is not None: + action_index(episode.previous_action, config) + for column, values in episode.observations.items(): + if column not in categorical: + for value in values: + numeric_scalar(value, column) + for column, value in episode.static_inputs.items(): + if column not in categorical: + numeric_scalar(value, column) + self.metadata.append( + EpisodeMetadata( + episode.episode_id, + episode.group_id, + len(episode.actions), + episode_content_key(episode), + fingerprint(episode.model_dump(mode="json")), + ) + ) + + def __len__(self): + return len(self.sources) + + def __getitem__(self, index): + if index not in self.cache: + self.cache[index] = _load(self.sources[index]) + if len(self.cache) > self.cache_size: + self.cache.popitem(last=False) + self.cache.move_to_end(index) + return self.cache[index] + + def view(self): + return EpisodeView([(self, i) for i in range(len(self))]) + + +class EpisodeView: + """A lightweight ordered partition; storage and metadata remain corpus-owned.""" + + def __init__(self, entries): + self.entries = tuple(entries) + + def __len__(self): + return len(self.entries) + + def __iter__(self): + return (self[i] for i in range(len(self))) + + def __getitem__(self, index): + corpus, position = self.entries[index] + return corpus[position] + + def metadata(self, index): + corpus, position = self.entries[index] + return corpus.metadata[position] + + def subset(self, indices): + return EpisodeView([self.entries[i] for i in indices]) + + @classmethod + def combine(cls, views): + return cls([entry for view in views for entry in view.entries]) + + +def _canonical(value): + if isinstance(value, dict): + return {key: _canonical(item) for key, item in value.items()} + if isinstance(value, (tuple, list)): + return [_canonical(item) for item in value] + if isinstance(value, (int, float)) and not isinstance(value, bool): + return ["number", str(Decimal(str(value)).normalize())] + return value + + +def episode_content_key(episode, *, length=None, returns=None) -> str: + """Hash learning content, independently of log format, IDs and provenance.""" + length = len(episode.actions) if length is None else length + values = { + "observations": { + column: entries[:length] for column, entries in episode.observations.items() + }, + "actions": episode.actions[:length], + "static_inputs": episode.static_inputs, + "previous_action": episode.previous_action, + "starts_episode": episode.starts_episode, + "returns": returns, + } + return hashlib.sha256( + json.dumps(_canonical(values), sort_keys=True, allow_nan=False).encode() + ).hexdigest() diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py index cdb3997b..b40ad905 100644 --- a/src/sequifier/io/iql_dataset.py +++ b/src/sequifier/io/iql_dataset.py @@ -34,12 +34,7 @@ def __init__(self, policy_dataset): self.episode_ids = policy_dataset.episode_ids digest = hashlib.sha256() for episode_index, episode in enumerate(policy_dataset.episodes): - reference = policy_dataset.episodes.sources[episode_index] - source = ( - getattr(reference, "path", None) - or getattr(reference, "paths", None) - or episode_index - ) + source = episode_index transitions = episode.transitions if transitions is None or any( getattr(transitions, field) is None @@ -84,6 +79,9 @@ def __init__(self, policy_dataset): ) self.data_contract = digest.hexdigest() + def use_config(self, config): + self.view = PolicyWindowView(config) + @staticmethod def _successor(episode, step): transitions = episode.transitions diff --git a/src/sequifier/io/partitioning.py b/src/sequifier/io/partitioning.py new file mode 100644 index 00000000..395bd97a --- /dev/null +++ b/src/sequifier/io/partitioning.py @@ -0,0 +1,35 @@ +"""Leakage-safe connected components and deterministic held-out partitions.""" + +import random + + +def split_components(records, leakage_keys, validation_fraction, seed): + if not 0 < validation_fraction < 1: + raise ValueError("validation_fraction must be in (0,1)") + records = list(records) + parents = list(range(len(records))) + + def root(index): + while parents[index] != index: + parents[index] = parents[parents[index]] + index = parents[index] + return index + + seen = {} + for index, record in enumerate(records): + for key in leakage_keys(record): + if key in seen: + parents[root(index)] = root(seen[key]) + seen[key] = index + groups = {} + for index, record in enumerate(records): + groups.setdefault(root(index), []).append(record) + components = list(groups.values()) + if len(components) < 2: + raise ValueError("splitting requires at least two independent components") + random.Random(seed).shuffle(components) + count = min(len(components) - 1, max(1, int(len(components) * validation_fraction))) + return tuple( + [record for group in groups for record in group] + for groups in (components[count:], components[:count]) + ) diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index 90dadcc3..1732b54b 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -2,19 +2,18 @@ import hashlib import json -import random from bisect import bisect_right from collections import OrderedDict -from collections.abc import Iterator, Sequence -from decimal import Decimal +from collections.abc import Sequence import polars as pl import torch from torch.utils.data import Dataset, Sampler from sequifier.config.policy_config import CategoricalPolicyConfig -from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.io.batch import SequifierBatch +from sequifier.io.episode_corpus import EpisodeCorpus, EpisodeView, episode_content_key +from sequifier.io.partitioning import split_components from sequifier.io.policy_window import ( PolicyWindowView, action_global_ids, @@ -29,68 +28,9 @@ normalize_values, ) from sequifier.io.sequence import SequenceEpisode -from sequifier.io.values import numeric_scalar -from sequifier.trajectory.importers import source_episodes from sequifier.trajectory.returns import return_to_go -def _load(source) -> SequenceEpisode: - if isinstance(source, SequenceEpisode): - return SequenceEpisode.model_validate(source.model_dump()) - return source.load() - - -class EpisodeCollection: - """Keep source-backed episodes on disk, with a bounded per-worker cache.""" - - def __init__(self, sources, cache_size): - self.sources = list(sources) - self.cache_size = cache_size - self.cache: OrderedDict[int, SequenceEpisode] = OrderedDict() - - def __len__(self): - return len(self.sources) - - def __iter__(self) -> Iterator[SequenceEpisode]: - return (self[index] for index in range(len(self))) - - def __getitem__(self, index: int) -> SequenceEpisode: - if index not in self.cache: - self.cache[index] = _load(self.sources[index]) - if len(self.cache) > self.cache_size: - self.cache.popitem(last=False) - self.cache.move_to_end(index) - return self.cache[index] - - -def _canonical(value): - if isinstance(value, dict): - return {key: _canonical(item) for key, item in value.items()} - if isinstance(value, (tuple, list)): - return [_canonical(item) for item in value] - if isinstance(value, (int, float)) and not isinstance(value, bool): - return ["number", str(Decimal(str(value)).normalize())] - return value - - -def episode_content_key(episode, *, length=None, returns=None) -> str: - """Hash learning content, independently of log format, IDs and provenance.""" - length = len(episode.actions) if length is None else length - values = { - "observations": { - column: entries[:length] for column, entries in episode.observations.items() - }, - "actions": episode.actions[:length], - "static_inputs": episode.static_inputs, - "previous_action": episode.previous_action, - "starts_episode": episode.starts_episode, - "returns": returns, - } - return hashlib.sha256( - json.dumps(_canonical(values), sort_keys=True, allow_nan=False).encode() - ).hexdigest() - - def _return_labels(episode: SequenceEpisode, length: int, gamma: float): if episode.transitions is None or episode.transitions.rewards is None: raise ValueError("return-conditioned examples require recorded rewards") @@ -102,7 +42,7 @@ class PolicySequenceDataset(Dataset): def __init__( self, - sources: Sequence, + sources: Sequence | EpisodeView, config: CategoricalPolicyConfig, *, episode_cache_size: int = 8, @@ -120,74 +60,51 @@ def __init__( self.lengths = [] self.cumulative_lengths = [] self._labels = OrderedDict() + view = ( + sources + if isinstance(sources, EpisodeView) + else EpisodeCorpus(sources, config, episode_cache_size).view() + ) eligible = [] seen = set() - expanded = [ - episode - for source in sources - for episode in ( - source_episodes(source, config) - if isinstance(source, SequenceSourceConfig) - else [source] - ) - ] - categorical = ( - set(config.categorical_columns) | config.categorical_n_classes.keys() - ) - for source in expanded: - episode = _load(source) - if episode.episode_id in seen: - raise ValueError(f"duplicate episode ID {episode.episode_id!r}") - seen.add(episode.episode_id) - self.all_episode_ids.append(episode.episode_id) - if set(episode.observations) != set(config.observation_columns) or set( - episode.static_inputs - ) != set(config.static_columns): - raise ValueError( - "sequence observations/static inputs do not match the policy schema" - ) - # Validate all labels, even in an excluded unknown return tail. - for value in episode.actions: - action_index(value, config) - if episode.previous_action is not None: - action_index(episode.previous_action, config) - for column, values in episode.observations.items(): - for value in values: - if column not in categorical: - numeric_scalar(value, column) - for column, value in episode.static_inputs.items(): - if column not in categorical: - numeric_scalar(value, column) - length = ( - episode.return_length(config.returns.finite_horizon) - if config.return_conditioned - else len(episode.actions) - ) - if length is None: - self.exclusions[episode.episode_id] = ( - "missing_rewards" - if episode.transitions is None - or episode.transitions.rewards is None - else "unknown_return_tail" - ) - continue - returns = ( - _return_labels(episode, length, config.returns.gamma) - if config.return_conditioned - else None - ) - eligible.append(source) - self.episode_ids.append(episode.episode_id) - self.content_keys.append( - episode_content_key(episode, length=length, returns=returns) - ) - self.demonstration_keys.append(episode_content_key(episode, length=length)) - self.group_ids.append(episode.group_id) + for index in range(len(view)): + record = view.metadata(index) + episode_id = record.episode_id + if episode_id in seen: + raise ValueError(f"duplicate episode ID {episode_id!r}") + seen.add(episode_id) + self.all_episode_ids.append(episode_id) + length = record.length + content = demonstration = record.content_key + if config.return_conditioned: + episode = view[index] + length = episode.return_length(config.returns.finite_horizon) + if length is None: + self.exclusions[episode_id] = ( + "missing_rewards" + if episode.transitions is None + or episode.transitions.rewards is None + else "unknown_return_tail" + ) + continue + returns = _return_labels(episode, length, config.returns.gamma) + content = episode_content_key(episode, length=length, returns=returns) + demonstration = episode_content_key(episode, length=length) + eligible.append(index) + self.episode_ids.append(episode_id) + self.content_keys.append(content) + self.demonstration_keys.append(demonstration) + self.group_ids.append(record.group_id) self.lengths.append(length) self.cumulative_lengths.append( length + (self.cumulative_lengths[-1] if self.cumulative_lengths else 0) ) - self.episodes = EpisodeCollection(eligible, episode_cache_size) + self.episodes = view.subset(eligible) + self.episode_fingerprints = [ + self.episodes.metadata(i).fingerprint for i in range(len(self.episodes)) + ] + self._validated_config = None + self._validated_statistics = None def __len__(self): return self.cumulative_lengths[-1] if self.cumulative_lengths else 0 @@ -241,7 +158,10 @@ def fit_categorical_config(self) -> CategoricalPolicyConfig: return CategoricalPolicyConfig.model_validate(values) def use_config(self, config: CategoricalPolicyConfig) -> None: + if self._validated_config == config: + return self.config = config + self._validated_statistics = None # Validate held-out values against the fixed training codec before updates. for episode, length in zip(self.episodes, self.lengths): for column, values in episode.observations.items(): @@ -250,8 +170,12 @@ def use_config(self, config: CategoricalPolicyConfig) -> None: for column, value in episode.static_inputs.items(): encode_feature(value, column, config) + self._validated_config = config.model_copy(deep=True) + def validate_normalization(self, statistics): """Check FP32 transforms episode-by-episode without constructing windows.""" + if self._validated_statistics == statistics: + return for index, (episode, length) in enumerate(zip(self.episodes, self.lengths)): for column in self.config.observation_columns + self.config.static_columns: if column in self.config.categorical_n_classes: @@ -280,6 +204,8 @@ def validate_normalization(self, statistics): f"episode {episode.episode_id}: non-finite conditioned return" ) + self._validated_statistics = {k: dict(v) for k, v in statistics.items()} + def _episode_labels(self, index: int): if index not in self._labels: episode = self.episodes[index] @@ -376,45 +302,48 @@ def __iter__(self): def split_by_episode( dataset, validation_fraction=0.2, seed=1010, *, group_identical=False ): - if not 0 < validation_fraction < 1 or len(dataset.episodes) < 2: - raise ValueError( - "episode splitting requires at least two eligible episodes and validation_fraction in (0,1)" - ) - # Union both collection groups and repeated content, including transitive ties. - parents = list(range(len(dataset.episodes))) - - def root(index): - while parents[index] != index: - parents[index] = parents[parents[index]] - index = parents[index] - return index - - seen = {} - for index in range(len(parents)): - keys = ( - [("group", dataset.group_ids[index])] - if dataset.group_ids[index] is not None - else [] + def keys(index): + result = ( + [] + if dataset.group_ids[index] is None + else [("group", dataset.group_ids[index])] ) if group_identical: - keys.append(("content", dataset.demonstration_keys[index])) - for key in keys: - if key in seen: - parents[root(index)] = root(seen[key]) - seen[key] = index - groups = {} - for index, source in enumerate(dataset.episodes.sources): - groups.setdefault(root(index), []).append(source) - partitions = list(groups.values()) - if len(partitions) < 2: - raise ValueError("splitting requires at least two independent episode groups") - random.Random(seed).shuffle(partitions) - count = min(len(partitions) - 1, max(1, int(len(partitions) * validation_fraction))) + result.append(("content", dataset.demonstration_keys[index])) + return result + + partitions = split_components( + range(len(dataset.episodes)), keys, validation_fraction, seed + ) return tuple( PolicySequenceDataset( - [source for group in partition for source in group], + dataset.episodes.subset(indices), dataset.config, episode_cache_size=dataset.episode_cache_size, ) - for partition in (partitions[count:], partitions[:count]) + for indices in partitions ) + + +class PolicyDataCatalog: + """Run-owned corpora and reusable decision views for named input sources.""" + + def __init__(self, sources, config, cache_size): + self.sources, self.config, self.cache_size = sources, config, cache_size + self.corpora = {} + self.datasets = {} + + def dataset(self, names): + key = tuple(names) + if key not in self.datasets: + for name in key: + if name not in self.corpora: + self.corpora[name] = EpisodeCorpus( + [self.sources[name]], self.config, self.cache_size + ) + self.datasets[key] = PolicySequenceDataset( + EpisodeView.combine([self.corpora[name].view() for name in key]), + self.config, + episode_cache_size=self.cache_size, + ) + return self.datasets[key] diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index b1d15509..aafd48eb 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -1,17 +1,15 @@ -"""Lazy preference alternatives, connected-component splits and causal scoring.""" +"""Lazy preference storage and leakage-safe partition views.""" import hashlib -import random from dataclasses import dataclass -from typing import cast import torch -from torch.nn import functional as F -from sequifier.io.batch import SequifierInputs, collate_model_inputs -from sequifier.io.policy_dataset import PolicySequenceDataset, episode_content_key +from sequifier.fingerprints import fingerprint +from sequifier.io.episode_corpus import EpisodeView, episode_content_key +from sequifier.io.partitioning import split_components +from sequifier.io.policy_dataset import PolicySequenceDataset from sequifier.io.policy_window import PolicyWindowView, action_index -from sequifier.rl.run_state import fingerprint from sequifier.trajectory.preferences import load_preferences @@ -32,6 +30,8 @@ def __init__(self, pairs, corpora, config): name: {episode: i for i, episode in enumerate(data.episode_ids)} for name, data in corpora.items() } + self._fitting = None + self._validated_config = None self.contract = [] for pair in self.pairs: entries = [] @@ -100,19 +100,26 @@ def components(self, pair): keys.append(("group", episode.group_id)) return keys - def fitting_dataset(self, cache_size): - refs = {} - for pair in self.pairs: - for span in (pair.chosen, pair.rejected): - index = self.lookup[span.source][span.episode_id] - refs[(span.source, span.episode_id)] = self.corpora[ - span.source - ].episodes.sources[index] - return PolicySequenceDataset( - list(refs.values()), self.config, episode_cache_size=cache_size - ) + def fitting_dataset(self): + if self._fitting is None: + entries = {} + for pair in self.pairs: + for span in (pair.chosen, pair.rejected): + index = self.lookup[span.source][span.episode_id] + entries[(span.source, span.episode_id)] = self.corpora[ + span.source + ].episodes.entries[index] + cache_size = next(iter(self.corpora.values())).episode_cache_size + self._fitting = PolicySequenceDataset( + EpisodeView(entries.values()), + self.config, + episode_cache_size=cache_size, + ) + return self._fitting def use_config(self, config): + if self._validated_config == config: + return self.config = config view = PolicyWindowView(config) contexts = {} @@ -167,8 +174,10 @@ def use_config(self, config): f"pair {pair.pair_id}: initial {mapping} differ under frozen codec" ) + self._validated_config = config.model_copy(deep=True) + -def prepare_preferences(run, phase, config): +def prepare_preferences(run, phase, config, catalog): pairs = load_preferences(run.preferences[phase.preferences]).pairs validation = ( load_preferences(run.preferences[phase.validation_preferences]).pairs @@ -184,12 +193,7 @@ def prepare_preferences(run, phase, config): raise ValueError( f"preferences reference unknown sources: {sorted(names - run.sources.keys())}" ) - corpora = { - name: PolicySequenceDataset( - [run.sources[name]], config, episode_cache_size=run.episode_cache_size - ) - for name in names - } + corpora = {name: catalog.dataset([name]) for name in sorted(names)} data = PreferenceDataset(all_pairs, corpora, config) if validation is not None: left = {key for pair in pairs for key in data.components(pair)} @@ -199,79 +203,10 @@ def prepare_preferences(run, phase, config): f"preference train/validation components overlap: {next(iter(left & right))}" ) else: - parents = list(range(len(pairs))) - - def root(i): - while parents[i] != i: - parents[i] = parents[parents[i]] - i = parents[i] - return i - - seen = {} - for i, pair in enumerate(pairs): - for key in data.components(pair): - if key in seen: - parents[root(i)] = root(seen[key]) - seen[key] = i - groups = {} - for i, pair in enumerate(pairs): - groups.setdefault(root(i), []).append(pair) - components = list(groups.values()) - if len(components) < 2: - raise ValueError( - "DPO splitting requires at least two independent preference components" - ) - random.Random(run.seed).shuffle(components) - count = min( - len(components) - 1, max(1, int(len(components) * run.validation_fraction)) + pairs, validation = split_components( + pairs, data.components, run.validation_fraction, run.seed ) - validation = [pair for group in components[:count] for pair in group] - pairs = [pair for group in components[count:] for pair in group] return PreferenceData( PreferenceDataset(pairs, corpora, config), PreferenceDataset(validation, corpora, config), ) - - -def sequence_log_probabilities(actor, episode, start, end, chunk_size): - """One FP32 score per executable decision; context/padding never contribute.""" - actor.eval() - view = PolicyWindowView(actor.config) - for offset in range(start, end, chunk_size): - positions = range(offset, min(offset + chunk_size, end)) - inputs = cast( - SequifierInputs, - collate_model_inputs( - [view.episode_inputs(episode, t, None) for t in positions] - ), - ).to(actor.device) - actions = torch.tensor( - [action_index(episode.actions[t], actor.config) for t in positions], - dtype=torch.long, - device=actor.device, - ) - scores = actor.evaluate_actions(inputs, actions).float() - if not torch.isfinite(scores).all(): - raise ValueError("non-finite sequence action log likelihood") - yield scores - - -def sequence_score(actor, episode, start, end, chunk_size): - return torch.stack( - [ - scores.sum() - for scores in sequence_log_probabilities( - actor, episode, start, end, chunk_size - ) - ] - ).sum() - - -def dpo_loss(chosen, rejected, reference_chosen, reference_rejected, beta): - margin = (chosen.float() - rejected.float()) - ( - reference_chosen.float() - reference_rejected.float() - ) - loss = -F.logsigmoid(beta * margin) - if not torch.isfinite(margin).all() or not torch.isfinite(loss).all(): - raise ValueError("non-finite DPO margin/loss") - return loss, margin diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index 2146173b..19c64451 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -26,13 +26,7 @@ class PolicyDecision: def validate_online_inputs(actor, inputs): inputs = inputs.to(actor.device) - actor.validate_inputs(inputs) - features, _ = actor.model_inputs(inputs) - if any( - tensor.is_floating_point() and not torch.isfinite(tensor).all() - for tensor in features.values() - ): - raise ValueError("frozen input transforms produced non-finite values") + actor.model_inputs(inputs) return inputs @@ -42,11 +36,16 @@ def decide(actor, value, inputs, generator, *, deterministic=False): actor.eval() if value is not None: value.eval() - inputs = validate_online_inputs(actor, inputs) - distribution = CategoricalActionDistribution(actor(inputs)[:, -1]) + inputs = inputs.to(actor.device) + features, metadata = actor.model_inputs(inputs) + distribution = CategoricalActionDistribution( + actor.prepared_logits(features, metadata)[:, -1] + ) action = distribution.select(deterministic, generator) estimate = ( - value(inputs) if value is not None else torch.zeros(1, device=actor.device) + value.prepared_value(features, metadata) + if value is not None + else torch.zeros(1, device=actor.device) ) if not torch.isfinite(estimate).all(): raise ValueError("non-finite collection value") diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index 0592db59..34210b43 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -1,23 +1,29 @@ """Canonical sigmoid DPO with bounded differentiable sequence scoring.""" import torch +from torch.nn import functional as F from sequifier.config.policy_plan import DPOPhaseConfig -from sequifier.io.preference_dataset import ( - PreferenceData, - dpo_loss, - sequence_log_probabilities, - sequence_score, -) +from sequifier.io.preference_dataset import PreferenceData from sequifier.rl.run_state import ( PhaseContext, Selection, actor_state, apply_gradients, - make_optimization, restore_actor_state, validate_permutation, ) +from sequifier.rl.scoring import sequence_log_probabilities, sequence_score + + +def dpo_loss(chosen, rejected, reference_chosen, reference_rejected, beta): + margin = (chosen.float() - rejected.float()) - ( + reference_chosen.float() - reference_rejected.float() + ) + loss = -F.logsigmoid(beta * margin) + if not torch.isfinite(margin).all() or not torch.isfinite(loss).all(): + raise ValueError("non-finite DPO margin/loss") + return loss, margin def pair_scores(actor, reference, dataset, pair, chunk_size): @@ -66,7 +72,7 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): ) if reference is None: raise ValueError("DPO requires a frozen reference actor") - optimizer, scheduler = make_optimization(actor, phase) + optimizer, scheduler = context.optimization["actor"].build(actor) sampler = torch.Generator().manual_seed(context.run.seed + context.index) restore_actor_state(context, optimizer, scheduler, sampler) state = context.resume_state or {} @@ -113,7 +119,9 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): actor, episode, span.start, span.end, phase.scoring_chunk_size ): (sign * coefficient * chunk.sum()).backward() - apply_gradients(actor, optimizer, scheduler, phase.gradient_clip) + apply_gradients( + actor, optimizer, scheduler, context.optimization["actor"].gradient_clip + ) transaction, position = update, position + len(group) final = update == phase.updates if update % phase.evaluation_interval == 0 or final: diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 51203af5..5bea1987 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -19,7 +19,6 @@ PhaseContext, Selection, actor_state, - make_optimization, require_equal, restore_actor_state, restore_rng_state, @@ -29,7 +28,7 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): actor, phase = context.actor.eval(), context.phase - optimizer, scheduler = make_optimization(actor, phase) + optimizer, scheduler = context.optimization["actor"].build(actor) sampler = torch.Generator().manual_seed(context.run.seed + context.index) restore_actor_state(context, optimizer, scheduler, sampler) state = context.resume_state or {} diff --git a/src/sequifier/rl/grpo_collector.py b/src/sequifier/rl/grpo_collector.py index f1150996..d67e2086 100644 --- a/src/sequifier/rl/grpo_collector.py +++ b/src/sequifier/rl/grpo_collector.py @@ -5,6 +5,7 @@ import torch +from sequifier.fingerprints import fingerprint from sequifier.rl.collection import decide, stream_seed, validate_online_inputs from sequifier.rl.distributions import action_generator from sequifier.rl.environment import ( @@ -15,7 +16,6 @@ ) from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer from sequifier.rl.history import PolicyHistory, frame_storage_bytes -from sequifier.rl.run_state import fingerprint class GroupCollectionError(RuntimeError): @@ -91,7 +91,7 @@ def collect_groups( used += frame_storage_bytes(frame) + 1024 if used > phase.max_buffer_bytes: raise ValueError("GRPO group exceeded max_buffer_bytes") - inputs = validate_online_inputs(actor, history.inputs()) + inputs = history.inputs().to(actor.device) decision = decide(actor, None, inputs, generator) reference_score = float( reference.evaluate_actions( diff --git a/src/sequifier/rl/grpo_update.py b/src/sequifier/rl/grpo_update.py index e08b5ca1..7f6d1475 100644 --- a/src/sequifier/rl/grpo_update.py +++ b/src/sequifier/rl/grpo_update.py @@ -110,7 +110,9 @@ def update_grpo( trajectory.actions ) scored += 1 - apply_gradients(actor, optimizer, scheduler, phase.gradient_clip) + apply_gradients( + actor, optimizer, scheduler, context.optimization["actor"].gradient_clip + ) optimizer_steps += 1 if stopped: break diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 4465675b..8d5f1458 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -6,7 +6,7 @@ import torch from sequifier.config.policy_plan import IQLPhaseConfig -from sequifier.io.iql_dataset import IQLBatch, IQLTransitionDataset +from sequifier.io.iql_dataset import IQLBatch from sequifier.io.tensors import collate_tensor_records from sequifier.rl.models import make_iql_models from sequifier.rl.offline_data import OfflineData @@ -16,9 +16,7 @@ actor_state, apply_gradients, clone_weights, - make_optimization, restore_actor_state, - validate_finite_state, ) @@ -75,10 +73,7 @@ def _evaluate(actor, value, q1, q2, dataset, batch_size, gamma, device): def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): actor, phase = context.actor.eval(), context.phase - data = tuple( - IQLTransitionDataset(part) - for part in (context.data.training, context.data.validation) - ) + data = context.data.prepare_transitions() value, q1, q2, targets = make_iql_models( actor.execution_config, actor.interface_name, len(actor.config.action_values) ) @@ -86,15 +81,14 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): module.eval() modules = {"actor": actor, "value": value, "critic": torch.nn.ModuleList([q1, q2])} plans = { - role: make_optimization(module, phase, role) for role, module in modules.items() + role: context.optimization[role].build(module) + for role, module in modules.items() } optimizers = {role: plan[0] for role, plan in plans.items()} schedulers = {role: plan[1] for role, plan in plans.items()} sampler = torch.Generator().manual_seed(context.run.seed + context.index) state = context.resume_state or {} if state: - for key in ("networks", "optimizers", "schedulers"): - validate_finite_state(state[key], f"iql.{key}") for name, module in ( ("value", value), ("q1", q1), @@ -154,13 +148,18 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): raise ValueError(f"non-finite IQL {role} objective") optimizers[role].zero_grad(set_to_none=True) loss.backward() - for module in modules.values(): + for role, module in modules.items(): torch.nn.utils.clip_grad_norm_( - module.parameters(), phase.gradient_clip, error_if_nonfinite=True + module.parameters(), + context.optimization[role].gradient_clip, + error_if_nonfinite=True, ) for role, module in modules.items(): apply_gradients( - module, optimizers[role], schedulers[role], phase.gradient_clip + module, + optimizers[role], + schedulers[role], + context.optimization[role].gradient_clip, ) _polyak(targets[0], q1, phase.target_update_rate) _polyak(targets[1], q2, phase.target_update_rate) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 6077ff66..2faac0ac 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -1,12 +1,11 @@ -"""Sequential policy coordinator and the sole versioned policy run checkpoint.""" +"""Sequential policy coordination over prepared data and validated recovery state.""" from copy import deepcopy from dataclasses import replace from pathlib import Path -import torch - from sequifier.config.train_config import PolicyRunConfig +from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism from sequifier.io.preference_dataset import PreferenceData from sequifier.rl.checkpoint import load_policy, save_policy @@ -22,15 +21,13 @@ validate_declared_policy, ) from sequifier.rl.registry import BUILTIN_ALGORITHMS, validate_registry +from sequifier.rl.run_checkpoint import RunCheckpoint from sequifier.rl.run_state import ( ActorRepresentation, PhaseContext, - PolicyPhaseResult, PolicyRunResult, - atomic_save, capture_rng_state, clone_weights, - fingerprint, require_equal, restore_rng_state, weight_fingerprint, @@ -53,47 +50,20 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: plan = _plan_contract(config) data_identity = input_identity(config) checkpoint = Path(config.checkpoint_path) - if config.resume.policy == "required" and not checkpoint.is_file(): - raise FileNotFoundError(checkpoint) - payload = None - if config.resume.policy != "never" and checkpoint.is_file(): - payload = torch.load(checkpoint, map_location="cpu", weights_only=False) - if ( - not isinstance(payload, dict) - or payload.get("artifact_type") != "sequifier_policy_run" - or payload.get("format_version") != 1 - ): - raise ValueError( - "policy resume requires a version-1 multi-phase policy run checkpoint" - ) - require_equal(plan, payload["plan"], "resume.plan") - require_equal( - fingerprint(plan), payload["plan_fingerprint"], "resume.plan_fingerprint" - ) - require_equal(data_identity, payload["data_identity"], "resume.data") + recovery = RunCheckpoint.load(checkpoint, config, plan, data_identity) + resuming = recovery is not None adapter_contract = preflight_environments(config) - if payload: + if recovery is not None: require_equal( - adapter_contract, payload["adapter_contract"], "resume.environments" - ) - representation = ActorRepresentation(payload["representation"]) - require_equal( - representation.identity, - payload["representation_fingerprint"], - "resume.representation_fingerprint", + adapter_contract, recovery.adapter_contract, "resume.environments" ) + representation = recovery.representation validate_declared_policy(config.policy, representation.policy, "resume.policy") data = prepare_data(config, representation.policy) actor = build_actor( config, representation.policy, representation.values["statistics"] ) - representation.validate_actor(actor, config.units, "resume.representation") - require_equal( - weight_fingerprint(payload["run_actor"]), - payload["run_actor_identity"], - "resume.incoming_actor.weights", - ) - actor.network.load_state_dict(payload["run_actor"], strict=True) + recovery.validate_actor(actor, config.units) else: data = prepare_data(config, config.policy) if config.initialize_from: @@ -115,7 +85,7 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: else: first = data[0] fitting = ( - first.training.fitting_dataset(config.episode_cache_size) + first.training.fitting_dataset() if isinstance(first, PreferenceData) else first.training if isinstance(first, OfflineData) @@ -145,8 +115,8 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: ) validate_data(config, data, actor) data_contract = [item.contract if item is not None else None for item in data] - if payload: - require_equal(data_contract, payload["data_contract"], "resume.partitions") + if recovery is not None: + require_equal(data_contract, recovery.data_contract, "resume.partitions") for phase in config.phases: if phase.representation is not None: require_equal( @@ -161,87 +131,22 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: resolve_normalization(actor.config, normalization), f"phases.{phase.name}.normalization", ) - phase_index = payload["phase_index"] if payload else 0 - results = ( - [ - PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}) - for result in payload["completed_results"] - ] - if payload - else [] - ) - selected_actors = payload["selected_actors"] if payload else [] - current_state = payload["phase_state"] if payload else None - if ( - not isinstance(phase_index, int) - or not 0 <= phase_index <= len(config.phases) - or len(results) != phase_index - or len(selected_actors) != phase_index - ): - raise ValueError("checkpoint phase index/completed handoffs disagree") - if payload and payload["status"] not in {"ready", "running", "handoff", "complete"}: - raise ValueError("invalid policy checkpoint status") - if payload: - status = payload["status"] - if ( - (status == "running") != (current_state is not None) - or status == "complete" - and phase_index != len(config.phases) - or status == "ready" - and phase_index != 0 - or status == "handoff" - and phase_index == 0 - ): - raise ValueError( - "checkpoint status disagrees with the committed phase boundary" - ) - require_equal( + if recovery is None: + recovery = RunCheckpoint( + plan, + representation, + data_identity, + data_contract, + adapter_contract, { - "phase_index": phase_index, - "transaction": current_state["transaction"] if current_state else None, + "path": config.initialize_from, + "weight_hash": weight_fingerprint(actor.network.state_dict()), }, - payload["last_committed_transaction"], - "resume.last_committed_transaction", - ) - if current_state is not None: - if phase_index == len(config.phases): - raise ValueError("completed run cannot have a live phase state") - phase = config.phases[phase_index] - entry = BUILTIN_ALGORITHMS[phase.algorithm] - require_equal( - {"algorithm": phase.algorithm, "version": entry.state_version}, - payload["phase_runtime"] if payload else None, - "resume.phase_runtime", + clone_weights(actor.network), + capture_rng_state(actor.device), ) - if current_state.get("version") != entry.state_version: - raise ValueError("unsupported phase checkpoint state version") - if current_state is not None: - restored_actor = deepcopy(actor) - restored_actor.network.load_state_dict(current_state["actor"], strict=True) - if any( - not torch.isfinite(t).all() - for t in restored_actor.network.state_dict().values() - ): - raise ValueError("non-finite checkpoint actor") - if getattr(config.phases[phase_index], "reference", None) is not None: - reference_weights = current_state["reference"] - restored_actor.network.load_state_dict(reference_weights, strict=True) - require_equal( - weight_fingerprint(reference_weights), - current_state["reference_identity"], - "resume.embedded_reference", - ) - if any(not torch.isfinite(t).all() for t in reference_weights.values()): - raise ValueError("non-finite checkpoint reference") - initialization = ( - payload["initialization"] - if payload - else { - "path": config.initialize_from, - "weight_hash": weight_fingerprint(actor.network.state_dict()), - } - ) - if payload and config.initialize_from and Path(config.initialize_from).is_file(): + initialization = recovery.initialization + if resuming and config.initialize_from and Path(config.initialize_from).is_file(): initial = load_policy( config.initialize_from, device=config.device, @@ -253,18 +158,7 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: weight_fingerprint(initial.network.state_dict()), "resume.initialize_from.weights", ) - if ( - current_state is not None - and getattr(config.phases[phase_index], "reference", None) is not None - ): - declaration = getattr(config.phases[phase_index], "reference") - if declaration.source == "phase_start": - require_equal( - weight_fingerprint(current_state["reference"]), - weight_fingerprint(actor.network.state_dict()), - "resume.phase_start_reference.weights", - ) - reference_contract = dict(payload["reference_contract"]) if payload else {} + reference_contract = recovery.reference_contract for index, phase in enumerate(config.phases): reference = getattr(phase, "reference", None) if reference is None or reference.source != "artifact": @@ -286,84 +180,21 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: ) reference_contract[str(index)] = identity elif ( - not payload - or index > phase_index - or index == phase_index - and current_state is None + index > recovery.phase_index + or index == recovery.phase_index + and recovery.phase_state is None ): raise FileNotFoundError( f"reference artifact required before phase start: {path}" ) - for index, (result, weights) in enumerate(zip(results, selected_actors)): - require_equal(index, result.index, "resume.completed_phase.index") - require_equal( - config.phases[index].name, result.name, "resume.completed_phase.name" - ) - require_equal( - config.phases[index].algorithm, - result.algorithm, - "resume.completed_phase.algorithm", - ) - require_equal( - result.selected_weight_hash, - weight_fingerprint(weights), - "resume.selected_actor.weights", - ) - if any(not torch.isfinite(t).all() for t in weights.values()): - raise ValueError("non-finite completed phase actor") - if selected_actors: - require_equal( - weight_fingerprint(actor.network.state_dict()), - weight_fingerprint(selected_actors[-1]), - "resume.incoming_selected_actor", - ) saved_random = capture_rng_state(actor.device) - preflight_optimization(config, actor, data) - restore_rng_state(actor.device, payload["random"] if payload else saved_random) - - def envelope(status, state=None): - runtime = ( - None - if phase_index == len(config.phases) - else { - "algorithm": config.phases[phase_index].algorithm, - "version": BUILTIN_ALGORITHMS[ - config.phases[phase_index].algorithm - ].state_version, - } - ) - return { - "artifact_type": "sequifier_policy_run", - "format_version": 1, - "plan": plan, - "plan_fingerprint": fingerprint(plan), - "representation": representation.canonical, - "representation_fingerprint": representation.identity, - "data_identity": data_identity, - "data_contract": data_contract, - "adapter_contract": adapter_contract, - "reference_contract": reference_contract, - "random": capture_rng_state(actor.device), - "phase_index": phase_index, - "status": status, - "completed_results": [r.as_dict() for r in results], - "selected_actors": selected_actors, - "run_actor": clone_weights(actor.network), - "run_actor_identity": weight_fingerprint(actor.network.state_dict()), - "initialization": initialization, - "resume_request": config.resume.model_dump(mode="json"), - "phase_runtime": runtime, - "phase_state": state, - "last_committed_transaction": { - "phase_index": phase_index, - "transaction": state["transaction"] if state else None, - }, - } + optimization = preflight_optimization(config, actor, data) + restore_rng_state(actor.device, recovery.random if resuming else saved_random) published = set() def publish_completed(): - for result, weights in zip(results, selected_actors): + for result, weights in zip(recovery.results, recovery.selected_actors): if result.index in published: continue portable = deepcopy(actor).eval() @@ -375,10 +206,12 @@ def publish_completed(): write_json_atomic(directory / "report.json", result.as_dict()) published.add(result.index) - if payload is None: + if not resuming: # Persist the complete fitted representation before the first update. - atomic_save(checkpoint, envelope("ready")) - while phase_index < len(config.phases): + recovery.save(checkpoint, actor) + while recovery.phase_index < len(config.phases): + phase_index = recovery.phase_index + current_state = recovery.phase_state phase = config.phases[phase_index] require_equal( data_identity, @@ -425,7 +258,7 @@ def commit(state): representation.validate_actor( phase_actor, config.units, f"phases.{phase.name}.actor" ) - atomic_save(checkpoint, envelope("running", state)) + recovery.save(checkpoint, actor, state) context = PhaseContext( config, @@ -438,6 +271,7 @@ def commit(state): reference_identity, current_state, commit, + optimization[phase_index], ) actor, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) representation.validate_actor( @@ -462,11 +296,8 @@ def commit(state): ), }, ) - results.append(result) - selected_actors.append(clone_weights(actor.network)) - phase_index += 1 - current_state = None - atomic_save(checkpoint, envelope("handoff")) + recovery.finish_phase(actor, result) + recovery.save(checkpoint, actor) publish_completed() publish_completed() report = { @@ -479,16 +310,16 @@ def commit(state): "reference_provenance": reference_contract, "initialization": initialization, "resume": config.resume.model_dump(mode="json"), - "phases": [r.as_dict() for r in results], + "phases": [r.as_dict() for r in recovery.results], "model_path": config.model_path, "checkpoint_path": config.checkpoint_path, } save_policy(actor.eval(), config.model_path, metadata=report) write_json_atomic(config.report_path, report) - atomic_save(checkpoint, envelope("complete")) + recovery.save(checkpoint, actor) return PolicyRunResult( actor, - tuple(results), + tuple(recovery.results), representation, config.model_path, config.report_path, diff --git a/src/sequifier/rl/models.py b/src/sequifier/rl/models.py index 8c4557a3..e0278108 100644 --- a/src/sequifier/rl/models.py +++ b/src/sequifier/rl/models.py @@ -39,8 +39,12 @@ def forward(self, history: SequifierInputs, action: torch.Tensor | None = None): features = normalize_policy_features( history.inputs, history.metadata, self.interface ) + return self.prepared_value(features, history.metadata, action) + + def prepared_value(self, features, metadata, action=None): + """Score the same normalized history used by the collecting actor.""" representation = self.network.encode( - features, history.metadata, interface_name=self.interface_name + features, metadata, interface_name=self.interface_name ) state = representation[:, -1] if self.action_embedding is not None: diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index 820a42c7..55056129 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -1,25 +1,32 @@ """Offline phase partitioning and frozen-codec validation.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from sequifier.io.iql_dataset import IQLTransitionDataset from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode -from sequifier.rl.run_state import fingerprint -@dataclass(frozen=True) +@dataclass class OfflineData: training: PolicySequenceDataset validation: PolicySequenceDataset + transitions: tuple[IQLTransitionDataset, IQLTransitionDataset] | None = field( + default=None, init=False + ) + + def prepare_transitions(self): + if self.transitions is None: + self.transitions = ( + IQLTransitionDataset(self.training), + IQLTransitionDataset(self.validation), + ) + return self.transitions @property def contract(self): return [ { - "episodes": [ - fingerprint(episode.model_dump(mode="json")) - for episode in part.episodes - ], + "episodes": part.episode_fingerprints, "episode_ids": part.episode_ids, "decisions": len(part), } @@ -29,16 +36,13 @@ def contract(self): def use_config(self, config): for part in (self.training, self.validation): part.use_config(config) + if self.transitions is not None: + for transitions in self.transitions: + transitions.use_config(config) -def prepare_offline(run, phase, config): - def corpus(names): - return PolicySequenceDataset( - [run.sources[name] for name in names], - config, - episode_cache_size=run.episode_cache_size, - ) - +def prepare_offline(run, phase, config, catalog): + corpus = catalog.dataset training = corpus(phase.sources) if phase.validation_sources: validation = corpus(phase.validation_sources) @@ -57,7 +61,4 @@ def corpus(names): raise ValueError( f"phases.{phase.name}: {attribute} crosses train/validation partitions" ) - if phase.algorithm == "iql": - for part in (training, validation): - IQLTransitionDataset(part) return OfflineData(training, validation) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 184777bb..86a8d1a5 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -63,6 +63,7 @@ def policy_execution_config( update={ "input_columns": columns, "target_columns": [target], + "categorical_decoder_special_tokens": {target: []}, "policy": PolicySchemaConfig.model_validate( config.model_dump(include=set(PolicySchemaConfig.model_fields)) ), @@ -101,10 +102,8 @@ def policy_execution_config( global_training, metadata.storage_layout, ) - resolved.policy = config - resolved.categorical_columns = categorical - resolved.categorical_decoder_special_tokens = {} - resolved.selected_columns_statistics = metadata.selected_columns_statistics + if resolved.target_decoder_ids[target] != decoder_ids: + raise ValueError("policy decoder IDs must exactly match executable actions") return execution_config( model.model_copy(update={"interfaces": {config.interface_name: interface}}), {config.interface_name: resolved}, @@ -132,6 +131,10 @@ def from_network(cls, network, execution_config, interface_name: str): raise ValueError( f"Model interface {interface_name!r} has no policy alignment" ) + if interface.target_decoder_ids[ + interface.target_columns[0] + ] != action_global_ids(interface.policy): + raise ValueError("policy decoder IDs must exactly match executable actions") policy = cls() policy.config = interface.policy policy.execution_config = execution_config @@ -172,17 +175,26 @@ def model_inputs( ) -> tuple[dict[str, Tensor], dict[str, Tensor]]: """Encode raw named histories without mutating stored data or masks.""" self.validate_inputs(inputs) - return normalize_policy_features( + features = normalize_policy_features( inputs.inputs, inputs.metadata, self.interface - ), inputs.metadata + ) + if any( + t.is_floating_point() and not torch.isfinite(t).all() + for t in features.values() + ): + raise ValueError("frozen input transforms produced non-finite values") + return features, inputs.metadata def forward(self, inputs: SequifierInputs) -> Tensor: - features, metadata = self.model_inputs(inputs) + return self.prepared_logits(*self.model_inputs(inputs)) + + def prepared_logits(self, features, metadata) -> Tensor: + """Execute features already checked and normalized by model_inputs.""" logits = self.network( features, metadata, interface_name=self.interface_name ).logits[self.interface.target_columns[0]] return logits.masked_fill( - ~inputs.attention_valid_mask[:, -logits.shape[1] :, None], 0.0 + ~metadata["attention_valid_mask"][:, -logits.shape[1] :, None], 0.0 ) def distribution(self, inputs: SequifierInputs) -> CategoricalActionDistribution: diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index 37178caf..7ae664e5 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -20,11 +20,9 @@ Selection, actor_state, clone_weights, - make_optimization, require_equal, restore_actor_state, restore_rng_state, - validate_finite_state, ) from sequifier.rl.snapshots import snapshots_equal @@ -34,8 +32,8 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): value = HistoryCritic( deepcopy(actor.execution_config), actor.interface_name, None ).eval() - optimizer, scheduler = make_optimization(actor, phase) - value_optimizer, value_scheduler = make_optimization(value, phase, "value") + optimizer, scheduler = context.optimization["actor"].build(actor) + value_optimizer, value_scheduler = context.optimization["value"].build(value) sampler = torch.Generator().manual_seed(context.run.seed + context.index) state = context.resume_state or {} transaction, env_steps = state.get("transaction", 0), state.get("env_steps", 0) @@ -51,8 +49,6 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): ): raise ValueError("PPO checkpoint is not a complete collection/update boundary") if state: - for key in ("value", "value_optimizer", "value_scheduler"): - validate_finite_state(state[key], f"ppo.{key}") if any( not isinstance(clock, int) or isinstance(clock, bool) @@ -121,6 +117,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): sampler, transaction, clocks, + context.optimization, ) env_steps, transaction = env_steps + horizon, transaction + 1 if ( diff --git a/src/sequifier/rl/ppo_collector.py b/src/sequifier/rl/ppo_collector.py index e66ce17a..42b29f17 100644 --- a/src/sequifier/rl/ppo_collector.py +++ b/src/sequifier/rl/ppo_collector.py @@ -91,11 +91,12 @@ def collect(self, horizon, policy_version): successor_value = 0.0 if result.observation is not None: self.history.advance(decision.action, result.observation) - successor_inputs = validate_online_inputs( - self.actor, self.history.inputs() - ) + successor_inputs = self.history.inputs().to(self.actor.device) + features, metadata = self.actor.model_inputs(successor_inputs) if not result.terminated: - successor_value = float(self.value(successor_inputs)[0]) + successor_value = float( + self.value.prepared_value(features, metadata)[0] + ) if not torch.isfinite( torch.tensor(successor_value, dtype=torch.float32) ): diff --git a/src/sequifier/rl/ppo_update.py b/src/sequifier/rl/ppo_update.py index 3670a91e..d73aa86f 100644 --- a/src/sequifier/rl/ppo_update.py +++ b/src/sequifier/rl/ppo_update.py @@ -64,7 +64,16 @@ def validate_collection_scores(actor, value, buffer, batch_size): def update_ppo( - actor, value, buffer, phase, optimizers, schedulers, sampler, policy_version, clocks + actor, + value, + buffer, + phase, + optimizers, + schedulers, + sampler, + policy_version, + clocks, + optimization, ): buffer.require_version(policy_version) if actor.ppo_policy_version != policy_version: @@ -93,8 +102,8 @@ def update_ppo( norms = {"actor": [], "value": []} stopped = False gradient_limits = { - "actor": phase.actor_gradient_clip, - "value": phase.value_gradient_clip, + "actor": optimization["actor"].gradient_clip, + "value": optimization["value"].gradient_clip, } group_size = phase.minibatch_size * phase.accumulation_steps epochs = 0 diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 0e0c95a3..65886105 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -5,7 +5,7 @@ import torch -from sequifier.io.iql_dataset import IQLTransitionDataset +from sequifier.io.policy_dataset import PolicyDataCatalog from sequifier.io.policy_window import PolicyWindowView from sequifier.io.preference_dataset import PreferenceData from sequifier.model.factory import build_transformer_network @@ -19,12 +19,7 @@ from sequifier.rl.policy import CategoricalPolicy, policy_execution_config from sequifier.rl.ppo_evaluation import require_evaluation_timer from sequifier.rl.registry import BUILTIN_ALGORITHMS -from sequifier.rl.run_state import ( - capture_rng_state, - make_optimization, - require_equal, - restore_rng_state, -) +from sequifier.rl.run_state import capture_rng_state, require_equal, restore_rng_state from sequifier.trajectory.importers import source_paths @@ -160,22 +155,46 @@ def build_actor(config, fitted, statistics): def prepare_data(config, policy): - return [ - entry.prepare_data(config, phase, policy) if entry.prepare_data else None - for phase in config.phases - for entry in [BUILTIN_ALGORITHMS[phase.algorithm]] - ] + catalog = PolicyDataCatalog(config.sources, policy, config.episode_cache_size) + prepared = {} + result = [] + for phase in config.phases: + entry = BUILTIN_ALGORITHMS[phase.algorithm] + if entry.prepare_data is None: + result.append(None) + continue + # Budgets and optimizer choices do not change the underlying partition. + key = ( + ("preferences", phase.preferences, phase.validation_preferences) + if phase.algorithm == "dpo" + else ( + "offline", + tuple(phase.sources), + tuple(phase.validation_sources or []), + ) + ) + if key not in prepared: + prepared[key] = entry.prepare_data(config, phase, policy, catalog) + item = prepared[key] + if phase.algorithm == "iql": + item.prepare_transitions() + result.append(item) + return result def validate_data(config, data, actor): policy = actor.config - for phase, item in zip(config.phases, data): + validated = set() + for item in data: + if id(item) in validated: + continue + validated.add(id(item)) if isinstance(item, OfflineData): item.use_config(policy) - for part in (item.training, item.validation): + for index, part in enumerate((item.training, item.validation)): part.validate_normalization(actor.interface.selected_columns_statistics) - if phase.algorithm == "iql": - transitions = IQLTransitionDataset(part) + if item.transitions is not None: + transitions = item.transitions[index] for episode in part.episodes: step = len(episode.actions) - 1 successor = transitions._successor(episode, step) @@ -195,7 +214,7 @@ def validate_data(config, data, actor): item.training.use_config(policy) item.validation.use_config(policy) for partition in (item.training, item.validation): - fitting = partition.fitting_dataset(config.episode_cache_size) + fitting = partition.fitting_dataset() fitting.use_config(policy) fitting.validate_normalization( actor.interface.selected_columns_statistics @@ -235,18 +254,11 @@ def preflight_environments(config): def preflight_optimization(config, actor, data): """Resolve constructors and authored scheduler budgets before creating outputs.""" + plans = [] for phase, part in zip(config.phases, data): size = len(part.training) if part is not None else None - maximum = phase.optimizer_step_budget(size) - for role in ("actor", "value", "critic"): - if not hasattr(phase, f"{role}_optimizer"): - continue - scheduler = getattr(phase, f"{role}_scheduler") - if ( - scheduler.name == "OneCycleLR" - and scheduler.arguments.get("total_steps") != maximum - ): - raise ValueError( - f"phases.{phase.name}.{role}_scheduler.total_steps must equal {maximum}" - ) - make_optimization(actor, phase, role) + roles = phase.optimization_plans(size) + for plan in roles.values(): + plan.build(actor) + plans.append(roles) + return plans diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py new file mode 100644 index 00000000..ca53f76b --- /dev/null +++ b/src/sequifier/rl/run_checkpoint.py @@ -0,0 +1,300 @@ +"""Self-contained policy recovery with one authoritative committed phase boundary.""" + +import math +from dataclasses import dataclass, field +from pathlib import Path + +import torch + +from sequifier.rl.registry import BUILTIN_ALGORITHMS +from sequifier.rl.run_state import ( + ActorRepresentation, + PolicyPhaseResult, + atomic_save, + capture_rng_state, + clone_weights, + require_equal, + weight_fingerprint, +) + + +@dataclass +class RunCheckpoint: + plan: dict + representation: ActorRepresentation + data_identity: dict + data_contract: list + adapter_contract: dict + initialization: dict + incoming_actor: dict + random: dict + reference_contract: dict = field(default_factory=dict) + results: list[PolicyPhaseResult] = field(default_factory=list) + selected_actors: list[dict] = field(default_factory=list) + phase_state: dict | None = None + + @property + def phase_index(self): + return len(self.results) + + @property + def status(self): + if self.phase_index == len(self.plan["phases"]): + return "complete" + if self.phase_state is not None: + return "running" + return "handoff" if self.results else "ready" + + @classmethod + def load(cls, path, config, plan, data_identity): + path = Path(path) + if config.resume.policy == "never": + return None + if not path.is_file(): + if config.resume.policy == "required": + raise FileNotFoundError(path) + return None + payload = torch.load(path, map_location="cpu", weights_only=False) + if ( + not isinstance(payload, dict) + or payload.get("artifact_type") != "sequifier_policy_run" + or payload.get("format_version") != 2 + ): + raise ValueError("policy resume requires a version-2 policy run checkpoint") + try: + require_equal(plan, payload["plan"], "resume.plan") + require_equal(data_identity, payload["data_identity"], "resume.data") + snapshots = payload["snapshots"] + for identity, weights in snapshots.items(): + require_equal( + identity, weight_fingerprint(weights), "resume.snapshot.weights" + ) + validate_finite_state(weights, "snapshot") + results = [ + PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}) + for result in payload["completed_results"] + ] + state = payload["phase_state"] + if state is not None: + state = dict(state) + state["actor_identity"] = state["actor"] + state["actor"] = snapshots[state["actor"]] + state["reference_identity"] = state["reference"] + state["reference"] = ( + snapshots[state["reference"]] + if state["reference"] is not None + else None + ) + selection = dict(state["selection"]) + selection["best_weights"] = ( + snapshots[selection["best_weights"]] + if selection["best_weights"] is not None + else None + ) + state["selection"] = selection + state["random"] = payload["random"] + checkpoint = cls( + payload["plan"], + ActorRepresentation(payload["representation"]), + payload["data_identity"], + payload["data_contract"], + payload["adapter_contract"], + payload["initialization"], + snapshots[payload["incoming_actor"]], + payload["random"], + payload["reference_contract"], + results, + [snapshots[result.selected_weight_hash] for result in results], + state, + ) + require_equal( + checkpoint.representation.identity, + payload["representation_fingerprint"], + "resume.representation_fingerprint", + ) + checkpoint.validate(config) + return checkpoint + except (KeyError, TypeError, AttributeError) as exc: + raise ValueError(f"malformed policy run checkpoint: {exc}") from exc + + def validate(self, config): + if len(self.results) > len(config.phases) or len(self.selected_actors) != len( + self.results + ): + raise ValueError("checkpoint completed handoffs disagree with the plan") + # Parsing the owned representation also validates the persisted policy schema. + self.representation.policy + for index, (result, weights) in enumerate( + zip(self.results, self.selected_actors) + ): + phase = config.phases[index] + require_equal( + (index, phase.name, phase.algorithm, phase.handoff), + (result.index, result.name, result.algorithm, result.handoff), + "resume.completed_phase", + ) + require_equal( + result.selected_weight_hash, + weight_fingerprint(weights), + "resume.selected_actor.weights", + ) + validate_finite_state(result.as_dict(), "completed_phase") + incoming_identity = weight_fingerprint(self.incoming_actor) + require_equal( + self.results[-1].selected_weight_hash + if self.results + else self.initialization["weight_hash"], + incoming_identity, + "resume.incoming_actor", + ) + state = self.phase_state + if state is None: + return + if self.phase_index == len(config.phases): + raise ValueError("completed run cannot have a live phase state") + phase = config.phases[self.phase_index] + if state["version"] != BUILTIN_ALGORITHMS[phase.algorithm].state_version: + raise ValueError("unsupported phase checkpoint state version") + transaction = state["transaction"] + if ( + not isinstance(transaction, int) or isinstance(transaction, bool) + ) or transaction < 1: + raise ValueError( + "phase checkpoint must follow a completed optimizer transaction" + ) + require_equal( + weight_fingerprint(state["actor"]), + state["actor_identity"], + "resume.phase_actor.weights", + ) + selection = state["selection"] + best = selection["best_transaction"] + if best is not None and ( + (not isinstance(best, int) or isinstance(best, bool)) + or not 1 <= best <= transaction + ): + raise ValueError( + "checkpoint selected transaction is outside committed progress" + ) + if (selection["best_metric"] is None) != ( + selection["best_weights"] is None + ) or (best is None) != (selection["best_weights"] is None): + raise ValueError( + "checkpoint best actor, metric and transaction must be recorded together" + ) + reference = getattr(phase, "reference", None) + if (reference is None) != (state["reference"] is None): + raise ValueError("checkpoint reference disagrees with the phase") + if reference is not None: + identity = weight_fingerprint(state["reference"]) + require_equal( + identity, state["reference_identity"], "resume.embedded_reference" + ) + expected = ( + incoming_identity + if reference.source == "phase_start" + else self.reference_contract[str(self.phase_index)] + ) + require_equal(expected, identity, "resume.reference.weights") + validate_finite_state(state, "phase") + + def validate_actor(self, actor, units): + self.representation.validate_actor(actor, units, "resume.representation") + expected = actor.network.state_dict() + weights = [self.incoming_actor, *self.selected_actors] + if self.phase_state is not None: + state = self.phase_state + weights += [ + state["actor"], + state["reference"], + state["selection"]["best_weights"], + ] + for snapshot in weights: + if snapshot is None: + continue + if snapshot.keys() != expected.keys() or any( + snapshot[k].shape != v.shape or snapshot[k].dtype != v.dtype + for k, v in expected.items() + ): + raise ValueError( + "checkpoint actor tensors disagree with the resolved model" + ) + actor.network.load_state_dict(self.incoming_actor, strict=True) + + def save(self, path, actor, state=None): + """Commit a transaction or handoff, storing identical actors only once.""" + self.incoming_actor = clone_weights(actor.network) + self.random = capture_rng_state(actor.device) + self.phase_state = state + snapshots = {} + + def store(weights): + if weights is None: + return None + identity = weight_fingerprint(weights) + snapshots.setdefault(identity, weights) + return identity + + incoming = store(self.incoming_actor) + for weights in self.selected_actors: + store(weights) + encoded = None + if state is not None: + encoded = { + k: v + for k, v in state.items() + if k not in {"actor_identity", "reference_identity", "random"} + } + encoded["actor"] = store(state["actor"]) + encoded["reference"] = store(state["reference"]) + encoded["selection"] = { + **state["selection"], + "best_weights": store(state["selection"]["best_weights"]), + } + atomic_save( + path, + { + "artifact_type": "sequifier_policy_run", + "format_version": 2, + "plan": self.plan, + "representation": self.representation.canonical, + "representation_fingerprint": self.representation.identity, + "data_identity": self.data_identity, + "data_contract": self.data_contract, + "adapter_contract": self.adapter_contract, + "reference_contract": self.reference_contract, + "initialization": self.initialization, + "random": self.random, + "snapshots": snapshots, + "incoming_actor": incoming, + "completed_results": [result.as_dict() for result in self.results], + "phase_state": encoded, + }, + ) + + def finish_phase(self, actor, result): + require_equal(self.phase_index, result.index, "handoff.phase_index") + weights = clone_weights(actor.network) + require_equal( + result.selected_weight_hash, + weight_fingerprint(weights), + "handoff.selected_actor", + ) + self.results.append(result) + self.selected_actors.append(weights) + self.phase_state = None + + +def validate_finite_state(value, field): + """Check algorithm-owned optimizer/network trees before restoring them.""" + if isinstance(value, torch.Tensor): + if not torch.isfinite(value).all(): + raise ValueError(f"non-finite checkpoint tensor: {field}") + elif isinstance(value, dict): + for key, item in value.items(): + validate_finite_state(item, f"{field}.{key}") + elif isinstance(value, (tuple, list)): + for index, item in enumerate(value): + validate_finite_state(item, f"{field}[{index}]") + elif isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"non-finite checkpoint value: {field}") diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 2d01cd27..a9a70643 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -12,8 +12,8 @@ import torch from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_optimization import OptimizationPlan from sequifier.config.policy_plan import PhaseConfig -from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class from sequifier.rl.policy import CategoricalPolicy from sequifier.runtime.random_state import RandomState, RandomStateManager @@ -24,12 +24,6 @@ DataType = TypeVar("DataType") -def fingerprint(value): - return hashlib.sha256( - json.dumps(value, sort_keys=True, allow_nan=False).encode() - ).hexdigest() - - def clone_weights(module): return { name: tensor.detach().cpu().clone() @@ -188,6 +182,7 @@ class PhaseContext(Generic[PhaseType, DataType]): reference_identity: str | None resume_state: dict | None commit: Callable[[dict], None] + optimization: dict[str, OptimizationPlan] class Selection: @@ -202,12 +197,6 @@ def __init__(self, phase, monitor, direction, state=None): self.best_weights = state["best_weights"] self.best_transaction = state["best_transaction"] self.last_metric = state["last_metric"] - if self.best_metric is not None and not math.isfinite(self.best_metric): - raise ValueError("checkpoint selection metric must be finite") - if (self.best_metric is None) != (self.best_weights is None): - raise ValueError( - "checkpoint best actor and metric must be recorded together" - ) def observe(self, actor, metric, transaction): if metric is not None and not math.isfinite(metric): @@ -265,19 +254,6 @@ def finish(self, context, transaction, clocks, metrics, provenance): return actor, result -def make_optimization(module, phase, role="actor"): - optimizer_spec = getattr(phase, f"{role}_optimizer") - scheduler_spec = getattr(phase, f"{role}_scheduler") - optimizer = get_optimizer_class(optimizer_spec.name)( - [p for p in module.parameters() if p.requires_grad], - **{"lr": getattr(phase, f"{role}_learning_rate"), **optimizer_spec.arguments}, - ) - scheduler = get_scheduler_class(scheduler_spec.name)( - optimizer, **scheduler_spec.arguments - ) - return optimizer, scheduler - - def apply_gradients(module, optimizer, scheduler, clip): torch.nn.utils.clip_grad_norm_(module.parameters(), clip, error_if_nonfinite=True) optimizer.step() @@ -315,33 +291,6 @@ def restore_actor_state(context, optimizer, scheduler, sampler): state = context.resume_state if state is None: return - if state.get("version") != 1 or state["transaction"] < 0: - raise ValueError(f"invalid {context.phase.algorithm} phase checkpoint") - if ( - not isinstance(state["transaction"], int) - or isinstance(state["transaction"], bool) - or state["transaction"] < 1 - ): - raise ValueError( - "phase checkpoint must follow a completed optimizer transaction" - ) - require_equal( - weight_fingerprint(state["actor"]), - state["actor_identity"], - "resume.phase_actor.weights", - ) - selection = state["selection"] - if ( - selection["best_transaction"] is not None - and not 1 <= selection["best_transaction"] <= state["transaction"] - ): - raise ValueError( - "checkpoint selected transaction is outside committed progress" - ) - validate_finite_state(state["optimizer"], "optimizer") - validate_finite_state(state["scheduler"], "scheduler") - if selection["best_weights"] is not None: - validate_finite_state(selection["best_weights"], "selection.best_weights") context.actor.network.load_state_dict(state["actor"], strict=True) optimizer.load_state_dict(state["optimizer"]) scheduler.load_state_dict(state["scheduler"]) @@ -349,21 +298,6 @@ def restore_actor_state(context, optimizer, scheduler, sampler): restore_rng_state(context.actor.device, state["random"]) -def validate_finite_state(value, field): - """Check algorithm-owned optimizer/network trees before restoring them.""" - if isinstance(value, torch.Tensor): - if not torch.isfinite(value).all(): - raise ValueError(f"non-finite checkpoint tensor: {field}") - elif isinstance(value, dict): - for key, item in value.items(): - validate_finite_state(item, f"{field}.{key}") - elif isinstance(value, (tuple, list)): - for index, item in enumerate(value): - validate_finite_state(item, f"{field}[{index}]") - elif isinstance(value, float) and not math.isfinite(value): - raise ValueError(f"non-finite checkpoint value: {field}") - - def validate_permutation(order, size, position): if ( not isinstance(position, int) diff --git a/src/sequifier/rl/scoring.py b/src/sequifier/rl/scoring.py new file mode 100644 index 00000000..f5ebf030 --- /dev/null +++ b/src/sequifier/rl/scoring.py @@ -0,0 +1,42 @@ +"""Bounded differentiable scores for causal action sequences.""" + +from typing import cast + +import torch + +from sequifier.io.batch import SequifierInputs, collate_model_inputs +from sequifier.io.policy_window import PolicyWindowView, action_index + + +def sequence_log_probabilities(actor, episode, start, end, chunk_size): + """One FP32 score per executable decision; context/padding never contribute.""" + actor.eval() + view = PolicyWindowView(actor.config) + for offset in range(start, end, chunk_size): + positions = range(offset, min(offset + chunk_size, end)) + inputs = cast( + SequifierInputs, + collate_model_inputs( + [view.episode_inputs(episode, t, None) for t in positions] + ), + ).to(actor.device) + actions = torch.tensor( + [action_index(episode.actions[t], actor.config) for t in positions], + dtype=torch.long, + device=actor.device, + ) + scores = actor.evaluate_actions(inputs, actions).float() + if not torch.isfinite(scores).all(): + raise ValueError("non-finite sequence action log likelihood") + yield scores + + +def sequence_score(actor, episode, start, end, chunk_size): + return torch.stack( + [ + scores.sum() + for scores in sequence_log_probabilities( + actor, episode, start, end, chunk_size + ) + ] + ).sum() diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index a33cec54..14fa4e18 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -11,7 +11,6 @@ Selection, actor_state, apply_gradients, - make_optimization, restore_actor_state, validate_permutation, ) @@ -39,7 +38,7 @@ def evaluate_imitation(actor, dataset, batch_size): def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, OfflineData]): actor, phase, data = context.actor.eval(), context.phase, context.data - optimizer, scheduler = make_optimization(actor, phase) + optimizer, scheduler = context.optimization["actor"].build(actor) sampler = torch.Generator().manual_seed(context.run.seed + context.index) restore_actor_state(context, optimizer, scheduler, sampler) state = context.resume_state or {} @@ -68,7 +67,7 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin raise ValueError("completed imitation checkpoint has a live sampler order") while epoch < phase.epochs: if order is None: - order = torch.randperm(len(training), generator=sampler).tolist() + order = list(training.sampler(sampler)) group = order[position : position + size] optimizer.zero_grad(set_to_none=True) total = 0.0 @@ -80,7 +79,9 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin raise ValueError("non-finite imitation action likelihood") loss.backward() total += float(loss.detach()) - apply_gradients(actor, optimizer, scheduler, phase.gradient_clip) + apply_gradients( + actor, optimizer, scheduler, context.optimization["actor"].gradient_clip + ) transaction += 1 position += len(group) if position == len(training): diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 6c4e4ea5..94ff8c38 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -5,13 +5,13 @@ import os import shutil import tempfile +from bisect import bisect_right from dataclasses import dataclass -from functools import lru_cache from pathlib import Path import polars as pl -from sequifier.config.metadata import load_dataset_metadata +from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata from sequifier.config.policy_config import PolicySchemaConfig from sequifier.config.policy_train_config import ( PolicyTableColumns, @@ -139,29 +139,18 @@ def _scan(paths: tuple[Path, ...]): def _prepared_table( paths: tuple[Path, ...], source: SequenceSourceConfig -) -> pl.DataFrame: - if source.metadata_config_path is None: - raise ValueError("prepared histories require metadata_config_path") - signatures = tuple( - (path.stat().st_mtime_ns, path.stat().st_size, path.stat().st_ino) - for path in (*paths, Path(source.metadata_config_path)) - ) - return _prepared_table_contents(paths, source.metadata_config_path, signatures) - - -@lru_cache(maxsize=1) -def _prepared_table_contents( - paths: tuple[Path, ...], metadata_path: str, signatures: tuple ) -> pl.DataFrame: """Recover unique positions from existing windows, validating overlaps. Split target bounds select decisions. A preceding action may be recovered from stored context, but observations outside the split are not targets. """ + if source.metadata_config_path is None: + raise ValueError("prepared histories require metadata_config_path") from sequifier.helpers import PANDAS_TO_TORCH_TYPES, read_data from sequifier.io.pt_payload import load_pt_payload, tabular_tensor_batch - metadata = load_dataset_metadata(metadata_path) + metadata = load_dataset_metadata(source.metadata_config_path) if metadata.depth_layouts: raise ValueError( "policy table import currently supports scalar prepared columns" @@ -214,31 +203,67 @@ def _prepared_table_contents( return pl.DataFrame(list(rows.values())).sort(["sequenceId", "itemPosition"]) +class IndexedTable: + """Canonical sorted storage with direct row-group access per episode. + + The temporary directory lives as long as its episode references. CSV parsing + and sorting happen once; cache misses never filter the original source. + """ + + def __init__(self, data, sequence_column): + import pyarrow.parquet as pq + + self.directory = tempfile.TemporaryDirectory(prefix="sequifier-episodes-") + self.path = Path(self.directory.name) / "episodes.parquet" + data.sort(sequence_column, maintain_order=True).sink_parquet( + self.path, row_group_size=8192 + ) + table = pq.ParquetFile(self.path) + self.offsets = [0] + for i in range(table.num_row_groups): + self.offsets.append(self.offsets[-1] + table.metadata.row_group(i).num_rows) + self.ranges = {} + offset = 0 + for batch in table.iter_batches(columns=[sequence_column]): + for identity in batch.column(0).to_pylist(): + if isinstance(identity, bool) or not isinstance(identity, (int, str)): + raise ValueError( + "sequence IDs must be non-null integers or strings" + ) + if identity not in self.ranges: + self.ranges[identity] = [offset, offset + 1] + else: + self.ranges[identity][1] = offset + 1 + offset += 1 + table.close() + + def load(self, identity): + import pyarrow.parquet as pq + + start, end = self.ranges[identity] + first = bisect_right(self.offsets, start) - 1 + last = bisect_right(self.offsets, end - 1) - 1 + with pq.ParquetFile(self.path) as table: + rows = table.read_row_groups(list(range(first, last + 1))) + return pl.from_arrow(rows.slice(start - self.offsets[first], end - start)) + + @dataclass(frozen=True) class TableEpisodeReference: - paths: tuple[Path, ...] source: SequenceSourceConfig policy: PolicySchemaConfig sequence_id: str | int + table: IndexedTable + metadata: DatasetMetadata | None = None prepared: bool = False + static_table: IndexedTable | None = None def load(self) -> SequenceEpisode: columns = self.source.columns - data = ( - _prepared_table(self.paths, self.source).filter( - pl.col(columns.sequence_id) == self.sequence_id - ) - if self.prepared - else _scan(self.paths) - .filter(pl.col(columns.sequence_id) == self.sequence_id) - .collect() - ) - data = ordered_sequence_rows(data, columns.item_position) - metadata = ( - load_dataset_metadata(self.source.metadata_config_path) - if self.source.metadata_config_path - else None + data = ordered_sequence_rows( + self.table.load(self.sequence_id), columns.item_position ) + metadata = self.metadata model_encoded = self.prepared or self.source.encoding == "model" def decode(value, column): @@ -295,12 +320,8 @@ def decode(value, column): ) data = decisions static_data = None - if self.source.static_data_path is not None: - static_data = ( - _scan((Path(self.source.static_data_path),)) - .filter(pl.col(columns.sequence_id) == self.sequence_id) - .collect() - ) + if self.static_table is not None: + static_data = self.static_table.load(self.sequence_id) return table_episode( data, self.policy, @@ -526,30 +547,29 @@ def source_episodes(source: SequenceSourceConfig, policy: PolicySchemaConfig): raise ValueError( "policy tables require sequenceId and itemPosition, or explicit column mappings" ) - ids = ( - data.select(columns.sequence_id) - .unique(maintain_order=True) - .collect()[columns.sequence_id] - .to_list() - ) - if any( - isinstance(value, bool) or not isinstance(value, (int, str)) for value in ids - ): - raise ValueError("sequence IDs must be non-null integers or strings") + table = IndexedTable(data, columns.sequence_id) + static_table = None if source.static_data_path: - static_ids = ( - _scan((Path(source.static_data_path),)) - .select(columns.sequence_id) - .collect()[columns.sequence_id] - .to_list() + static_table = IndexedTable( + _scan((Path(source.static_data_path),)), columns.sequence_id ) - if len(static_ids) != len(set(static_ids)) or not set(ids) <= set(static_ids): + if ( + any(end - start != 1 for start, end in static_table.ranges.values()) + or not table.ranges.keys() <= static_table.ranges.keys() + ): raise ValueError( "static data must cover every selected sequence ID without duplicates" ) + metadata = ( + load_dataset_metadata(source.metadata_config_path) + if source.metadata_config_path + else None + ) return [ - TableEpisodeReference(paths, source, policy, identity, prepared) - for identity in ids + TableEpisodeReference( + source, policy, identity, table, metadata, prepared, static_table + ) + for identity in table.ranges ] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..123a4787 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,6 @@ +"""Use the working tree, independently of any installed Sequifier version.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) diff --git a/tests/unit/test_policy_architecture.py b/tests/unit/test_policy_architecture.py new file mode 100644 index 00000000..810a6c36 --- /dev/null +++ b/tests/unit/test_policy_architecture.py @@ -0,0 +1,560 @@ +"""Behavioral coverage for resolved policy contracts, data ownership and recovery.""" + +import json +import runpy +from copy import deepcopy +from pathlib import Path +from unittest.mock import Mock + +import polars as pl +import pytest +import torch +import yaml + +from sequifier.config.metadata import DatasetMetadata +from sequifier.config.policy_plan import PolicyImitationPhaseConfig +from sequifier.config.train_config import ( + GlobalTrainingSpecModel, + ModelInterfaceSpecModel, + PolicyRunConfig, + _resolve_interface, +) +from sequifier.io.partitioning import split_components +from sequifier.io.policy_window import PolicyWindowView, action_global_ids +from sequifier.rl import run_checkpoint +from sequifier.rl.checkpoint import load_policy, save_policy +from sequifier.rl.collection import decide +from sequifier.rl.distributions import action_generator +from sequifier.rl.dpo import dpo_loss +from sequifier.rl.learner import run_policy_training +from sequifier.rl.models import HistoryCritic +from sequifier.rl.preparation import ( + build_actor, + input_identity, + prepare_data, + validate_data, +) +from sequifier.rl.run_checkpoint import RunCheckpoint +from sequifier.rl.run_state import weight_fingerprint +from sequifier.trajectory import importers + +EXAMPLES = Path(__file__).resolve().parents[2] / "documentation/example_inputs" + + +@pytest.fixture(scope="module") +def run_values(tmp_path_factory): + torch.set_num_threads(1) + root = tmp_path_factory.mktemp("policy") + runpy.run_path(str(EXAMPLES / "policy_control_data.py"))["generate"](root / "data") + path = root / "data/control-decisions.csv" + data = pl.read_csv(path) + data.filter( + pl.col("contextId").is_in([f"context-{i}" for i in range(4)]) + ).write_csv(path) + path = root / "data/control-preferences.json" + preferences = json.loads(path.read_text()) + preferences["pairs"] = preferences["pairs"][:4] + path.write_text(json.dumps(preferences)) + values = yaml.safe_load( + (EXAMPLES / "configs/train-multiphase-control.yaml").read_text() + ) + values.update( + project_root=str(root), + context_length=2, + episode_cache_size=1, + resume={"policy": "never", "mode": "exact"}, + ) + values["model"]["backbone"]["architecture"].update( + dim_model=8, + max_context_length=2, + attention={"n_heads": 2}, + feed_forward={"dim": 16}, + ) + for phase in values["phases"]: + phase.update(evaluation_interval=1, checkpoint_interval=1) + if phase["algorithm"] == "policy_imitation": + phase.update(epochs=1, batch_size=8) + elif phase["algorithm"] == "iql": + phase.update(updates=2, batch_size=4) + elif phase["algorithm"] == "dpo": + phase.update(updates=2, batch_size=2, scoring_chunk_size=3) + elif phase["algorithm"] == "grpo": + phase.update( + total_groups=2, + groups_per_update=1, + group_size=2, + max_steps=2, + optimization_epochs=1, + batch_size=2, + ) + elif phase["algorithm"] == "ppo": + phase.update( + total_env_steps=4, + rollout_length=2, + optimization_epochs=1, + minibatch_size=2, + ) + if "environment" in phase: + phase["environment"]["config"] = {"horizon": 2} + evaluation = phase["evaluation"] + evaluation.update(seeds=[17], max_episode_steps=2) + evaluation["environment"]["config"] = {"horizon": 2} + return values + + +def config_for(values, name, *, resume="never"): + values = deepcopy(values) + values["model_name"] = name + values["resume"]["policy"] = resume + return PolicyRunConfig.model_validate(values) + + +@pytest.fixture(scope="module") +def completed(run_values): + config = config_for(run_values, "uninterrupted") + return config, run_policy_training(config) + + +def test_action_vocabulary_and_portable_roundtrip(run_values, tmp_path): + config = config_for(run_values, "vocabulary") + actor = build_actor(config, config.policy, {"cue": {"mean": 0.0, "std": 1.0}}) + target = actor.interface.target_columns[0] + assert ( + actor.interface.target_decoder_ids[target] + == action_global_ids(actor.config) + == [3, 4] + ) + assert actor.interface.target_global_to_decoder[target] == [-1, -1, -1, 0, 1] + assert actor.interface.target_n_classes[target] == 2 + assert actor.interface.categorical_decoder_special_tokens == {target: []} + history = PolicyWindowView(actor.config).inputs([{"cue": 1.0}], [2], None, {}) + logits = actor(history) + assert logits.shape[-1] == 2 + path = tmp_path / "actor.pt" + save_policy(actor, path) + restored = load_policy(path) + assert torch.equal(restored(history), logits) + # Validate a decision once, sharing the checked transforms with the critic. + critic = HistoryCritic(deepcopy(actor.execution_config), actor.interface_name, None) + actor.validate_inputs = Mock(wraps=actor.validate_inputs) + decide(actor, critic, history, action_generator(actor.device, 1)) + actor.validate_inputs.assert_called_once() + broken = deepcopy(history) + broken.inputs.pop("cue") + with pytest.raises(ValueError, match="persisted named feature schema"): + decide(actor, None, broken, action_generator(actor.device, 1)) + + +def test_supervised_decoder_defaults_stay_unchanged(): + spec = ModelInterfaceSpecModel.model_validate( + { + "input_columns": ["x"], + "target_columns": ["y"], + "ingestion": {"type": "embedding", "output_dim": 4}, + "decoder": {"type": "linear", "prediction_length": 1}, + } + ) + metadata = DatasetMetadata( + window_length=3, + column_data_types={"x": "Float32", "y": "Int64"}, + n_classes={"y": 5}, + selected_columns_statistics={"x": {"mean": 2.0, "std": 3.0}}, + ) + training = GlobalTrainingSpecModel( + training_objective="causal", + inference_batch_size=1, + context_length=2, + batch_size=1, + learning_rate=1e-3, + ) + resolved = _resolve_interface( + "forecast", spec, metadata, training, metadata.storage_layout + ) + assert resolved.policy is None + assert resolved.target_decoder_ids == {"y": [1, 3, 4]} + assert resolved.categorical_decoder_special_tokens == {"y": ["other"]} + assert resolved.selected_columns_statistics == metadata.selected_columns_statistics + + +def test_shared_corpus_transitions_and_bounded_access(run_values, monkeypatch): + scan = Mock(wraps=importers._scan) + monkeypatch.setattr(importers, "_scan", scan) + from sequifier.io.iql_dataset import IQLTransitionDataset + + initialize = IQLTransitionDataset.__init__ + calls = [] + + def count(self, dataset): + calls.append(dataset) + initialize(self, dataset) + + monkeypatch.setattr(IQLTransitionDataset, "__init__", count) + config = config_for(run_values, "views") + data = prepare_data(config, config.policy) + assert data[0] is data[1] + assert scan.call_count == 1 + assert len(calls) == 2 + actor = build_actor(config, config.policy, {"cue": {"mean": 0.0, "std": 1.0}}) + validate_data(config, data, actor) + assert len(calls) == 2 + transitions = data[1].prepare_transitions() + assert transitions is data[1].prepare_transitions() + training, validation = data[0].training, data[0].validation + assert set(training.group_ids).isdisjoint(validation.group_ids) + assert set(training.demonstration_keys).isdisjoint(validation.demonstration_keys) + for part in (training, validation): + for index in range(len(part)): + part[index] + for corpus, _ in part.episodes.entries: + assert len(corpus.cache) <= config.episode_cache_size + assert scan.call_count == 1 + order = list(training.sampler(torch.Generator().manual_seed(1))) + assert sorted(order) == list(range(len(training))) + episode_indices = [ + next(i for i, end in enumerate(training.cumulative_lengths) if decision < end) + for decision in order + ] + assert len( + [ + i + for i, e in enumerate(episode_indices) + if i == 0 or e != episode_indices[i - 1] + ] + ) == len(training.episodes) + + +def test_transitive_components_are_never_split(): + keys = {0: ["a"], 1: ["a", "b"], 2: ["b"], 3: ["c"], 4: ["d"]} + training, validation = split_components(keys, keys.__getitem__, 0.4, 17) + assert {0, 1, 2} <= set(training) or {0, 1, 2} <= set(validation) + assert (training, validation) == split_components(keys, keys.__getitem__, 0.4, 17) + with pytest.raises(ValueError, match="independent components"): + split_components([0, 1, 2], keys.__getitem__, 0.5, 1) + + +@pytest.mark.parametrize("role", ["actor", "value", "critic"]) +def test_optimizer_roles_share_validation(run_values, role): + values = deepcopy(run_values["phases"][1]) + values[f"{role}_optimizer"] = {"name": "Adam", "arguments": {"lr": 1e-3}} + with pytest.raises(ValueError, match="phase learning rate"): + config_for({**run_values, "phases": [values]}, f"bad-{role}") + values.pop(f"{role}_optimizer") + values[f"{role}_scheduler"] = { + "name": "OneCycleLR", + "arguments": {"total_steps": 3, "max_lr": 0.001}, + } + with pytest.raises(ValueError, match="budget"): + config_for({**run_values, "phases": [values]}, f"bad-budget-{role}") + + +def test_imitation_scheduler_budget_uses_actual_partition(): + phase = PolicyImitationPhaseConfig.model_validate( + { + "name": "imitation", + "algorithm": "policy_imitation", + "sources": ["data"], + "epochs": 2, + "batch_size": 3, + "accumulation_steps": 2, + "actor_scheduler": { + "name": "OneCycleLR", + "arguments": {"total_steps": 4, "max_lr": 0.001}, + }, + } + ) + assert phase.optimization_plans(10)["actor"].step_budget == 4 + with pytest.raises(ValueError, match="budget"): + phase.optimization_plans(20) + + +def test_completed_run_resume_and_snapshot_deduplication(run_values, completed): + config, result = completed + resumed = run_policy_training( + config_for(run_values, config.model_name, resume="required") + ) + assert weight_fingerprint( + resumed.policy.network.state_dict() + ) == weight_fingerprint(result.policy.network.state_dict()) + assert resumed.phases == result.phases + payload = torch.load(config.checkpoint_path, weights_only=False) + assert payload["incoming_actor"] == result.phases[-1].selected_weight_hash + assert len(payload["snapshots"]) <= len(result.phases) + assert payload["phase_state"] is None + assert ( + not {"phase_index", "status", "last_committed_transaction", "selected_actors"} + & payload.keys() + ) + + +@pytest.mark.parametrize("phase_index", range(5)) +def test_interrupted_resume_matches_uninterrupted( + run_values, completed, monkeypatch, phase_index +): + config = config_for(run_values, f"interrupt-{phase_index}") + save = run_checkpoint.atomic_save + + def interrupt(path, payload): + save(path, payload) + if ( + len(payload["completed_results"]) == phase_index + and payload["phase_state"] is not None + ): + raise InterruptedError("simulated interruption after committed update") + + with monkeypatch.context() as patch: + patch.setattr(run_checkpoint, "atomic_save", interrupt) + with pytest.raises(InterruptedError): + run_policy_training(config) + recovered = run_policy_training( + config_for(run_values, config.model_name, resume="required") + ) + assert [p.selected_weight_hash for p in recovered.phases] == [ + p.selected_weight_hash for p in completed[1].phases + ] + assert weight_fingerprint( + recovered.policy.network.state_dict() + ) == weight_fingerprint(completed[1].policy.network.state_dict()) + + +def test_handoff_resume_republishes_missing_artifacts( + run_values, completed, monkeypatch +): + config = config_for(run_values, "handoff") + save = run_checkpoint.atomic_save + + def interrupt(path, payload): + save(path, payload) + if len(payload["completed_results"]) == 1 and payload["phase_state"] is None: + raise InterruptedError("simulated interruption before publication") + + with monkeypatch.context() as patch: + patch.setattr(run_checkpoint, "atomic_save", interrupt) + with pytest.raises(InterruptedError): + run_policy_training(config) + assert not (Path(config.phase_directory) / "000-imitation/actor.pt").exists() + result = run_policy_training( + config_for(run_values, config.model_name, resume="required") + ) + assert (Path(config.phase_directory) / "000-imitation/actor.pt").is_file() + assert [p.selected_weight_hash for p in result.phases] == [ + p.selected_weight_hash for p in completed[1].phases + ] + + +@pytest.mark.parametrize("corruption", ["weights", "result", "boundary"]) +def test_corrupt_checkpoint_is_rejected(run_values, completed, monkeypatch, corruption): + original_config, _ = completed + config = config_for(run_values, original_config.model_name, resume="required") + payload = torch.load(config.checkpoint_path, weights_only=False) + if corruption == "weights": + weights = next(iter(payload["snapshots"].values())) + next(iter(weights.values())).fill_(float("nan")) + elif corruption == "result": + payload["completed_results"][0]["index"] = 1 + else: + payload["phase_state"] = {"version": 1} + monkeypatch.setattr(torch, "load", lambda *args, **kwargs: payload) + plan = config.model_dump(mode="json", exclude={"resume"}) + with pytest.raises(ValueError): + RunCheckpoint.load(config.checkpoint_path, config, plan, input_identity(config)) + + +def test_dpo_objective_retains_gradient_semantics(): + chosen = torch.tensor([1.5], requires_grad=True) + loss, margin = dpo_loss( + chosen, torch.tensor([0.5]), torch.tensor([0.3]), torch.tensor([0.1]), 0.2 + ) + expected = -torch.nn.functional.logsigmoid(0.2 * ((chosen - 0.5) - (0.3 - 0.1))) + assert torch.equal(loss, expected) + loss.sum().backward() + assert chosen.grad is not None and chosen.grad.item() < 0 + assert margin.item() == pytest.approx(0.8) + + +def test_indexed_table_reads_exact_episode_across_row_groups(): + # One episode crosses two row-group boundaries; its neighbors must be excluded. + table = importers.IndexedTable( + pl.DataFrame( + { + "id": [1] * 7000 + [2] * 12000 + [3] * 1000, + "position": list(range(20000)), + } + ).lazy(), + "id", + ) + assert table.load(1)["position"].to_list() == list(range(7000)) + assert table.load(2)["position"].to_list() == list(range(7000, 19000)) + assert table.load(3)["position"].to_list() == list(range(19000, 20000)) + + +def test_canonical_import_preserves_episode_semantics(run_values, tmp_path): + from sequifier.config.policy_train_config import SequenceSourceConfig + from sequifier.config.sequence_import_config import SequenceImportConfig + + config = config_for(run_values, "import") + source = config.sources["decisions"] + expected = { + reference.load().episode_id: reference.load().model_dump() + for reference in importers.source_episodes(source, config.policy) + } + manifest = importers.import_sequences( + SequenceImportConfig.model_validate( + { + "operation": "import_sequences", + "project_root": str(tmp_path), + "output_path": "canonical", + "source": source.model_dump(), + "policy": config.policy.model_dump( + exclude={"interface_name", "context_length"} + ), + "episodes_per_shard": 3, + } + ) + ) + actual = { + reference.load().episode_id: reference.load().model_dump() + for reference in importers.source_episodes( + SequenceSourceConfig(manifest=str(manifest)), config.policy + ) + } + assert actual == expected + + +def test_return_conditioning_and_fitted_categorical_inputs(run_values): + from sequifier.config.policy_config import CategoricalPolicyConfig + from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode + from sequifier.io.sequence import SequenceEpisode, TransitionColumns + + config = CategoricalPolicyConfig.model_validate( + { + "observation_columns": ["cue"], + "categorical_columns": ["cue"], + "static_columns": ["bias"], + "context_length": 2, + "interface_name": "actor", + "action_values": ["left", "right"], + "returns": {"gamma": 0.5, "scale": 2.0}, + } + ) + episodes = [ + SequenceEpisode( + episode_id=str(i), + item_positions=(0, 1), + observations={"cue": ("red", "blue")}, + static_inputs={"bias": float(i)}, + actions=("left", "right"), + transitions=TransitionColumns(rewards=(1.0, 2.0), terminated=(False, True)), + ) + for i in range(4) + ] + dataset = PolicySequenceDataset(episodes, config, episode_cache_size=1) + training, validation = split_by_episode(dataset, 0.25, 1) + fitted = training.fit_categorical_config() + assert fitted.id_maps["cue"]["red"] != fitted.id_maps["cue"]["blue"] + for part in (training, validation): + part.use_config(fitted) + part.validate_normalization(training.normalization_statistics()) + assert part[0].inputs["__policy_return"].tolist() == [[0.0, 2.0]] + assert part[1].inputs["__policy_return"].tolist() == [[2.0, 2.0]] + values = deepcopy(run_values) + values["model"]["interfaces"]["actor"]["policy"].update( + return_conditioned=True, + observation_columns=["cue"], + categorical_columns=["cue"], + static_columns=["bias"], + ) + values["model"]["interfaces"]["actor"]["input_columns"] = ["cue", "bias"] + values["model"]["interfaces"]["actor"]["ingestion"]["feature_embedding_dims"] = { + "cue": 2, + "bias": 2, + "__policy_previous_action": 2, + "__policy_return": 2, + } + values["phases"] = values["phases"][:1] + run = config_for(values, "conditioned") + actor = build_actor(run, fitted, training.normalization_statistics()) + assert "__policy_return" not in actor.interface.selected_columns_statistics + assert actor(training[0].model_inputs).shape[-1] == 2 + + +def test_embedded_reference_survives_artifact_removal( + run_values, tmp_path, monkeypatch +): + values = deepcopy(run_values) + values["phases"] = [values["phases"][2]] + config = config_for(values, "reference-seed") + actor = build_actor(config, config.policy, {"cue": {"mean": 0.0, "std": 1.0}}) + path = tmp_path / "reference.pt" + save_policy(actor, path) + values["phases"][0]["reference"] = {"source": "artifact", "path": str(path)} + config = config_for(values, "reference-removal") + save = run_checkpoint.atomic_save + + def interrupt(path, payload): + save(path, payload) + if payload["phase_state"] is not None: + raise InterruptedError("committed DPO reference") + + with monkeypatch.context() as patch: + patch.setattr(run_checkpoint, "atomic_save", interrupt) + with pytest.raises(InterruptedError): + run_policy_training(config) + path.unlink() + result = run_policy_training( + config_for(values, config.model_name, resume="required") + ) + assert len(result.phases) == 1 and result.phases[0].algorithm == "dpo" + + +def test_indexed_tables_preserve_model_encoding_and_static_fields(tmp_path): + from sequifier.config.policy_config import PolicySchemaConfig + from sequifier.config.policy_train_config import SequenceSourceConfig + + pl.DataFrame( + { + "sequenceId": [2, 1, 2, 1], + "itemPosition": [1, 1, 0, 0], + "cue": [1.0, -1.0, -1.0, 1.0], + "action": [4, 3, 3, 4], + } + ).write_csv(tmp_path / "episodes.csv") + pl.DataFrame({"sequenceId": [1, 2], "bias": [2.0, -2.0]}).write_csv( + tmp_path / "static.csv" + ) + metadata = DatasetMetadata( + window_length=3, + column_data_types={"cue": "Float32", "bias": "Float32", "action": "Int64"}, + n_classes={"action": 5}, + id_maps={"action": {"left": 3, "right": 4}}, + selected_columns_statistics={ + "cue": {"mean": 10.0, "std": 2.0}, + "bias": {"mean": 3.0, "std": 4.0}, + }, + ) + (tmp_path / "metadata.json").write_text(metadata.model_dump_json()) + source = SequenceSourceConfig( + paths=[str(tmp_path / "episodes.csv")], + static_data_path=str(tmp_path / "static.csv"), + metadata_config_path=str(tmp_path / "metadata.json"), + encoding="model", + ) + policy = PolicySchemaConfig( + observation_columns=["cue"], + static_columns=["bias"], + action_values=["left", "right"], + return_conditioned=False, + ) + episodes = { + episode.episode_id: episode + for reference in importers.source_episodes(source, policy) + for episode in [reference.load()] + } + assert episodes["1"].actions == ("right", "left") + assert episodes["2"].actions == ("left", "right") + assert episodes["1"].observations["cue"] == pytest.approx((12.0, 8.0)) + assert episodes["1"].static_inputs["bias"] == pytest.approx(11.0) + pl.DataFrame({"sequenceId": [1, 1, 2], "bias": [2.0, 2.0, -2.0]}).write_csv( + tmp_path / "static.csv" + ) + with pytest.raises(ValueError, match="without duplicates"): + importers.source_episodes(source, policy) From ee0e411ad8518efcce964798d1f673d599567ea1 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 18:40:05 +0100 Subject: [PATCH 19/71] consolidate --- documentation/policy-learning.md | 9 +- release-notes/unreleased-policy-training.md | 10 +- src/sequifier/consolidated-docs.md | 9 +- src/sequifier/io/preference_dataset.py | 9 +- src/sequifier/preprocess.py | 2 +- src/sequifier/rl/batch_order.py | 76 +++ src/sequifier/rl/dpo.py | 45 +- src/sequifier/rl/grpo.py | 6 +- src/sequifier/rl/inference.py | 2 +- src/sequifier/rl/iql.py | 6 +- src/sequifier/rl/learner.py | 11 +- src/sequifier/rl/offline_data.py | 16 +- src/sequifier/rl/ppo.py | 6 +- src/sequifier/rl/preparation.py | 19 +- src/sequifier/rl/registry.py | 39 +- src/sequifier/rl/run_checkpoint.py | 85 +-- src/sequifier/rl/run_state.py | 26 +- src/sequifier/rl/training.py | 61 +- src/sequifier/rl_api.py | 3 +- src/sequifier/trajectory/export.py | 158 +++++ src/sequifier/trajectory/importers.py | 658 +------------------- src/sequifier/trajectory/prepared.py | 74 +++ src/sequifier/trajectory/sources.py | 104 ++++ src/sequifier/trajectory/tables.py | 358 +++++++++++ tests/conftest.py | 6 - tests/unit/test_policy_architecture.py | 560 ----------------- 26 files changed, 915 insertions(+), 1443 deletions(-) create mode 100644 src/sequifier/rl/batch_order.py create mode 100644 src/sequifier/trajectory/export.py create mode 100644 src/sequifier/trajectory/prepared.py create mode 100644 src/sequifier/trajectory/sources.py create mode 100644 src/sequifier/trajectory/tables.py delete mode 100644 tests/conftest.py delete mode 100644 tests/unit/test_policy_architecture.py diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index ebb782fb..26684ad8 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -372,14 +372,15 @@ row-group access. The episode cache stays bounded; partition views retain metada and validated transition indices. Imitation shuffles episodes and then decisions within each episode to preserve storage locality. -The only policy-training checkpoint is `sequifier_policy_run`, format version 2. +The only policy-training checkpoint is `sequifier_policy_run`, format version 3. Completed phase results determine the phase boundary; an optional live phase -state records the latest committed transaction. Actor snapshots are stored once -per weight fingerprint and remain embedded for self-contained recovery. +state records the latest committed transaction. Incoming and selected actors are +stored as direct tensor state dictionaries for self-contained recovery. It contains the resolved plan, immutable representation and its fingerprint, data -and adapter identities, run RNG, completed results, actor snapshots, and optional +and adapter identities, run RNG, completed results, selected actors, and optional versioned phase state. DPO/GRPO reference weights are embedded. Optimizer, scheduler, sampler, selection, evaluation and phase counters remain phase-local. +Unreleased version-2 run checkpoints are not supported. Writes use a temporary file and atomic replacement at completed updates or handoffs. No partial group is checkpointable. A committed handoff advances the diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index d45e07e0..428391d7 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -15,17 +15,19 @@ - Version-3 local online adapters serve PPO and GRPO. Exact continuation requires validated snapshots or documented reproducible group starts; non-exact restarts report a discontinuity. Failed/partial groups are never reused. -- One atomic `sequifier_policy_run` checkpoint embeds phase references, selected +- One atomic version-3 `sequifier_policy_run` checkpoint embeds phase references, selected actors, source/representation hashes, sampler/RNG state and committed progress. + Actor state dictionaries are stored directly; unreleased version-2 checkpoints + are not supported. The former unreleased single-algorithm configuration/checkpoint paths are removed. - Actor-only portable artifacts remain usable for inference, rollout and cross-run initialization. Default output is `models//actor.pt`; phase reports and selected actors use indexed directories. - Templates, examples, public exports and guides use the canonical schema. The pre-existing supervised training runtime remains independent. -- No tests or training jobs were written/run for this change. Static compilation, - lint and type analysis are the verification scope; numerical learning and exact - continuation acceptance evidence remains to be established. +- Behavioral tests cover portable actors, partition sharing, all five algorithms' + interrupted continuation, phase handoff, and artifact republication. Offline + continuation also covers accumulation, epoch boundaries, and sparse evaluation. Scope: one categorical action, scalar/categorical inputs, causal fixed histories, local single-process FP32. Return conditioning supports imitation only. Text/LLM, diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 76e4a265..605050bd 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -2719,14 +2719,15 @@ row-group access. The episode cache stays bounded; partition views retain metada and validated transition indices. Imitation shuffles episodes and then decisions within each episode to preserve storage locality. -The only policy-training checkpoint is `sequifier_policy_run`, format version 2. +The only policy-training checkpoint is `sequifier_policy_run`, format version 3. Completed phase results determine the phase boundary; an optional live phase -state records the latest committed transaction. Actor snapshots are stored once -per weight fingerprint and remain embedded for self-contained recovery. +state records the latest committed transaction. Incoming and selected actors are +stored as direct tensor state dictionaries for self-contained recovery. It contains the resolved plan, immutable representation and its fingerprint, data -and adapter identities, run RNG, completed results, actor snapshots, and optional +and adapter identities, run RNG, completed results, selected actors, and optional versioned phase state. DPO/GRPO reference weights are embedded. Optimizer, scheduler, sampler, selection, evaluation and phase counters remain phase-local. +Unreleased version-2 run checkpoints are not supported. Writes use a temporary file and atomic replacement at completed updates or handoffs. No partial group is checkpointable. A committed handoff advances the diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index aafd48eb..a78e4e9c 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -177,7 +177,10 @@ def use_config(self, config): self._validated_config = config.model_copy(deep=True) -def prepare_preferences(run, phase, config, catalog): +def prepare_preferences(run, phase, config, catalog, cache): + key = ("preferences", phase.preferences, phase.validation_preferences) + if key in cache: + return cache[key] pairs = load_preferences(run.preferences[phase.preferences]).pairs validation = ( load_preferences(run.preferences[phase.validation_preferences]).pairs @@ -206,7 +209,9 @@ def prepare_preferences(run, phase, config, catalog): pairs, validation = split_components( pairs, data.components, run.validation_fraction, run.seed ) - return PreferenceData( + result = PreferenceData( PreferenceDataset(pairs, corpora, config), PreferenceDataset(validation, corpora, config), ) + cache[key] = result + return result diff --git a/src/sequifier/preprocess.py b/src/sequifier/preprocess.py index d334016e..4f38df49 100644 --- a/src/sequifier/preprocess.py +++ b/src/sequifier/preprocess.py @@ -720,7 +720,7 @@ def preprocess(args: Any, args_config: dict[str, Any]) -> None: from sequifier.config.sequence_import_config import SequenceImportConfig if isinstance(config, SequenceImportConfig): - from sequifier.trajectory.importers import import_sequences + from sequifier.trajectory.export import import_sequences import_sequences(config) else: diff --git a/src/sequifier/rl/batch_order.py b/src/sequifier/rl/batch_order.py new file mode 100644 index 00000000..cb916d32 --- /dev/null +++ b/src/sequifier/rl/batch_order.py @@ -0,0 +1,76 @@ +"""Resumable dataset permutations at complete accumulation boundaries.""" + +import torch + + +class BatchOrder: + def __init__(self, size, group_size, seed, *, sample=None, state=None): + if size < 1 or group_size < 1: + raise ValueError("batch order requires positive dataset and group sizes") + self.size, self.group_size, self.sample = size, group_size, sample + self.generator = torch.Generator().manual_seed(seed) + self.epoch, self.position, self.order = 0, 0, None + if state is not None: + self.epoch = state["epoch"] + self.position = state["position"] + self.order = state["order"] + self._validate() + self.generator.set_state(state["generator"]) + + @property + def transactions(self): + per_epoch = (self.size + self.group_size - 1) // self.group_size + return self.epoch * per_epoch + self.position // self.group_size + + def _validate(self): + if ( + not isinstance(self.epoch, int) + or isinstance(self.epoch, bool) + or self.epoch < 0 + ): + raise ValueError("checkpoint sampler epoch is invalid") + if ( + not isinstance(self.position, int) + or isinstance(self.position, bool) + or not 0 <= self.position < self.size + or self.position % self.group_size + ): + raise ValueError( + "checkpoint sampler is not a complete accumulation boundary" + ) + if self.order is None: + if self.position: + raise ValueError("checkpoint sampler has a position without an order") + elif ( + not isinstance(self.order, list) + or any( + not isinstance(index, int) or isinstance(index, bool) + for index in self.order + ) + or sorted(self.order) != list(range(self.size)) + ): + raise ValueError("checkpoint sampler order is not the dataset permutation") + elif not self.position: + raise ValueError("checkpoint sampler has an unconsumed order") + + def next(self): + if self.order is None: + self.order = ( + list(self.sample(self.generator)) + if self.sample is not None + else torch.randperm(self.size, generator=self.generator).tolist() + ) + group = self.order[self.position : self.position + self.group_size] + self.position += len(group) + if self.position == self.size: + self.epoch += 1 + self.position, self.order = 0, None + return group + + def state_dict(self): + return { + "epoch": self.epoch, + "position": self.position, + "order": self.order, + "generator": self.generator.get_state(), + } diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index 34210b43..bc57139c 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -5,13 +5,14 @@ from sequifier.config.policy_plan import DPOPhaseConfig from sequifier.io.preference_dataset import PreferenceData +from sequifier.rl.batch_order import BatchOrder from sequifier.rl.run_state import ( PhaseContext, Selection, actor_state, apply_gradients, + phase_events, restore_actor_state, - validate_permutation, ) from sequifier.rl.scoring import sequence_log_probabilities, sequence_score @@ -73,30 +74,22 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): if reference is None: raise ValueError("DPO requires a frozen reference actor") optimizer, scheduler = context.optimization["actor"].build(actor) - sampler = torch.Generator().manual_seed(context.run.seed + context.index) - restore_actor_state(context, optimizer, scheduler, sampler) + restore_actor_state(context, optimizer, scheduler) state = context.resume_state or {} transaction, metrics = state.get("transaction", 0), list(state.get("metrics", [])) if not 0 <= transaction <= phase.updates: raise ValueError("DPO checkpoint exceeds authored update budget") selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) - order, position = state.get("order", []), state.get("position", 0) - if state: - validate_permutation(order, len(data.training), position) - size = phase.batch_size * phase.accumulation_steps - cycle = (len(data.training) + size - 1) // size - expected = min(((transaction - 1) % cycle + 1) * size, len(data.training)) - if position != expected: - raise ValueError( - "DPO sampler position disagrees with optimizer transaction" - ) + batches = BatchOrder( + len(data.training), + phase.batch_size * phase.accumulation_steps, + context.run.seed + context.index, + state=state["batch_order"] if state else None, + ) + if transaction != batches.transactions: + raise ValueError("DPO sampler position disagrees with optimizer transaction") for update in range(transaction + 1, phase.updates + 1): - if position == len(order): - order, position = ( - torch.randperm(len(data.training), generator=sampler).tolist(), - 0, - ) - group = order[position : position + phase.batch_size * phase.accumulation_steps] + group = batches.next() optimizer.zero_grad(set_to_none=True) loss_total = 0.0 for index in group: @@ -122,9 +115,11 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): apply_gradients( actor, optimizer, scheduler, context.optimization["actor"].gradient_clip ) - transaction, position = update, position + len(group) - final = update == phase.updates - if update % phase.evaluation_interval == 0 or final: + transaction = update + evaluate, checkpoint = phase_events( + phase, update, final=update == phase.updates + ) + if evaluate: held_out = evaluate_dpo(actor, reference, data.validation, phase) selection.observe(actor, held_out["dpo_loss"], update) metrics.append( @@ -134,18 +129,16 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): **{f"validation_{key}": value for key, value in held_out.items()}, } ) - if update % phase.checkpoint_interval == 0 or final: + if checkpoint: context.commit( actor_state( context, optimizer, scheduler, - sampler, selection, update, metrics, - order=order, - position=position, + batch_order=batches.state_dict(), ) ) return selection.finish( diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 5bea1987..53665944 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -30,7 +30,9 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): actor, phase = context.actor.eval(), context.phase optimizer, scheduler = context.optimization["actor"].build(actor) sampler = torch.Generator().manual_seed(context.run.seed + context.index) - restore_actor_state(context, optimizer, scheduler, sampler) + restore_actor_state(context, optimizer, scheduler) + if context.resume_state is not None: + sampler.set_state(context.resume_state["sampler"]) state = context.resume_state or {} transaction, metrics = state.get("transaction", 0), list(state.get("metrics", [])) groups, env_steps, optimizer_steps = ( @@ -163,10 +165,10 @@ def evaluate(): context, optimizer, scheduler, - sampler, selection, transaction, metrics, + sampler=sampler.get_state(), groups=groups, env_steps=env_steps, optimizer_steps=optimizer_steps, diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 57e13be9..ac51b183 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -18,9 +18,9 @@ from sequifier.rl.runtime import collect_policy_rollout from sequifier.rollout.factories import load_adapter from sequifier.rollout.history import numeric_scalar -from sequifier.trajectory.importers import table_episode from sequifier.trajectory.schema import SeedHistory from sequifier.trajectory.store import save_trajectory +from sequifier.trajectory.tables import table_episode def infer_policy(config: PolicyInferenceConfig) -> None: diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 8d5f1458..bd64632f 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -100,7 +100,9 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): for role in ("value", "critic"): optimizers[role].load_state_dict(state["optimizers"][role]) schedulers[role].load_state_dict(state["schedulers"][role]) - restore_actor_state(context, optimizers["actor"], schedulers["actor"], sampler) + restore_actor_state(context, optimizers["actor"], schedulers["actor"]) + if state: + sampler.set_state(state["sampler"]) selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) metrics = list(state.get("metrics", [])) transaction = state.get("transaction", 0) @@ -197,10 +199,10 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): context, optimizers["actor"], schedulers["actor"], - sampler, selection, update, metrics, + sampler=sampler.get_state(), networks={ name: clone_weights(module) for name, module in ( diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 2faac0ac..c4f098c7 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -20,7 +20,7 @@ validate_data, validate_declared_policy, ) -from sequifier.rl.registry import BUILTIN_ALGORITHMS, validate_registry +from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_checkpoint import RunCheckpoint from sequifier.rl.run_state import ( ActorRepresentation, @@ -45,7 +45,6 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: """Validate the whole run, execute phase-local lifecycles, and export its final actor.""" if not isinstance(config, PolicyRunConfig): raise TypeError("policy training requires PolicyRunConfig") - validate_registry() configure_determinism(config.seed, config.enforce_determinism) plan = _plan_contract(config) data_identity = input_identity(config) @@ -208,7 +207,7 @@ def publish_completed(): if not resuming: # Persist the complete fitted representation before the first update. - recovery.save(checkpoint, actor) + recovery.save(checkpoint, actor.device) while recovery.phase_index < len(config.phases): phase_index = recovery.phase_index current_state = recovery.phase_state @@ -258,7 +257,7 @@ def commit(state): representation.validate_actor( phase_actor, config.units, f"phases.{phase.name}.actor" ) - recovery.save(checkpoint, actor, state) + recovery.save(checkpoint, actor.device, state) context = PhaseContext( config, @@ -297,7 +296,7 @@ def commit(state): }, ) recovery.finish_phase(actor, result) - recovery.save(checkpoint, actor) + recovery.save(checkpoint, actor.device) publish_completed() publish_completed() report = { @@ -316,7 +315,7 @@ def commit(state): } save_policy(actor.eval(), config.model_path, metadata=report) write_json_atomic(config.report_path, report) - recovery.save(checkpoint, actor) + recovery.save(checkpoint, actor.device) return PolicyRunResult( actor, tuple(recovery.results), diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index 55056129..84061c47 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -41,7 +41,11 @@ def use_config(self, config): transitions.use_config(config) -def prepare_offline(run, phase, config, catalog): +def prepare_offline(run, phase, config, catalog, cache): + # The cache belongs to one run: budgets and optimizers do not affect partitions. + key = ("offline", tuple(phase.sources), tuple(phase.validation_sources or [])) + if key in cache: + return cache[key] corpus = catalog.dataset training = corpus(phase.sources) if phase.validation_sources: @@ -61,4 +65,12 @@ def prepare_offline(run, phase, config, catalog): raise ValueError( f"phases.{phase.name}: {attribute} crosses train/validation partitions" ) - return OfflineData(training, validation) + data = OfflineData(training, validation) + cache[key] = data + return data + + +def prepare_iql(run, phase, config, catalog, cache): + data = prepare_offline(run, phase, config, catalog, cache) + data.prepare_transitions() + return data diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index 7ae664e5..e78e91db 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -59,7 +59,9 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): value.load_state_dict(state["value"], strict=True) value_optimizer.load_state_dict(state["value_optimizer"]) value_scheduler.load_state_dict(state["value_scheduler"]) - restore_actor_state(context, optimizer, scheduler, sampler) + restore_actor_state(context, optimizer, scheduler) + if context.resume_state is not None: + sampler.set_state(context.resume_state["sampler"]) selection = Selection( phase, phase.evaluation.metric, @@ -159,10 +161,10 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): context, optimizer, scheduler, - sampler, selection, transaction, metrics, + sampler=sampler.get_state(), value=clone_weights(value), value_optimizer=value_optimizer.state_dict(), value_scheduler=value_scheduler.state_dict(), diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 65886105..d690afa2 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -20,7 +20,7 @@ from sequifier.rl.ppo_evaluation import require_evaluation_timer from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_state import capture_rng_state, require_equal, restore_rng_state -from sequifier.trajectory.importers import source_paths +from sequifier.trajectory.sources import source_paths def file_hash(path): @@ -163,22 +163,7 @@ def prepare_data(config, policy): if entry.prepare_data is None: result.append(None) continue - # Budgets and optimizer choices do not change the underlying partition. - key = ( - ("preferences", phase.preferences, phase.validation_preferences) - if phase.algorithm == "dpo" - else ( - "offline", - tuple(phase.sources), - tuple(phase.validation_sources or []), - ) - ) - if key not in prepared: - prepared[key] = entry.prepare_data(config, phase, policy, catalog) - item = prepared[key] - if phase.algorithm == "iql": - item.prepare_transitions() - result.append(item) + result.append(entry.prepare_data(config, phase, policy, catalog, prepared)) return result diff --git a/src/sequifier/rl/registry.py b/src/sequifier/rl/registry.py index e869c1a6..fd185581 100644 --- a/src/sequifier/rl/registry.py +++ b/src/sequifier/rl/registry.py @@ -1,51 +1,28 @@ -"""One explicit built-in inventory for phase configuration and execution.""" +"""Dispatch built-in phases to their runtimes and data preparation functions.""" from dataclasses import dataclass -from typing import Callable, get_args +from typing import Callable -from sequifier.config.policy_plan import ( - DPOPhaseConfig, - GRPOPhaseConfig, - IQLPhaseConfig, - PolicyImitationPhaseConfig, - PolicyPhase, - PPOPhaseConfig, -) from sequifier.io.preference_dataset import prepare_preferences from sequifier.rl.dpo import run_dpo_phase from sequifier.rl.grpo import run_grpo_phase from sequifier.rl.iql import run_iql_phase -from sequifier.rl.offline_data import prepare_offline +from sequifier.rl.offline_data import prepare_iql, prepare_offline from sequifier.rl.ppo import run_ppo_phase from sequifier.rl.training import run_imitation_phase @dataclass(frozen=True) class BuiltinAlgorithm: - config_type: type runtime: Callable prepare_data: Callable | None = None state_version: int = 1 BUILTIN_ALGORITHMS = { - "policy_imitation": BuiltinAlgorithm( - PolicyImitationPhaseConfig, run_imitation_phase, prepare_offline - ), - "iql": BuiltinAlgorithm(IQLPhaseConfig, run_iql_phase, prepare_offline), - "ppo": BuiltinAlgorithm(PPOPhaseConfig, run_ppo_phase), - "dpo": BuiltinAlgorithm(DPOPhaseConfig, run_dpo_phase, prepare_preferences), - "grpo": BuiltinAlgorithm(GRPOPhaseConfig, run_grpo_phase), + "policy_imitation": BuiltinAlgorithm(run_imitation_phase, prepare_offline), + "iql": BuiltinAlgorithm(run_iql_phase, prepare_iql), + "ppo": BuiltinAlgorithm(run_ppo_phase), + "dpo": BuiltinAlgorithm(run_dpo_phase, prepare_preferences), + "grpo": BuiltinAlgorithm(run_grpo_phase), } - - -def validate_registry(): - union_types = set(get_args(get_args(PolicyPhase)[0])) - registered = {entry.config_type for entry in BUILTIN_ALGORITHMS.values()} - if union_types != registered or any( - get_args(entry.config_type.model_fields["algorithm"].annotation) != (name,) - for name, entry in BUILTIN_ALGORITHMS.items() - ): - raise RuntimeError( - "built-in policy algorithm registry and discriminated union disagree" - ) diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index ca53f76b..152527c6 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -58,41 +58,16 @@ def load(cls, path, config, plan, data_identity): if ( not isinstance(payload, dict) or payload.get("artifact_type") != "sequifier_policy_run" - or payload.get("format_version") != 2 + or payload.get("format_version") != 3 ): - raise ValueError("policy resume requires a version-2 policy run checkpoint") + raise ValueError("policy resume requires a version-3 policy run checkpoint") try: require_equal(plan, payload["plan"], "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") - snapshots = payload["snapshots"] - for identity, weights in snapshots.items(): - require_equal( - identity, weight_fingerprint(weights), "resume.snapshot.weights" - ) - validate_finite_state(weights, "snapshot") results = [ PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}) for result in payload["completed_results"] ] - state = payload["phase_state"] - if state is not None: - state = dict(state) - state["actor_identity"] = state["actor"] - state["actor"] = snapshots[state["actor"]] - state["reference_identity"] = state["reference"] - state["reference"] = ( - snapshots[state["reference"]] - if state["reference"] is not None - else None - ) - selection = dict(state["selection"]) - selection["best_weights"] = ( - snapshots[selection["best_weights"]] - if selection["best_weights"] is not None - else None - ) - state["selection"] = selection - state["random"] = payload["random"] checkpoint = cls( payload["plan"], ActorRepresentation(payload["representation"]), @@ -100,12 +75,12 @@ def load(cls, path, config, plan, data_identity): payload["data_contract"], payload["adapter_contract"], payload["initialization"], - snapshots[payload["incoming_actor"]], + payload["incoming_actor"], payload["random"], payload["reference_contract"], results, - [snapshots[result.selected_weight_hash] for result in results], - state, + payload["selected_actors"], + payload["phase_state"], ) require_equal( checkpoint.representation.identity, @@ -114,7 +89,7 @@ def load(cls, path, config, plan, data_identity): ) checkpoint.validate(config) return checkpoint - except (KeyError, TypeError, AttributeError) as exc: + except (KeyError, TypeError, AttributeError, IndexError) as exc: raise ValueError(f"malformed policy run checkpoint: {exc}") from exc def validate(self, config): @@ -122,6 +97,8 @@ def validate(self, config): self.results ): raise ValueError("checkpoint completed handoffs disagree with the plan") + validate_finite_state(self.incoming_actor, "incoming_actor") + validate_finite_state(self.selected_actors, "selected_actors") # Parsing the owned representation also validates the persisted policy schema. self.representation.policy for index, (result, weights) in enumerate( @@ -162,11 +139,6 @@ def validate(self, config): raise ValueError( "phase checkpoint must follow a completed optimizer transaction" ) - require_equal( - weight_fingerprint(state["actor"]), - state["actor_identity"], - "resume.phase_actor.weights", - ) selection = state["selection"] best = selection["best_transaction"] if best is not None and ( @@ -221,41 +193,15 @@ def validate_actor(self, actor, units): ) actor.network.load_state_dict(self.incoming_actor, strict=True) - def save(self, path, actor, state=None): - """Commit a transaction or handoff, storing identical actors only once.""" - self.incoming_actor = clone_weights(actor.network) - self.random = capture_rng_state(actor.device) + def save(self, path, device, state=None): + """Atomically commit the run and its algorithm-owned recovery state.""" + self.random = capture_rng_state(device) self.phase_state = state - snapshots = {} - - def store(weights): - if weights is None: - return None - identity = weight_fingerprint(weights) - snapshots.setdefault(identity, weights) - return identity - - incoming = store(self.incoming_actor) - for weights in self.selected_actors: - store(weights) - encoded = None - if state is not None: - encoded = { - k: v - for k, v in state.items() - if k not in {"actor_identity", "reference_identity", "random"} - } - encoded["actor"] = store(state["actor"]) - encoded["reference"] = store(state["reference"]) - encoded["selection"] = { - **state["selection"], - "best_weights": store(state["selection"]["best_weights"]), - } atomic_save( path, { "artifact_type": "sequifier_policy_run", - "format_version": 2, + "format_version": 3, "plan": self.plan, "representation": self.representation.canonical, "representation_fingerprint": self.representation.identity, @@ -265,10 +211,10 @@ def store(weights): "reference_contract": self.reference_contract, "initialization": self.initialization, "random": self.random, - "snapshots": snapshots, - "incoming_actor": incoming, + "incoming_actor": self.incoming_actor, + "selected_actors": self.selected_actors, "completed_results": [result.as_dict() for result in self.results], - "phase_state": encoded, + "phase_state": self.phase_state, }, ) @@ -282,6 +228,7 @@ def finish_phase(self, actor, result): ) self.results.append(result) self.selected_actors.append(weights) + self.incoming_actor = weights self.phase_state = None diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index a9a70643..99945958 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -265,16 +265,22 @@ def apply_gradients(module, optimizer, scheduler, clip): optimizer.zero_grad(set_to_none=True) +def phase_events(phase, transaction, *, final): + """Evaluation and checkpointing use the committed transaction clock.""" + return ( + final or transaction % phase.evaluation_interval == 0, + final or transaction % phase.checkpoint_interval == 0, + ) + + def actor_state( - context, optimizer, scheduler, sampler, selection, transaction, metrics, **extra + context, optimizer, scheduler, selection, transaction, metrics, **extra ): return { "version": 1, "actor": clone_weights(context.actor.network), - "actor_identity": weight_fingerprint(context.actor.network.state_dict()), "optimizer": optimizer.state_dict(), "scheduler": scheduler.state_dict(), - "sampler": sampler.get_state(), "selection": selection.state_dict(), "transaction": transaction, "metrics": metrics, @@ -287,23 +293,11 @@ def actor_state( } -def restore_actor_state(context, optimizer, scheduler, sampler): +def restore_actor_state(context, optimizer, scheduler): state = context.resume_state if state is None: return context.actor.network.load_state_dict(state["actor"], strict=True) optimizer.load_state_dict(state["optimizer"]) scheduler.load_state_dict(state["scheduler"]) - sampler.set_state(state["sampler"]) restore_rng_state(context.actor.device, state["random"]) - - -def validate_permutation(order, size, position): - if ( - not isinstance(position, int) - or isinstance(position, bool) - or not 0 <= position <= size - ): - raise ValueError("checkpoint sampler position is invalid") - if sorted(order) != list(range(size)): - raise ValueError("checkpoint sampler order is not the dataset permutation") diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 14fa4e18..7a8bd55c 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -5,14 +5,15 @@ from sequifier.config.policy_plan import PolicyImitationPhaseConfig from sequifier.io.batch import collate_sequence_batches from sequifier.io.policy_window import action_global_ids, action_target +from sequifier.rl.batch_order import BatchOrder from sequifier.rl.offline_data import OfflineData from sequifier.rl.run_state import ( PhaseContext, Selection, actor_state, apply_gradients, + phase_events, restore_actor_state, - validate_permutation, ) @@ -39,36 +40,25 @@ def evaluate_imitation(actor, dataset, batch_size): def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, OfflineData]): actor, phase, data = context.actor.eval(), context.phase, context.data optimizer, scheduler = context.optimization["actor"].build(actor) - sampler = torch.Generator().manual_seed(context.run.seed + context.index) - restore_actor_state(context, optimizer, scheduler, sampler) + restore_actor_state(context, optimizer, scheduler) state = context.resume_state or {} selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) transaction = state.get("transaction", 0) - epoch, position = state.get("epoch", 0), state.get("position", 0) - order, metrics = state.get("order"), list(state.get("metrics", [])) + metrics = list(state.get("metrics", [])) training = data.training - size = phase.batch_size * phase.accumulation_steps - per_epoch = (len(training) + size - 1) // size - if ( - not isinstance(epoch, int) - or not 0 <= epoch <= phase.epochs - or transaction != epoch * per_epoch + (position + size - 1) // size - ): + batches = BatchOrder( + len(training), + phase.batch_size * phase.accumulation_steps, + context.run.seed + context.index, + sample=training.sampler, + state=state["batch_order"] if state else None, + ) + if batches.epoch > phase.epochs or transaction != batches.transactions: raise ValueError("imitation checkpoint epoch/transaction clocks disagree") - if order is not None: - validate_permutation(order, len(training), position) - if position >= len(training) or position % size: - raise ValueError( - "imitation checkpoint is not a complete accumulation boundary" - ) - elif position != 0: - raise ValueError("imitation checkpoint has a sampler position without an order") - if epoch == phase.epochs and (order is not None or position): + if batches.epoch == phase.epochs and batches.order is not None: raise ValueError("completed imitation checkpoint has a live sampler order") - while epoch < phase.epochs: - if order is None: - order = list(training.sampler(sampler)) - group = order[position : position + size] + while batches.epoch < phase.epochs: + group = batches.next() optimizer.zero_grad(set_to_none=True) total = 0.0 for start in range(0, len(group), phase.batch_size): @@ -83,41 +73,36 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin actor, optimizer, scheduler, context.optimization["actor"].gradient_clip ) transaction += 1 - position += len(group) - if position == len(training): - epoch += 1 - position, order = 0, None - final = epoch == phase.epochs - if transaction % phase.evaluation_interval == 0 or final: + evaluate, checkpoint = phase_events( + phase, transaction, final=batches.epoch == phase.epochs + ) + if evaluate: score = evaluate_imitation(actor, data.validation, phase.batch_size) selection.observe(actor, score, transaction) metrics.append( { "transaction": transaction, - "epochs_completed": epoch, + "epochs_completed": batches.epoch, "training_action_nll": total, phase.monitor: score, } ) - if transaction % phase.checkpoint_interval == 0 or final: + if checkpoint: context.commit( actor_state( context, optimizer, scheduler, - sampler, selection, transaction, metrics, - epoch=epoch, - position=position, - order=order, + batch_order=batches.state_dict(), ) ) return selection.finish( context, transaction, - {"optimizer_steps": transaction, "epochs": epoch}, + {"optimizer_steps": transaction, "epochs": batches.epoch}, metrics, {"data": data.contract}, ) diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index 2c6808d8..13bc7857 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -51,7 +51,8 @@ PolicyRunResult, ) from sequifier.rl.runtime import PolicyExecutor, collect_policy_rollout -from sequifier.trajectory.importers import import_sequences, source_episodes +from sequifier.trajectory.export import import_sequences +from sequifier.trajectory.importers import source_episodes from sequifier.trajectory.preferences import ( PreferenceManifest, PreferencePair, diff --git a/src/sequifier/trajectory/export.py b/src/sequifier/trajectory/export.py new file mode 100644 index 00000000..a821a9ea --- /dev/null +++ b/src/sequifier/trajectory/export.py @@ -0,0 +1,158 @@ +"""Atomically export canonical episodes as checksummed raw Parquet shards.""" + +import json +import os +import shutil +import tempfile +from pathlib import Path + +from sequifier.config.policy_train_config import PolicyTableColumns +from sequifier.io.sequence import TransitionColumns +from sequifier.trajectory.importers import source_episodes +from sequifier.trajectory.sources import file_checksum, source_paths + + +def import_sequences(config) -> Path: + """Commit canonical columnar episodes as checksummed raw Parquet shards.""" + import pyarrow as pa + import pyarrow.parquet as pq + + source = config.resolved_source() + destination = (Path(config.project_root) / config.output_path).resolve() + if destination.exists(): + raise ValueError( + "sequence import output_path must name a new dataset directory" + ) + destination.parent.mkdir(parents=True, exist_ok=True) + columns = PolicyTableColumns( + action="__executed_action", + reward="__reward", + terminated="__terminated", + truncated="__truncated", + previous_action="__previous_action", + bootstrap_allowed="__bootstrap", + group="__group", + episode_id="episodeId", + successors={ + name: "__next_" + name for name in config.policy.observation_columns + }, + ) + fields = config.policy.observation_columns + config.policy.static_columns + reserved = { + value for value in columns.model_dump().values() if isinstance(value, str) + } | set(columns.successors.values()) + if set(fields) & reserved: + raise ValueError( + "policy feature names collide with sequence import metadata columns" + ) + references = source_episodes(source, config.policy) + if not references: + raise ValueError("sequence import requires at least one episode") + real = ( + set(fields) + - set(config.policy.categorical_columns) + - config.policy.categorical_n_classes.keys() + ) + action_type = ( + pa.string() if isinstance(config.policy.action_values[0], str) else pa.float64() + ) + types = {name: pa.float64() for name in real | {"__reward"}} + types.update( + {name: pa.bool_() for name in ("__terminated", "__truncated", "__bootstrap")} + ) + types.update( + { + "__executed_action": action_type, + "__previous_action": action_type, + "__group": pa.string(), + } + ) + stage = Path(tempfile.mkdtemp(prefix=".sequifier-import-", dir=destination.parent)) + tables, shards, seen, schema = [], [], set(), None + + def flush(): + nonlocal schema + if tables: + table = pa.concat_tables(tables, promote_options="permissive") + schema = schema or table.schema + name = f"part-{len(shards):05d}.parquet" + pq.write_table(table.cast(schema), stage / name) + shards.append( + { + "path": name, + "sha256": file_checksum(stage / name), + "decisions": table.num_rows, + } + ) + tables.clear() + + try: + for index, reference in enumerate(references): + episode = reference.load() + if episode.episode_id in seen: + raise ValueError("duplicate episode IDs in sequence import") + seen.add(episode.episode_id) + length = len(episode.actions) + transitions = episode.transitions or TransitionColumns() + values = { + **episode.observations, + **{ + name: [value] * length + for name, value in episode.static_inputs.items() + }, + "sequenceId": [index] * length, + "itemPosition": episode.item_positions, + "episodeId": [episode.episode_id] * length, + "__executed_action": episode.actions, + "__previous_action": (episode.previous_action, *episode.actions[:-1]), + "__group": [episode.group_id] * length, + **{ + column: getattr(transitions, field) or [None] * length + for field, column in { + "rewards": "__reward", + "terminated": "__terminated", + "truncated": "__truncated", + "bootstrap_allowed": "__bootstrap", + }.items() + }, + } + arrays = { + name: pa.array(entries, type=types.get(name)) + for name, entries in values.items() + } + for name, column in columns.successors.items(): + arrays[column] = pa.array( + [ + successor[name] if successor is not None else None + for successor in (transitions.successors or [None] * length) + ], + type=arrays[name].type, + ) + tables.append(pa.table(arrays)) + if (index + 1) % config.episodes_per_shard == 0: + flush() + flush() + manifest = { + "format": "sequifier.sequence-data", + "version": 1, + "episodes": len(seen), + "decisions": sum(shard["decisions"] for shard in shards), + "columns": columns.model_dump(), + "policy_schema": config.policy.model_dump(mode="json"), + "shards": shards, + "provenance": { + "source": source.model_dump(mode="json"), + "files": [ + {"path": str(path), "sha256": file_checksum(path)} + for path in source_paths(source) + ], + }, + } + (stage / "manifest.json").write_text( + json.dumps(manifest, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + os.rename(stage, destination) + return destination / "manifest.json" + finally: + if stage.exists(): + shutil.rmtree(stage) diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 94ff8c38..c49f1f60 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -1,74 +1,15 @@ -"""Adapters from tables, prepared histories and rollout logs to sequence data.""" +"""Episode readers for resolved tables, prepared histories, and rollout logs.""" -import hashlib -import json -import os -import shutil -import tempfile -from bisect import bisect_right from dataclasses import dataclass from pathlib import Path -import polars as pl - -from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata from sequifier.config.policy_config import PolicySchemaConfig -from sequifier.config.policy_train_config import ( - PolicyTableColumns, - SequenceSourceConfig, -) -from sequifier.io.preprocessing import denormalize_values -from sequifier.io.sequence import ( - SequenceEpisode, - TransitionColumns, - ordered_sequence_rows, -) -from sequifier.io.values import domain_scalar, numeric_scalar +from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.io.sequence import SequenceEpisode, TransitionColumns from sequifier.trajectory.schema import Trajectory, validate_trajectory +from sequifier.trajectory.sources import resolve_source from sequifier.trajectory.store import load_trajectory - - -def _checksum(path): - digest = hashlib.sha256() - with open(path, "rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def source_paths(source: SequenceSourceConfig) -> list[Path]: - if source.manifest is None: - paths = [Path(path) for path in source.paths] - else: - manifest_path = Path(source.manifest) - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if not isinstance(manifest, dict) or manifest.get("version") != 1: - raise ValueError("source manifest must have version 1") - sequence_manifest = manifest.get("format") == "sequifier.sequence-data" - entries = manifest.get("shards" if sequence_manifest else "episodes") - if not isinstance(entries, list) or not entries: - raise ValueError("source manifest requires a nonempty episodes list") - paths = [] - for entry in entries: - if ( - not isinstance(entry, dict) - or not isinstance(entry.get("path"), str) - or not entry["path"] - ): - raise ValueError("each manifest entry requires a nonempty path") - path = manifest_path.parent / entry["path"] - if sequence_manifest and _checksum(path) != entry.get("sha256"): - raise ValueError(f"sequence shard checksum disagrees with {path}") - if ( - "episode_id" in entry - and load_trajectory(path).episode_id != entry["episode_id"] - ): - raise ValueError(f"manifest episode ID disagrees with {path}") - paths.append(path) - paths = [path.resolve() for path in paths] - if not paths or len(set(paths)) != len(paths): - raise ValueError("sources require nonempty, unique paths") - return paths +from sequifier.trajectory.tables import table_episodes def sequence_from_trajectory( @@ -125,595 +66,14 @@ def load(self) -> SequenceEpisode: ) -def _scan(paths: tuple[Path, ...]): - scans = [] - for path in paths: - if path.suffix.lower() == ".csv": - scans.append(pl.scan_csv(path, infer_schema_length=None)) - elif path.suffix.lower() == ".parquet": - scans.append(pl.scan_parquet(path)) - else: - raise ValueError("table sources require CSV or Parquet files") - return pl.concat(scans, how="vertical_relaxed") - - -def _prepared_table( - paths: tuple[Path, ...], source: SequenceSourceConfig -) -> pl.DataFrame: - """Recover unique positions from existing windows, validating overlaps. - - Split target bounds select decisions. A preceding action may be recovered - from stored context, but observations outside the split are not targets. - """ - if source.metadata_config_path is None: - raise ValueError("prepared histories require metadata_config_path") - from sequifier.helpers import PANDAS_TO_TORCH_TYPES, read_data - from sequifier.io.pt_payload import load_pt_payload, tabular_tensor_batch - - metadata = load_dataset_metadata(source.metadata_config_path) - if metadata.depth_layouts: - raise ValueError( - "policy table import currently supports scalar prepared columns" - ) - rows = {} - for path in paths: - if path.suffix.lower() == ".pt": - batch = load_pt_payload(path, n_classes=metadata.n_classes) - else: - data = read_data(str(path), path.suffix.lower().lstrip(".")) - batch = tabular_tensor_batch( - data, - { - column: PANDAS_TO_TORCH_TYPES[dtype] - for column, dtype in metadata.column_data_types.items() - }, - sorted(set(data["inputCol"].to_list())), - metadata.window_length, - ) - sequences, ids, starts, pads = ( - batch.sequences, - batch.sequence_ids, - batch.start_item_positions, - batch.left_pad_lengths, - ) - lows, highs = batch.split_start_item_positions, batch.split_end_item_positions - for row in range(len(ids)): - for offset in range( - int(pads[row]), next(iter(sequences.values())).shape[1] - ): - position = int(starts[row]) + offset - key = (int(ids[row]), position) - values = { - column: tensor[row, offset].item() - for column, tensor in sequences.items() - } - record = { - "sequenceId": key[0], - "itemPosition": position, - **values, - "__decision": int(lows[row]) <= position < int(highs[row]), - } - if key in rows and rows[key] != record: - raise ValueError( - "prepared history windows disagree on overlapping positions or split bounds" - ) - rows[key] = record - if not rows: - raise ValueError("prepared source contains no valid positions") - return pl.DataFrame(list(rows.values())).sort(["sequenceId", "itemPosition"]) - - -class IndexedTable: - """Canonical sorted storage with direct row-group access per episode. - - The temporary directory lives as long as its episode references. CSV parsing - and sorting happen once; cache misses never filter the original source. - """ - - def __init__(self, data, sequence_column): - import pyarrow.parquet as pq - - self.directory = tempfile.TemporaryDirectory(prefix="sequifier-episodes-") - self.path = Path(self.directory.name) / "episodes.parquet" - data.sort(sequence_column, maintain_order=True).sink_parquet( - self.path, row_group_size=8192 - ) - table = pq.ParquetFile(self.path) - self.offsets = [0] - for i in range(table.num_row_groups): - self.offsets.append(self.offsets[-1] + table.metadata.row_group(i).num_rows) - self.ranges = {} - offset = 0 - for batch in table.iter_batches(columns=[sequence_column]): - for identity in batch.column(0).to_pylist(): - if isinstance(identity, bool) or not isinstance(identity, (int, str)): - raise ValueError( - "sequence IDs must be non-null integers or strings" - ) - if identity not in self.ranges: - self.ranges[identity] = [offset, offset + 1] - else: - self.ranges[identity][1] = offset + 1 - offset += 1 - table.close() - - def load(self, identity): - import pyarrow.parquet as pq - - start, end = self.ranges[identity] - first = bisect_right(self.offsets, start) - 1 - last = bisect_right(self.offsets, end - 1) - 1 - with pq.ParquetFile(self.path) as table: - rows = table.read_row_groups(list(range(first, last + 1))) - return pl.from_arrow(rows.slice(start - self.offsets[first], end - start)) - - -@dataclass(frozen=True) -class TableEpisodeReference: - source: SequenceSourceConfig - policy: PolicySchemaConfig - sequence_id: str | int - table: IndexedTable - metadata: DatasetMetadata | None = None - prepared: bool = False - static_table: IndexedTable | None = None - - def load(self) -> SequenceEpisode: - columns = self.source.columns - data = ordered_sequence_rows( - self.table.load(self.sequence_id), columns.item_position - ) - metadata = self.metadata - model_encoded = self.prepared or self.source.encoding == "model" - - def decode(value, column): - value = domain_scalar(value, column) - if model_encoded and metadata is not None: - if column in metadata.id_maps: - inverse = { - encoded: label - for label, encoded in metadata.id_maps[column].items() - } - if ( - isinstance(value, str) - or value != int(value) - or int(value) not in inverse - ): - raise ValueError(f"{column}: encoded ID has no domain label") - value = inverse[int(value)] - if column == (columns.action or self.policy.action_column): - if isinstance(value, str) and not isinstance( - self.policy.action_values[0], str - ): - try: - value = numeric_scalar(float(value), column) - except ValueError: - pass - matches = [ - action - for action in self.policy.action_values - if action == value or str(action) == str(value) - ] - if len(matches) == 1: - value = matches[0] - elif ( - metadata.normalize_real_columns - and column in metadata.selected_columns_statistics - ): - value = denormalize_values( - numeric_scalar(value, column), - metadata.selected_columns_statistics[column], - ) - return value - - preceding = None - if self.prepared: - decisions = data.filter(pl.col("__decision")) - if decisions.is_empty(): - raise ValueError("prepared episode contains no split-owned decisions") - first = decisions[columns.item_position][0] - previous = data.filter(pl.col(columns.item_position) == first - 1) - if len(previous): - preceding = decode( - previous[columns.action or self.policy.action_column][0], - columns.action or self.policy.action_column, - ) - data = decisions - static_data = None - if self.static_table is not None: - static_data = self.static_table.load(self.sequence_id) - return table_episode( - data, - self.policy, - columns, - sequence_id=self.sequence_id, - preceding=preceding, - static_data=static_data, - episode_end=self.source.episode_end, - episode_namespace=self.source.episode_namespace, - decode=decode, - ) - - -def table_episode( - data, - policy, - columns, - *, - sequence_id, - preceding=None, - static_data=None, - episode_end="unknown", - episode_namespace="", - decode=domain_scalar, -) -> SequenceEpisode: - """Canonical table-to-episode validation shared by training and inference.""" - data = ordered_sequence_rows(data, columns.item_position) - rows = list(data.iter_rows(named=True)) - if not rows: - raise ValueError("sequence source changed while loading") - action_column = columns.action or policy.action_column - observations, static_inputs = ( - {name: mapping.get(name, name) for name in names} - for names, mapping in ( - (policy.observation_columns, columns.observations), - (policy.static_columns, columns.static_inputs), - ) - ) - inputs = set(observations.values()) - if static_data is None: - inputs.update(static_inputs.values()) - if action_column in inputs or columns.previous_action == action_column: - raise ValueError( - "current action labels require distinct input/previous-action columns" - ) - if ( - set(columns.observations) - observations.keys() - or set(columns.static_inputs) - static_inputs.keys() - ): - raise ValueError("table mappings must name declared policy features") - required = ( - inputs - | {columns.item_position, action_column} - | set(columns.successors.values()) - ) - required.update( - filter( - None, - ( - columns.previous_action, - columns.bootstrap_allowed, - columns.group, - columns.episode_id, - ), - ) - ) - if required - set(data.columns): - raise ValueError( - f"policy table is missing columns: {sorted(required - set(data.columns))}" - ) - - def values(column, decoder=decode): - return tuple(decoder(row[column], column) for row in rows) - - def optional_values(column, decoder=decode): - if ( - column is None - or column not in data.columns - or data[column].null_count() == len(rows) - ): - return None - return values(column, decoder) - - def constant(entries, label): - unique = set(entries) - if len(unique) != 1: - raise ValueError(f"{label} must remain constant within an episode") - return next(iter(unique)) - - positions = values(columns.item_position, lambda value, _: value) - actions = values(action_column) - if columns.previous_action is not None: - preceding_actions = values( - columns.previous_action, - lambda value, _: None if value is None else decode(value, action_column), - ) - if preceding_actions[1:] != actions[:-1]: - raise ValueError( - "previous-action column must lag executed actions by exactly one decision" - ) - preceding = preceding_actions[0] - if static_data is not None: - if len(static_data) != 1: - raise ValueError("static data requires exactly one row per sequenceId") - static_rows = list(static_data.iter_rows(named=True)) - else: - static_rows = rows - static = { - name: constant( - (decode(row[column], column) for row in static_rows), - f"static input {name}", - ) - for name, column in static_inputs.items() - } - # Strict boolean and length checks belong to TransitionColumns/SequenceEpisode. - transitions = { - field: optional_values(getattr(columns, field), lambda value, _: value) - for field in ("terminated", "truncated", "bootstrap_allowed") - } - transitions = TransitionColumns(**transitions).model_dump(exclude_none=True) - declared = episode_end - if declared != "unknown": - current = transitions.get(declared) - if current is not None and not current[-1]: - raise ValueError("episode_end conflicts with the recorded final boundary") - transitions[declared] = (False,) * (len(rows) - 1) + (True,) - successors = None - if columns.successors: - if columns.successors.keys() != observations.keys(): - raise ValueError("successor mapping must cover every observation column") - successors = [] - for row in rows: - present = [ - row[column] is not None for column in columns.successors.values() - ] - if any(present) and not all(present): - raise ValueError("a missing successor must mask all observation fields") - successors.append( - { - name: decode(row[column], observations[name]) - for name, column in columns.successors.items() - } - if all(present) - else None - ) - groups = optional_values(columns.group, domain_scalar) - identity = ( - constant(values(columns.episode_id, domain_scalar), "episode ID") - if columns.episode_id - else sequence_id - ) - return SequenceEpisode( - episode_id=episode_namespace + str(identity), - item_positions=positions, - observations={name: values(column) for name, column in observations.items()}, - actions=actions, - static_inputs=static, - starts_episode=positions[0] == 0 and preceding is None, - previous_action=preceding, - group_id=str(constant(groups, "collection group")) - if groups is not None - else None, - transitions=TransitionColumns( - **transitions, - rewards=optional_values( - columns.reward, - lambda value, column: numeric_scalar(decode(value, column), "reward"), - ), - successors=tuple(successors) if successors is not None else None, - ), - ) - - def source_episodes(source: SequenceSourceConfig, policy: PolicySchemaConfig): - if source.split_index is not None: - from sequifier.config.policy_train_config import resolve_sequence_source - - source = resolve_sequence_source(source, Path.cwd()) - paths = tuple(source_paths(source)) - if source.manifest is not None: - manifest = json.loads(Path(source.manifest).read_text(encoding="utf-8")) - if manifest.get("format") == "sequifier.sequence-data": - recorded_columns = PolicyTableColumns.model_validate(manifest["columns"]) - if ( - source.columns != PolicyTableColumns() - and source.columns != recorded_columns - ): - raise ValueError( - "source mappings disagree with the committed sequence manifest" - ) - if source.encoding != "raw" or source.episode_end != "unknown": - raise ValueError( - "committed sequence data already declares its encoding and boundaries" - ) - source = source.model_copy( - update={"columns": recorded_columns, "format": "table"} - ) - kind = source.format - if kind == "auto": - suffixes = {path.suffix.lower() for path in paths} - if suffixes <= {".json"}: - kind = "trajectory" - elif suffixes <= {".csv", ".parquet"}: - kind = "table" - elif suffixes == {".pt"}: - kind = "prepared" - else: - raise ValueError( - "select a source format explicitly for mixed or unsupported files" - ) - if kind == "trajectory": + resolved = resolve_source(source) + source, paths = resolved.source, resolved.paths + if resolved.kind == "trajectory": return [ TrajectoryReference( path, policy, source.action_semantics, source.episode_namespace ) for path in paths ] - prepared = kind == "prepared" - data = _prepared_table(paths, source).lazy() if prepared else _scan(paths) - columns = source.columns - required = {columns.sequence_id, columns.item_position} - if required - set(data.collect_schema().names()): - raise ValueError( - "policy tables require sequenceId and itemPosition, or explicit column mappings" - ) - table = IndexedTable(data, columns.sequence_id) - static_table = None - if source.static_data_path: - static_table = IndexedTable( - _scan((Path(source.static_data_path),)), columns.sequence_id - ) - if ( - any(end - start != 1 for start, end in static_table.ranges.values()) - or not table.ranges.keys() <= static_table.ranges.keys() - ): - raise ValueError( - "static data must cover every selected sequence ID without duplicates" - ) - metadata = ( - load_dataset_metadata(source.metadata_config_path) - if source.metadata_config_path - else None - ) - return [ - TableEpisodeReference( - source, policy, identity, table, metadata, prepared, static_table - ) - for identity in table.ranges - ] - - -def import_sequences(config) -> Path: - """Commit canonical columnar episodes as checksummed raw Parquet shards.""" - import pyarrow as pa - import pyarrow.parquet as pq - - source = config.resolved_source() - destination = (Path(config.project_root) / config.output_path).resolve() - if destination.exists(): - raise ValueError( - "sequence import output_path must name a new dataset directory" - ) - destination.parent.mkdir(parents=True, exist_ok=True) - columns = PolicyTableColumns( - action="__executed_action", - reward="__reward", - terminated="__terminated", - truncated="__truncated", - previous_action="__previous_action", - bootstrap_allowed="__bootstrap", - group="__group", - episode_id="episodeId", - successors={ - name: "__next_" + name for name in config.policy.observation_columns - }, - ) - fields = config.policy.observation_columns + config.policy.static_columns - reserved = { - value for value in columns.model_dump().values() if isinstance(value, str) - } | set(columns.successors.values()) - if set(fields) & reserved: - raise ValueError( - "policy feature names collide with sequence import metadata columns" - ) - references = source_episodes(source, config.policy) - if not references: - raise ValueError("sequence import requires at least one episode") - real = ( - set(fields) - - set(config.policy.categorical_columns) - - config.policy.categorical_n_classes.keys() - ) - action_type = ( - pa.string() if isinstance(config.policy.action_values[0], str) else pa.float64() - ) - types = {name: pa.float64() for name in real | {"__reward"}} - types.update( - {name: pa.bool_() for name in ("__terminated", "__truncated", "__bootstrap")} - ) - types.update( - { - "__executed_action": action_type, - "__previous_action": action_type, - "__group": pa.string(), - } - ) - stage = Path(tempfile.mkdtemp(prefix=".sequifier-import-", dir=destination.parent)) - tables, shards, seen, schema = [], [], set(), None - - def flush(): - nonlocal schema - if tables: - table = pa.concat_tables(tables, promote_options="permissive") - schema = schema or table.schema - name = f"part-{len(shards):05d}.parquet" - pq.write_table(table.cast(schema), stage / name) - shards.append( - { - "path": name, - "sha256": _checksum(stage / name), - "decisions": table.num_rows, - } - ) - tables.clear() - - try: - for index, reference in enumerate(references): - episode = reference.load() - if episode.episode_id in seen: - raise ValueError("duplicate episode IDs in sequence import") - seen.add(episode.episode_id) - length = len(episode.actions) - transitions = episode.transitions or TransitionColumns() - values = { - **episode.observations, - **{ - name: [value] * length - for name, value in episode.static_inputs.items() - }, - "sequenceId": [index] * length, - "itemPosition": episode.item_positions, - "episodeId": [episode.episode_id] * length, - "__executed_action": episode.actions, - "__previous_action": (episode.previous_action, *episode.actions[:-1]), - "__group": [episode.group_id] * length, - **{ - column: getattr(transitions, field) or [None] * length - for field, column in { - "rewards": "__reward", - "terminated": "__terminated", - "truncated": "__truncated", - "bootstrap_allowed": "__bootstrap", - }.items() - }, - } - arrays = { - name: pa.array(entries, type=types.get(name)) - for name, entries in values.items() - } - for name, column in columns.successors.items(): - arrays[column] = pa.array( - [ - successor[name] if successor is not None else None - for successor in (transitions.successors or [None] * length) - ], - type=arrays[name].type, - ) - tables.append(pa.table(arrays)) - if (index + 1) % config.episodes_per_shard == 0: - flush() - flush() - manifest = { - "format": "sequifier.sequence-data", - "version": 1, - "episodes": len(seen), - "decisions": sum(shard["decisions"] for shard in shards), - "columns": columns.model_dump(), - "policy_schema": config.policy.model_dump(mode="json"), - "shards": shards, - "provenance": { - "source": source.model_dump(mode="json"), - "files": [ - {"path": str(path), "sha256": _checksum(path)} - for path in source_paths(source) - ], - }, - } - (stage / "manifest.json").write_text( - json.dumps(manifest, indent=2, allow_nan=False) + "\n", encoding="utf-8" - ) - os.rename(stage, destination) - return destination / "manifest.json" - finally: - if stage.exists(): - shutil.rmtree(stage) + return table_episodes(source, policy, paths, prepared=resolved.kind == "prepared") diff --git a/src/sequifier/trajectory/prepared.py b/src/sequifier/trajectory/prepared.py new file mode 100644 index 00000000..2d82029d --- /dev/null +++ b/src/sequifier/trajectory/prepared.py @@ -0,0 +1,74 @@ +"""Reconstruct split-owned decisions from overlapping prepared histories.""" + +from pathlib import Path + +import polars as pl + +from sequifier.config.metadata import load_dataset_metadata +from sequifier.config.policy_train_config import SequenceSourceConfig + + +def prepared_table( + paths: tuple[Path, ...], source: SequenceSourceConfig +) -> pl.DataFrame: + """Recover unique positions from existing windows, validating overlaps. + + Split target bounds select decisions. A preceding action may be recovered + from stored context, but observations outside the split are not targets. + """ + if source.metadata_config_path is None: + raise ValueError("prepared histories require metadata_config_path") + from sequifier.helpers import PANDAS_TO_TORCH_TYPES, read_data + from sequifier.io.pt_payload import load_pt_payload, tabular_tensor_batch + + metadata = load_dataset_metadata(source.metadata_config_path) + if metadata.depth_layouts: + raise ValueError( + "policy table import currently supports scalar prepared columns" + ) + rows = {} + for path in paths: + if path.suffix.lower() == ".pt": + batch = load_pt_payload(path, n_classes=metadata.n_classes) + else: + data = read_data(str(path), path.suffix.lower().lstrip(".")) + batch = tabular_tensor_batch( + data, + { + column: PANDAS_TO_TORCH_TYPES[dtype] + for column, dtype in metadata.column_data_types.items() + }, + sorted(set(data["inputCol"].to_list())), + metadata.window_length, + ) + sequences, ids, starts, pads = ( + batch.sequences, + batch.sequence_ids, + batch.start_item_positions, + batch.left_pad_lengths, + ) + lows, highs = batch.split_start_item_positions, batch.split_end_item_positions + for row in range(len(ids)): + for offset in range( + int(pads[row]), next(iter(sequences.values())).shape[1] + ): + position = int(starts[row]) + offset + key = (int(ids[row]), position) + values = { + column: tensor[row, offset].item() + for column, tensor in sequences.items() + } + record = { + "sequenceId": key[0], + "itemPosition": position, + **values, + "__decision": int(lows[row]) <= position < int(highs[row]), + } + if key in rows and rows[key] != record: + raise ValueError( + "prepared history windows disagree on overlapping positions or split bounds" + ) + rows[key] = record + if not rows: + raise ValueError("prepared source contains no valid positions") + return pl.DataFrame(list(rows.values())).sort(["sequenceId", "itemPosition"]) diff --git a/src/sequifier/trajectory/sources.py b/src/sequifier/trajectory/sources.py new file mode 100644 index 00000000..8771320e --- /dev/null +++ b/src/sequifier/trajectory/sources.py @@ -0,0 +1,104 @@ +"""Resolve sequence source files, manifests, and declared formats.""" + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path + +from sequifier.config.policy_train_config import ( + PolicyTableColumns, + SequenceSourceConfig, + resolve_sequence_source, +) +from sequifier.trajectory.store import load_trajectory + + +def file_checksum(path): + digest = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _source_files(source): + manifest = None + if source.manifest is None: + paths = [Path(path) for path in source.paths] + else: + manifest_path = Path(source.manifest) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict) or manifest.get("version") != 1: + raise ValueError("source manifest must have version 1") + sequence_manifest = manifest.get("format") == "sequifier.sequence-data" + entries = manifest.get("shards" if sequence_manifest else "episodes") + if not isinstance(entries, list) or not entries: + raise ValueError("source manifest requires a nonempty episodes list") + paths = [] + for entry in entries: + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("path"), str) + or not entry["path"] + ): + raise ValueError("each manifest entry requires a nonempty path") + path = manifest_path.parent / entry["path"] + if sequence_manifest and file_checksum(path) != entry.get("sha256"): + raise ValueError(f"sequence shard checksum disagrees with {path}") + if ( + "episode_id" in entry + and load_trajectory(path).episode_id != entry["episode_id"] + ): + raise ValueError(f"manifest episode ID disagrees with {path}") + paths.append(path) + paths = [path.resolve() for path in paths] + if not paths or len(set(paths)) != len(paths): + raise ValueError("sources require nonempty, unique paths") + return paths, manifest + + +def source_paths(source: SequenceSourceConfig) -> list[Path]: + return _source_files(source)[0] + + +@dataclass(frozen=True) +class ResolvedSequenceSource: + source: SequenceSourceConfig + paths: tuple[Path, ...] + kind: str + + +def resolve_source(source: SequenceSourceConfig) -> ResolvedSequenceSource: + if source.split_index is not None: + source = resolve_sequence_source(source, Path.cwd()) + paths, manifest = _source_files(source) + if manifest is not None and manifest.get("format") == "sequifier.sequence-data": + recorded_columns = PolicyTableColumns.model_validate(manifest["columns"]) + if ( + source.columns != PolicyTableColumns() + and source.columns != recorded_columns + ): + raise ValueError( + "source mappings disagree with the committed sequence manifest" + ) + if source.encoding != "raw" or source.episode_end != "unknown": + raise ValueError( + "committed sequence data already declares its encoding and boundaries" + ) + source = source.model_copy( + update={"columns": recorded_columns, "format": "table"} + ) + kind = source.format + if kind == "auto": + suffixes = {path.suffix.lower() for path in paths} + if suffixes <= {".json"}: + kind = "trajectory" + elif suffixes <= {".csv", ".parquet"}: + kind = "table" + elif suffixes == {".pt"}: + kind = "prepared" + else: + raise ValueError( + "select a source format explicitly for mixed or unsupported files" + ) + return ResolvedSequenceSource(source, tuple(paths), kind) diff --git a/src/sequifier/trajectory/tables.py b/src/sequifier/trajectory/tables.py new file mode 100644 index 00000000..ad0c152f --- /dev/null +++ b/src/sequifier/trajectory/tables.py @@ -0,0 +1,358 @@ +"""Indexed table episode readers and canonical table-to-episode validation.""" + +import tempfile +from bisect import bisect_right +from dataclasses import dataclass +from pathlib import Path + +import polars as pl + +from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata +from sequifier.config.policy_config import PolicySchemaConfig +from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.io.preprocessing import denormalize_values +from sequifier.io.sequence import ( + SequenceEpisode, + TransitionColumns, + ordered_sequence_rows, +) +from sequifier.io.values import domain_scalar, numeric_scalar +from sequifier.trajectory.prepared import prepared_table + + +def scan_tables(paths: tuple[Path, ...]): + scans = [] + for path in paths: + if path.suffix.lower() == ".csv": + scans.append(pl.scan_csv(path, infer_schema_length=None)) + elif path.suffix.lower() == ".parquet": + scans.append(pl.scan_parquet(path)) + else: + raise ValueError("table sources require CSV or Parquet files") + return pl.concat(scans, how="vertical_relaxed") + + +class IndexedTable: + """Canonical sorted storage with direct row-group access per episode. + + The temporary directory lives as long as its episode references. CSV parsing + and sorting happen once; cache misses never filter the original source. + """ + + def __init__(self, data, sequence_column): + import pyarrow.parquet as pq + + self.directory = tempfile.TemporaryDirectory(prefix="sequifier-episodes-") + self.path = Path(self.directory.name) / "episodes.parquet" + data.sort(sequence_column, maintain_order=True).sink_parquet( + self.path, row_group_size=8192 + ) + table = pq.ParquetFile(self.path) + self.offsets = [0] + for i in range(table.num_row_groups): + self.offsets.append(self.offsets[-1] + table.metadata.row_group(i).num_rows) + self.ranges = {} + offset = 0 + for batch in table.iter_batches(columns=[sequence_column]): + for identity in batch.column(0).to_pylist(): + if isinstance(identity, bool) or not isinstance(identity, (int, str)): + raise ValueError( + "sequence IDs must be non-null integers or strings" + ) + if identity not in self.ranges: + self.ranges[identity] = [offset, offset + 1] + else: + self.ranges[identity][1] = offset + 1 + offset += 1 + table.close() + + def load(self, identity): + import pyarrow.parquet as pq + + start, end = self.ranges[identity] + first = bisect_right(self.offsets, start) - 1 + last = bisect_right(self.offsets, end - 1) - 1 + with pq.ParquetFile(self.path) as table: + rows = table.read_row_groups(list(range(first, last + 1))) + return pl.from_arrow(rows.slice(start - self.offsets[first], end - start)) + + +@dataclass(frozen=True) +class TableEpisodeReference: + source: SequenceSourceConfig + policy: PolicySchemaConfig + sequence_id: str | int + table: IndexedTable + metadata: DatasetMetadata | None = None + prepared: bool = False + static_table: IndexedTable | None = None + + def load(self) -> SequenceEpisode: + columns = self.source.columns + data = ordered_sequence_rows( + self.table.load(self.sequence_id), columns.item_position + ) + metadata = self.metadata + model_encoded = self.prepared or self.source.encoding == "model" + + def decode(value, column): + value = domain_scalar(value, column) + if model_encoded and metadata is not None: + if column in metadata.id_maps: + inverse = { + encoded: label + for label, encoded in metadata.id_maps[column].items() + } + if ( + isinstance(value, str) + or value != int(value) + or int(value) not in inverse + ): + raise ValueError(f"{column}: encoded ID has no domain label") + value = inverse[int(value)] + if column == (columns.action or self.policy.action_column): + if isinstance(value, str) and not isinstance( + self.policy.action_values[0], str + ): + try: + value = numeric_scalar(float(value), column) + except ValueError: + pass + matches = [ + action + for action in self.policy.action_values + if action == value or str(action) == str(value) + ] + if len(matches) == 1: + value = matches[0] + elif ( + metadata.normalize_real_columns + and column in metadata.selected_columns_statistics + ): + value = denormalize_values( + numeric_scalar(value, column), + metadata.selected_columns_statistics[column], + ) + return value + + preceding = None + if self.prepared: + decisions = data.filter(pl.col("__decision")) + if decisions.is_empty(): + raise ValueError("prepared episode contains no split-owned decisions") + first = decisions[columns.item_position][0] + previous = data.filter(pl.col(columns.item_position) == first - 1) + if len(previous): + preceding = decode( + previous[columns.action or self.policy.action_column][0], + columns.action or self.policy.action_column, + ) + data = decisions + static_data = None + if self.static_table is not None: + static_data = self.static_table.load(self.sequence_id) + return table_episode( + data, + self.policy, + columns, + sequence_id=self.sequence_id, + preceding=preceding, + static_data=static_data, + episode_end=self.source.episode_end, + episode_namespace=self.source.episode_namespace, + decode=decode, + ) + + +def table_episode( + data, + policy, + columns, + *, + sequence_id, + preceding=None, + static_data=None, + episode_end="unknown", + episode_namespace="", + decode=domain_scalar, +) -> SequenceEpisode: + """Canonical table-to-episode validation shared by training and inference.""" + data = ordered_sequence_rows(data, columns.item_position) + rows = list(data.iter_rows(named=True)) + if not rows: + raise ValueError("sequence source changed while loading") + action_column = columns.action or policy.action_column + observations, static_inputs = ( + {name: mapping.get(name, name) for name in names} + for names, mapping in ( + (policy.observation_columns, columns.observations), + (policy.static_columns, columns.static_inputs), + ) + ) + inputs = set(observations.values()) + if static_data is None: + inputs.update(static_inputs.values()) + if action_column in inputs or columns.previous_action == action_column: + raise ValueError( + "current action labels require distinct input/previous-action columns" + ) + if ( + set(columns.observations) - observations.keys() + or set(columns.static_inputs) - static_inputs.keys() + ): + raise ValueError("table mappings must name declared policy features") + required = ( + inputs + | {columns.item_position, action_column} + | set(columns.successors.values()) + ) + required.update( + filter( + None, + ( + columns.previous_action, + columns.bootstrap_allowed, + columns.group, + columns.episode_id, + ), + ) + ) + if required - set(data.columns): + raise ValueError( + f"policy table is missing columns: {sorted(required - set(data.columns))}" + ) + + def values(column, decoder=decode): + return tuple(decoder(row[column], column) for row in rows) + + def optional_values(column, decoder=decode): + if ( + column is None + or column not in data.columns + or data[column].null_count() == len(rows) + ): + return None + return values(column, decoder) + + def constant(entries, label): + unique = set(entries) + if len(unique) != 1: + raise ValueError(f"{label} must remain constant within an episode") + return next(iter(unique)) + + positions = values(columns.item_position, lambda value, _: value) + actions = values(action_column) + if columns.previous_action is not None: + preceding_actions = values( + columns.previous_action, + lambda value, _: None if value is None else decode(value, action_column), + ) + if preceding_actions[1:] != actions[:-1]: + raise ValueError( + "previous-action column must lag executed actions by exactly one decision" + ) + preceding = preceding_actions[0] + if static_data is not None: + if len(static_data) != 1: + raise ValueError("static data requires exactly one row per sequenceId") + static_rows = list(static_data.iter_rows(named=True)) + else: + static_rows = rows + static = { + name: constant( + (decode(row[column], column) for row in static_rows), + f"static input {name}", + ) + for name, column in static_inputs.items() + } + # Strict boolean and length checks belong to TransitionColumns/SequenceEpisode. + transitions = { + field: optional_values(getattr(columns, field), lambda value, _: value) + for field in ("terminated", "truncated", "bootstrap_allowed") + } + transitions = TransitionColumns(**transitions).model_dump(exclude_none=True) + declared = episode_end + if declared != "unknown": + current = transitions.get(declared) + if current is not None and not current[-1]: + raise ValueError("episode_end conflicts with the recorded final boundary") + transitions[declared] = (False,) * (len(rows) - 1) + (True,) + successors = None + if columns.successors: + if columns.successors.keys() != observations.keys(): + raise ValueError("successor mapping must cover every observation column") + successors = [] + for row in rows: + present = [ + row[column] is not None for column in columns.successors.values() + ] + if any(present) and not all(present): + raise ValueError("a missing successor must mask all observation fields") + successors.append( + { + name: decode(row[column], observations[name]) + for name, column in columns.successors.items() + } + if all(present) + else None + ) + groups = optional_values(columns.group, domain_scalar) + identity = ( + constant(values(columns.episode_id, domain_scalar), "episode ID") + if columns.episode_id + else sequence_id + ) + return SequenceEpisode( + episode_id=episode_namespace + str(identity), + item_positions=positions, + observations={name: values(column) for name, column in observations.items()}, + actions=actions, + static_inputs=static, + starts_episode=positions[0] == 0 and preceding is None, + previous_action=preceding, + group_id=str(constant(groups, "collection group")) + if groups is not None + else None, + transitions=TransitionColumns( + **transitions, + rewards=optional_values( + columns.reward, + lambda value, column: numeric_scalar(decode(value, column), "reward"), + ), + successors=tuple(successors) if successors is not None else None, + ), + ) + + +def table_episodes(source, policy, paths, *, prepared=False): + data = prepared_table(paths, source).lazy() if prepared else scan_tables(paths) + columns = source.columns + required = {columns.sequence_id, columns.item_position} + if required - set(data.collect_schema().names()): + raise ValueError( + "policy tables require sequenceId and itemPosition, or explicit column mappings" + ) + table = IndexedTable(data, columns.sequence_id) + static_table = None + if source.static_data_path: + static_table = IndexedTable( + scan_tables((Path(source.static_data_path),)), columns.sequence_id + ) + if ( + any(end - start != 1 for start, end in static_table.ranges.values()) + or not table.ranges.keys() <= static_table.ranges.keys() + ): + raise ValueError( + "static data must cover every selected sequence ID without duplicates" + ) + metadata = ( + load_dataset_metadata(source.metadata_config_path) + if source.metadata_config_path + else None + ) + return [ + TableEpisodeReference( + source, policy, identity, table, metadata, prepared, static_table + ) + for identity in table.ranges + ] diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 123a4787..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Use the working tree, independently of any installed Sequifier version.""" - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) diff --git a/tests/unit/test_policy_architecture.py b/tests/unit/test_policy_architecture.py deleted file mode 100644 index 810a6c36..00000000 --- a/tests/unit/test_policy_architecture.py +++ /dev/null @@ -1,560 +0,0 @@ -"""Behavioral coverage for resolved policy contracts, data ownership and recovery.""" - -import json -import runpy -from copy import deepcopy -from pathlib import Path -from unittest.mock import Mock - -import polars as pl -import pytest -import torch -import yaml - -from sequifier.config.metadata import DatasetMetadata -from sequifier.config.policy_plan import PolicyImitationPhaseConfig -from sequifier.config.train_config import ( - GlobalTrainingSpecModel, - ModelInterfaceSpecModel, - PolicyRunConfig, - _resolve_interface, -) -from sequifier.io.partitioning import split_components -from sequifier.io.policy_window import PolicyWindowView, action_global_ids -from sequifier.rl import run_checkpoint -from sequifier.rl.checkpoint import load_policy, save_policy -from sequifier.rl.collection import decide -from sequifier.rl.distributions import action_generator -from sequifier.rl.dpo import dpo_loss -from sequifier.rl.learner import run_policy_training -from sequifier.rl.models import HistoryCritic -from sequifier.rl.preparation import ( - build_actor, - input_identity, - prepare_data, - validate_data, -) -from sequifier.rl.run_checkpoint import RunCheckpoint -from sequifier.rl.run_state import weight_fingerprint -from sequifier.trajectory import importers - -EXAMPLES = Path(__file__).resolve().parents[2] / "documentation/example_inputs" - - -@pytest.fixture(scope="module") -def run_values(tmp_path_factory): - torch.set_num_threads(1) - root = tmp_path_factory.mktemp("policy") - runpy.run_path(str(EXAMPLES / "policy_control_data.py"))["generate"](root / "data") - path = root / "data/control-decisions.csv" - data = pl.read_csv(path) - data.filter( - pl.col("contextId").is_in([f"context-{i}" for i in range(4)]) - ).write_csv(path) - path = root / "data/control-preferences.json" - preferences = json.loads(path.read_text()) - preferences["pairs"] = preferences["pairs"][:4] - path.write_text(json.dumps(preferences)) - values = yaml.safe_load( - (EXAMPLES / "configs/train-multiphase-control.yaml").read_text() - ) - values.update( - project_root=str(root), - context_length=2, - episode_cache_size=1, - resume={"policy": "never", "mode": "exact"}, - ) - values["model"]["backbone"]["architecture"].update( - dim_model=8, - max_context_length=2, - attention={"n_heads": 2}, - feed_forward={"dim": 16}, - ) - for phase in values["phases"]: - phase.update(evaluation_interval=1, checkpoint_interval=1) - if phase["algorithm"] == "policy_imitation": - phase.update(epochs=1, batch_size=8) - elif phase["algorithm"] == "iql": - phase.update(updates=2, batch_size=4) - elif phase["algorithm"] == "dpo": - phase.update(updates=2, batch_size=2, scoring_chunk_size=3) - elif phase["algorithm"] == "grpo": - phase.update( - total_groups=2, - groups_per_update=1, - group_size=2, - max_steps=2, - optimization_epochs=1, - batch_size=2, - ) - elif phase["algorithm"] == "ppo": - phase.update( - total_env_steps=4, - rollout_length=2, - optimization_epochs=1, - minibatch_size=2, - ) - if "environment" in phase: - phase["environment"]["config"] = {"horizon": 2} - evaluation = phase["evaluation"] - evaluation.update(seeds=[17], max_episode_steps=2) - evaluation["environment"]["config"] = {"horizon": 2} - return values - - -def config_for(values, name, *, resume="never"): - values = deepcopy(values) - values["model_name"] = name - values["resume"]["policy"] = resume - return PolicyRunConfig.model_validate(values) - - -@pytest.fixture(scope="module") -def completed(run_values): - config = config_for(run_values, "uninterrupted") - return config, run_policy_training(config) - - -def test_action_vocabulary_and_portable_roundtrip(run_values, tmp_path): - config = config_for(run_values, "vocabulary") - actor = build_actor(config, config.policy, {"cue": {"mean": 0.0, "std": 1.0}}) - target = actor.interface.target_columns[0] - assert ( - actor.interface.target_decoder_ids[target] - == action_global_ids(actor.config) - == [3, 4] - ) - assert actor.interface.target_global_to_decoder[target] == [-1, -1, -1, 0, 1] - assert actor.interface.target_n_classes[target] == 2 - assert actor.interface.categorical_decoder_special_tokens == {target: []} - history = PolicyWindowView(actor.config).inputs([{"cue": 1.0}], [2], None, {}) - logits = actor(history) - assert logits.shape[-1] == 2 - path = tmp_path / "actor.pt" - save_policy(actor, path) - restored = load_policy(path) - assert torch.equal(restored(history), logits) - # Validate a decision once, sharing the checked transforms with the critic. - critic = HistoryCritic(deepcopy(actor.execution_config), actor.interface_name, None) - actor.validate_inputs = Mock(wraps=actor.validate_inputs) - decide(actor, critic, history, action_generator(actor.device, 1)) - actor.validate_inputs.assert_called_once() - broken = deepcopy(history) - broken.inputs.pop("cue") - with pytest.raises(ValueError, match="persisted named feature schema"): - decide(actor, None, broken, action_generator(actor.device, 1)) - - -def test_supervised_decoder_defaults_stay_unchanged(): - spec = ModelInterfaceSpecModel.model_validate( - { - "input_columns": ["x"], - "target_columns": ["y"], - "ingestion": {"type": "embedding", "output_dim": 4}, - "decoder": {"type": "linear", "prediction_length": 1}, - } - ) - metadata = DatasetMetadata( - window_length=3, - column_data_types={"x": "Float32", "y": "Int64"}, - n_classes={"y": 5}, - selected_columns_statistics={"x": {"mean": 2.0, "std": 3.0}}, - ) - training = GlobalTrainingSpecModel( - training_objective="causal", - inference_batch_size=1, - context_length=2, - batch_size=1, - learning_rate=1e-3, - ) - resolved = _resolve_interface( - "forecast", spec, metadata, training, metadata.storage_layout - ) - assert resolved.policy is None - assert resolved.target_decoder_ids == {"y": [1, 3, 4]} - assert resolved.categorical_decoder_special_tokens == {"y": ["other"]} - assert resolved.selected_columns_statistics == metadata.selected_columns_statistics - - -def test_shared_corpus_transitions_and_bounded_access(run_values, monkeypatch): - scan = Mock(wraps=importers._scan) - monkeypatch.setattr(importers, "_scan", scan) - from sequifier.io.iql_dataset import IQLTransitionDataset - - initialize = IQLTransitionDataset.__init__ - calls = [] - - def count(self, dataset): - calls.append(dataset) - initialize(self, dataset) - - monkeypatch.setattr(IQLTransitionDataset, "__init__", count) - config = config_for(run_values, "views") - data = prepare_data(config, config.policy) - assert data[0] is data[1] - assert scan.call_count == 1 - assert len(calls) == 2 - actor = build_actor(config, config.policy, {"cue": {"mean": 0.0, "std": 1.0}}) - validate_data(config, data, actor) - assert len(calls) == 2 - transitions = data[1].prepare_transitions() - assert transitions is data[1].prepare_transitions() - training, validation = data[0].training, data[0].validation - assert set(training.group_ids).isdisjoint(validation.group_ids) - assert set(training.demonstration_keys).isdisjoint(validation.demonstration_keys) - for part in (training, validation): - for index in range(len(part)): - part[index] - for corpus, _ in part.episodes.entries: - assert len(corpus.cache) <= config.episode_cache_size - assert scan.call_count == 1 - order = list(training.sampler(torch.Generator().manual_seed(1))) - assert sorted(order) == list(range(len(training))) - episode_indices = [ - next(i for i, end in enumerate(training.cumulative_lengths) if decision < end) - for decision in order - ] - assert len( - [ - i - for i, e in enumerate(episode_indices) - if i == 0 or e != episode_indices[i - 1] - ] - ) == len(training.episodes) - - -def test_transitive_components_are_never_split(): - keys = {0: ["a"], 1: ["a", "b"], 2: ["b"], 3: ["c"], 4: ["d"]} - training, validation = split_components(keys, keys.__getitem__, 0.4, 17) - assert {0, 1, 2} <= set(training) or {0, 1, 2} <= set(validation) - assert (training, validation) == split_components(keys, keys.__getitem__, 0.4, 17) - with pytest.raises(ValueError, match="independent components"): - split_components([0, 1, 2], keys.__getitem__, 0.5, 1) - - -@pytest.mark.parametrize("role", ["actor", "value", "critic"]) -def test_optimizer_roles_share_validation(run_values, role): - values = deepcopy(run_values["phases"][1]) - values[f"{role}_optimizer"] = {"name": "Adam", "arguments": {"lr": 1e-3}} - with pytest.raises(ValueError, match="phase learning rate"): - config_for({**run_values, "phases": [values]}, f"bad-{role}") - values.pop(f"{role}_optimizer") - values[f"{role}_scheduler"] = { - "name": "OneCycleLR", - "arguments": {"total_steps": 3, "max_lr": 0.001}, - } - with pytest.raises(ValueError, match="budget"): - config_for({**run_values, "phases": [values]}, f"bad-budget-{role}") - - -def test_imitation_scheduler_budget_uses_actual_partition(): - phase = PolicyImitationPhaseConfig.model_validate( - { - "name": "imitation", - "algorithm": "policy_imitation", - "sources": ["data"], - "epochs": 2, - "batch_size": 3, - "accumulation_steps": 2, - "actor_scheduler": { - "name": "OneCycleLR", - "arguments": {"total_steps": 4, "max_lr": 0.001}, - }, - } - ) - assert phase.optimization_plans(10)["actor"].step_budget == 4 - with pytest.raises(ValueError, match="budget"): - phase.optimization_plans(20) - - -def test_completed_run_resume_and_snapshot_deduplication(run_values, completed): - config, result = completed - resumed = run_policy_training( - config_for(run_values, config.model_name, resume="required") - ) - assert weight_fingerprint( - resumed.policy.network.state_dict() - ) == weight_fingerprint(result.policy.network.state_dict()) - assert resumed.phases == result.phases - payload = torch.load(config.checkpoint_path, weights_only=False) - assert payload["incoming_actor"] == result.phases[-1].selected_weight_hash - assert len(payload["snapshots"]) <= len(result.phases) - assert payload["phase_state"] is None - assert ( - not {"phase_index", "status", "last_committed_transaction", "selected_actors"} - & payload.keys() - ) - - -@pytest.mark.parametrize("phase_index", range(5)) -def test_interrupted_resume_matches_uninterrupted( - run_values, completed, monkeypatch, phase_index -): - config = config_for(run_values, f"interrupt-{phase_index}") - save = run_checkpoint.atomic_save - - def interrupt(path, payload): - save(path, payload) - if ( - len(payload["completed_results"]) == phase_index - and payload["phase_state"] is not None - ): - raise InterruptedError("simulated interruption after committed update") - - with monkeypatch.context() as patch: - patch.setattr(run_checkpoint, "atomic_save", interrupt) - with pytest.raises(InterruptedError): - run_policy_training(config) - recovered = run_policy_training( - config_for(run_values, config.model_name, resume="required") - ) - assert [p.selected_weight_hash for p in recovered.phases] == [ - p.selected_weight_hash for p in completed[1].phases - ] - assert weight_fingerprint( - recovered.policy.network.state_dict() - ) == weight_fingerprint(completed[1].policy.network.state_dict()) - - -def test_handoff_resume_republishes_missing_artifacts( - run_values, completed, monkeypatch -): - config = config_for(run_values, "handoff") - save = run_checkpoint.atomic_save - - def interrupt(path, payload): - save(path, payload) - if len(payload["completed_results"]) == 1 and payload["phase_state"] is None: - raise InterruptedError("simulated interruption before publication") - - with monkeypatch.context() as patch: - patch.setattr(run_checkpoint, "atomic_save", interrupt) - with pytest.raises(InterruptedError): - run_policy_training(config) - assert not (Path(config.phase_directory) / "000-imitation/actor.pt").exists() - result = run_policy_training( - config_for(run_values, config.model_name, resume="required") - ) - assert (Path(config.phase_directory) / "000-imitation/actor.pt").is_file() - assert [p.selected_weight_hash for p in result.phases] == [ - p.selected_weight_hash for p in completed[1].phases - ] - - -@pytest.mark.parametrize("corruption", ["weights", "result", "boundary"]) -def test_corrupt_checkpoint_is_rejected(run_values, completed, monkeypatch, corruption): - original_config, _ = completed - config = config_for(run_values, original_config.model_name, resume="required") - payload = torch.load(config.checkpoint_path, weights_only=False) - if corruption == "weights": - weights = next(iter(payload["snapshots"].values())) - next(iter(weights.values())).fill_(float("nan")) - elif corruption == "result": - payload["completed_results"][0]["index"] = 1 - else: - payload["phase_state"] = {"version": 1} - monkeypatch.setattr(torch, "load", lambda *args, **kwargs: payload) - plan = config.model_dump(mode="json", exclude={"resume"}) - with pytest.raises(ValueError): - RunCheckpoint.load(config.checkpoint_path, config, plan, input_identity(config)) - - -def test_dpo_objective_retains_gradient_semantics(): - chosen = torch.tensor([1.5], requires_grad=True) - loss, margin = dpo_loss( - chosen, torch.tensor([0.5]), torch.tensor([0.3]), torch.tensor([0.1]), 0.2 - ) - expected = -torch.nn.functional.logsigmoid(0.2 * ((chosen - 0.5) - (0.3 - 0.1))) - assert torch.equal(loss, expected) - loss.sum().backward() - assert chosen.grad is not None and chosen.grad.item() < 0 - assert margin.item() == pytest.approx(0.8) - - -def test_indexed_table_reads_exact_episode_across_row_groups(): - # One episode crosses two row-group boundaries; its neighbors must be excluded. - table = importers.IndexedTable( - pl.DataFrame( - { - "id": [1] * 7000 + [2] * 12000 + [3] * 1000, - "position": list(range(20000)), - } - ).lazy(), - "id", - ) - assert table.load(1)["position"].to_list() == list(range(7000)) - assert table.load(2)["position"].to_list() == list(range(7000, 19000)) - assert table.load(3)["position"].to_list() == list(range(19000, 20000)) - - -def test_canonical_import_preserves_episode_semantics(run_values, tmp_path): - from sequifier.config.policy_train_config import SequenceSourceConfig - from sequifier.config.sequence_import_config import SequenceImportConfig - - config = config_for(run_values, "import") - source = config.sources["decisions"] - expected = { - reference.load().episode_id: reference.load().model_dump() - for reference in importers.source_episodes(source, config.policy) - } - manifest = importers.import_sequences( - SequenceImportConfig.model_validate( - { - "operation": "import_sequences", - "project_root": str(tmp_path), - "output_path": "canonical", - "source": source.model_dump(), - "policy": config.policy.model_dump( - exclude={"interface_name", "context_length"} - ), - "episodes_per_shard": 3, - } - ) - ) - actual = { - reference.load().episode_id: reference.load().model_dump() - for reference in importers.source_episodes( - SequenceSourceConfig(manifest=str(manifest)), config.policy - ) - } - assert actual == expected - - -def test_return_conditioning_and_fitted_categorical_inputs(run_values): - from sequifier.config.policy_config import CategoricalPolicyConfig - from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode - from sequifier.io.sequence import SequenceEpisode, TransitionColumns - - config = CategoricalPolicyConfig.model_validate( - { - "observation_columns": ["cue"], - "categorical_columns": ["cue"], - "static_columns": ["bias"], - "context_length": 2, - "interface_name": "actor", - "action_values": ["left", "right"], - "returns": {"gamma": 0.5, "scale": 2.0}, - } - ) - episodes = [ - SequenceEpisode( - episode_id=str(i), - item_positions=(0, 1), - observations={"cue": ("red", "blue")}, - static_inputs={"bias": float(i)}, - actions=("left", "right"), - transitions=TransitionColumns(rewards=(1.0, 2.0), terminated=(False, True)), - ) - for i in range(4) - ] - dataset = PolicySequenceDataset(episodes, config, episode_cache_size=1) - training, validation = split_by_episode(dataset, 0.25, 1) - fitted = training.fit_categorical_config() - assert fitted.id_maps["cue"]["red"] != fitted.id_maps["cue"]["blue"] - for part in (training, validation): - part.use_config(fitted) - part.validate_normalization(training.normalization_statistics()) - assert part[0].inputs["__policy_return"].tolist() == [[0.0, 2.0]] - assert part[1].inputs["__policy_return"].tolist() == [[2.0, 2.0]] - values = deepcopy(run_values) - values["model"]["interfaces"]["actor"]["policy"].update( - return_conditioned=True, - observation_columns=["cue"], - categorical_columns=["cue"], - static_columns=["bias"], - ) - values["model"]["interfaces"]["actor"]["input_columns"] = ["cue", "bias"] - values["model"]["interfaces"]["actor"]["ingestion"]["feature_embedding_dims"] = { - "cue": 2, - "bias": 2, - "__policy_previous_action": 2, - "__policy_return": 2, - } - values["phases"] = values["phases"][:1] - run = config_for(values, "conditioned") - actor = build_actor(run, fitted, training.normalization_statistics()) - assert "__policy_return" not in actor.interface.selected_columns_statistics - assert actor(training[0].model_inputs).shape[-1] == 2 - - -def test_embedded_reference_survives_artifact_removal( - run_values, tmp_path, monkeypatch -): - values = deepcopy(run_values) - values["phases"] = [values["phases"][2]] - config = config_for(values, "reference-seed") - actor = build_actor(config, config.policy, {"cue": {"mean": 0.0, "std": 1.0}}) - path = tmp_path / "reference.pt" - save_policy(actor, path) - values["phases"][0]["reference"] = {"source": "artifact", "path": str(path)} - config = config_for(values, "reference-removal") - save = run_checkpoint.atomic_save - - def interrupt(path, payload): - save(path, payload) - if payload["phase_state"] is not None: - raise InterruptedError("committed DPO reference") - - with monkeypatch.context() as patch: - patch.setattr(run_checkpoint, "atomic_save", interrupt) - with pytest.raises(InterruptedError): - run_policy_training(config) - path.unlink() - result = run_policy_training( - config_for(values, config.model_name, resume="required") - ) - assert len(result.phases) == 1 and result.phases[0].algorithm == "dpo" - - -def test_indexed_tables_preserve_model_encoding_and_static_fields(tmp_path): - from sequifier.config.policy_config import PolicySchemaConfig - from sequifier.config.policy_train_config import SequenceSourceConfig - - pl.DataFrame( - { - "sequenceId": [2, 1, 2, 1], - "itemPosition": [1, 1, 0, 0], - "cue": [1.0, -1.0, -1.0, 1.0], - "action": [4, 3, 3, 4], - } - ).write_csv(tmp_path / "episodes.csv") - pl.DataFrame({"sequenceId": [1, 2], "bias": [2.0, -2.0]}).write_csv( - tmp_path / "static.csv" - ) - metadata = DatasetMetadata( - window_length=3, - column_data_types={"cue": "Float32", "bias": "Float32", "action": "Int64"}, - n_classes={"action": 5}, - id_maps={"action": {"left": 3, "right": 4}}, - selected_columns_statistics={ - "cue": {"mean": 10.0, "std": 2.0}, - "bias": {"mean": 3.0, "std": 4.0}, - }, - ) - (tmp_path / "metadata.json").write_text(metadata.model_dump_json()) - source = SequenceSourceConfig( - paths=[str(tmp_path / "episodes.csv")], - static_data_path=str(tmp_path / "static.csv"), - metadata_config_path=str(tmp_path / "metadata.json"), - encoding="model", - ) - policy = PolicySchemaConfig( - observation_columns=["cue"], - static_columns=["bias"], - action_values=["left", "right"], - return_conditioned=False, - ) - episodes = { - episode.episode_id: episode - for reference in importers.source_episodes(source, policy) - for episode in [reference.load()] - } - assert episodes["1"].actions == ("right", "left") - assert episodes["2"].actions == ("left", "right") - assert episodes["1"].observations["cue"] == pytest.approx((12.0, 8.0)) - assert episodes["1"].static_inputs["bias"] == pytest.approx(11.0) - pl.DataFrame({"sequenceId": [1, 1, 2], "bias": [2.0, 2.0, -2.0]}).write_csv( - tmp_path / "static.csv" - ) - with pytest.raises(ValueError, match="without duplicates"): - importers.source_episodes(source, policy) From 23197b77248022d1aa88c0030f606b7434fa88fb Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 18:57:00 +0100 Subject: [PATCH 20/71] cleanup --- documentation/policy-learning.md | 43 +++++--- src/sequifier/consolidated-docs.md | 43 +++++--- src/sequifier/io/episode_corpus.py | 3 +- src/sequifier/io/iql_dataset.py | 18 ++++ src/sequifier/io/preference_dataset.py | 10 ++ src/sequifier/io/prepared_policy_data.py | 21 ++++ src/sequifier/rl/learner.py | 48 +++------ src/sequifier/rl/offline_data.py | 11 ++ src/sequifier/rl/phase_artifacts.py | 78 ++++++++++++++ src/sequifier/rl/preparation.py | 125 ++++++++++------------- src/sequifier/rl/registry.py | 3 +- src/sequifier/rl/run_checkpoint.py | 45 ++++---- src/sequifier/rl/run_state.py | 16 +++ src/sequifier/trajectory/importers.py | 23 +++-- src/sequifier/trajectory/sources.py | 50 +++++++-- 15 files changed, 364 insertions(+), 173 deletions(-) create mode 100644 src/sequifier/io/prepared_policy_data.py create mode 100644 src/sequifier/rl/phase_artifacts.py diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 26684ad8..0635ab19 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -372,22 +372,41 @@ row-group access. The episode cache stays bounded; partition views retain metada and validated transition indices. Imitation shuffles episodes and then decisions within each episode to preserve storage locality. -The only policy-training checkpoint is `sequifier_policy_run`, format version 3. +Inputs are resolved once into a run-owned catalog. Source resolution returns +verified file checksums, including the checksums checked against sequence manifests; +preparation reuses those resolved sources without resolving or hashing them again. +Every phase boundary explicitly rehashes all input files, manifests, metadata, +static data and preference files to detect modifications. Output and archive path +conflicts are checked separately before output mutation and at phase boundaries. +Prepared data supplies its fitting view and frozen-representation validation; +algorithm-specific transition checks remain with the IQL dataset. + +The only policy-training checkpoint is `sequifier_policy_run`, format version 4. Completed phase results determine the phase boundary; an optional live phase -state records the latest committed transaction. Incoming and selected actors are -stored as direct tensor state dictionaries for self-contained recovery. +state records the latest committed transaction. Only the incoming actor and +current phase's recovery tensors are embedded; earlier selected actors live in +their phase artifacts rather than being serialized with every checkpoint. It contains the resolved plan, immutable representation and its fingerprint, data -and adapter identities, run RNG, completed results, selected actors, and optional -versioned phase state. DPO/GRPO reference weights are embedded. Optimizer, scheduler, -sampler, selection, evaluation and phase counters remain phase-local. -Unreleased version-2 run checkpoints are not supported. +and adapter identities, run RNG, completed results, phase manifest paths and +checksums, and optional versioned phase state. DPO/GRPO reference weights are +embedded in live phase state. Optimizer, scheduler, sampler, selection, evaluation +and phase counters remain phase-local. Only version-4 checkpoints are supported. Writes use a temporary file and atomic replacement at completed updates or -handoffs. No partial group is checkpointable. A committed handoff advances the -phase index exactly once; resume never reruns completed phases or reselects their -actor. Phase artifacts/reports can be republished idempotently from the stored -selection. Representation and data/adapter/reference mismatches fail before -output mutation. +handoffs. No partial group is checkpointable. Handoffs flush the selected +`actor.pt`, then `report.json`, then a small `manifest.json` recording their SHA-256 +checksums, the selected weight hash, and result/representation fingerprints. Only +then is the run checkpoint committed with the manifest identity and incoming +actor. The checkpoint is the authoritative phase boundary: exports left by an +interrupted, uncommitted handoff can be reused if valid or replaced on retry. +A committed handoff advances the phase index exactly once; resume never reruns +completed phases or reselects their actor. + +Completed phase artifacts and manifests must be retained alongside the checkpoint +for recovery; the checkpoint alone is no longer self-contained. Resume verifies +their identities and fails on missing or modified files before output mutation, +without loading or rewriting earlier models. Representation and +data/adapter/reference mismatches also fail before output mutation. `resume.mode: exact` requires validated PPO simulator/RNG snapshots or GRPO's reproducible group-start capability and matching implementation identity. PPO diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 605050bd..0b8718f9 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -2719,22 +2719,41 @@ row-group access. The episode cache stays bounded; partition views retain metada and validated transition indices. Imitation shuffles episodes and then decisions within each episode to preserve storage locality. -The only policy-training checkpoint is `sequifier_policy_run`, format version 3. +Inputs are resolved once into a run-owned catalog. Source resolution returns +verified file checksums, including the checksums checked against sequence manifests; +preparation reuses those resolved sources without resolving or hashing them again. +Every phase boundary explicitly rehashes all input files, manifests, metadata, +static data and preference files to detect modifications. Output and archive path +conflicts are checked separately before output mutation and at phase boundaries. +Prepared data supplies its fitting view and frozen-representation validation; +algorithm-specific transition checks remain with the IQL dataset. + +The only policy-training checkpoint is `sequifier_policy_run`, format version 4. Completed phase results determine the phase boundary; an optional live phase -state records the latest committed transaction. Incoming and selected actors are -stored as direct tensor state dictionaries for self-contained recovery. +state records the latest committed transaction. Only the incoming actor and +current phase's recovery tensors are embedded; earlier selected actors live in +their phase artifacts rather than being serialized with every checkpoint. It contains the resolved plan, immutable representation and its fingerprint, data -and adapter identities, run RNG, completed results, selected actors, and optional -versioned phase state. DPO/GRPO reference weights are embedded. Optimizer, scheduler, -sampler, selection, evaluation and phase counters remain phase-local. -Unreleased version-2 run checkpoints are not supported. +and adapter identities, run RNG, completed results, phase manifest paths and +checksums, and optional versioned phase state. DPO/GRPO reference weights are +embedded in live phase state. Optimizer, scheduler, sampler, selection, evaluation +and phase counters remain phase-local. Only version-4 checkpoints are supported. Writes use a temporary file and atomic replacement at completed updates or -handoffs. No partial group is checkpointable. A committed handoff advances the -phase index exactly once; resume never reruns completed phases or reselects their -actor. Phase artifacts/reports can be republished idempotently from the stored -selection. Representation and data/adapter/reference mismatches fail before -output mutation. +handoffs. No partial group is checkpointable. Handoffs flush the selected +`actor.pt`, then `report.json`, then a small `manifest.json` recording their SHA-256 +checksums, the selected weight hash, and result/representation fingerprints. Only +then is the run checkpoint committed with the manifest identity and incoming +actor. The checkpoint is the authoritative phase boundary: exports left by an +interrupted, uncommitted handoff can be reused if valid or replaced on retry. +A committed handoff advances the phase index exactly once; resume never reruns +completed phases or reselects their actor. + +Completed phase artifacts and manifests must be retained alongside the checkpoint +for recovery; the checkpoint alone is no longer self-contained. Resume verifies +their identities and fails on missing or modified files before output mutation, +without loading or rewriting earlier models. Representation and +data/adapter/reference mismatches also fail before output mutation. `resume.mode: exact` requires validated PPO simulator/RNG snapshots or GRPO's reproducible group-start capability and matching implementation identity. PPO diff --git a/src/sequifier/io/episode_corpus.py b/src/sequifier/io/episode_corpus.py index 1ebfea0b..d02bb600 100644 --- a/src/sequifier/io/episode_corpus.py +++ b/src/sequifier/io/episode_corpus.py @@ -12,6 +12,7 @@ from sequifier.io.sequence import SequenceEpisode from sequifier.io.values import numeric_scalar from sequifier.trajectory.importers import source_episodes +from sequifier.trajectory.sources import ResolvedSequenceSource def _load(source) -> SequenceEpisode: @@ -40,7 +41,7 @@ def __init__(self, sources, config, cache_size=8): for source in sources for episode in ( source_episodes(source, config) - if isinstance(source, SequenceSourceConfig) + if isinstance(source, (SequenceSourceConfig, ResolvedSequenceSource)) else [source] ) ] diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py index b40ad905..d4c176d4 100644 --- a/src/sequifier/io/iql_dataset.py +++ b/src/sequifier/io/iql_dataset.py @@ -82,6 +82,24 @@ def __init__(self, policy_dataset): def use_config(self, config): self.view = PolicyWindowView(config) + def validate_normalization(self, actor): + # Interior successors are ordinary observations, already checked by the + # policy dataset. Only the final successor can contain additional values. + for episode in self.policy_dataset.episodes: + step = len(episode.actions) - 1 + successor = self._successor(episode, step) + if successor is None: + continue + inputs = self.view.successor_inputs(episode, step, successor) + features, _ = actor.model_inputs(inputs.to(actor.device)) + if any( + t.is_floating_point() and not torch.isfinite(t).all() + for t in features.values() + ): + raise ValueError( + f"episode {episode.episode_id}: non-finite normalized IQL successor" + ) + @staticmethod def _successor(episode, step): transitions = episode.transitions diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index a78e4e9c..a338f2dd 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -22,6 +22,16 @@ class PreferenceData: def contract(self): return [self.training.contract, self.validation.contract] + def fitting_dataset(self): + return self.training.fitting_dataset() + + def validate(self, actor): + for partition in (self.training, self.validation): + partition.use_config(actor.config) + fitting = partition.fitting_dataset() + fitting.use_config(actor.config) + fitting.validate_normalization(actor.interface.selected_columns_statistics) + class PreferenceDataset: def __init__(self, pairs, corpora, config): diff --git a/src/sequifier/io/prepared_policy_data.py b/src/sequifier/io/prepared_policy_data.py new file mode 100644 index 00000000..4f0ac9e0 --- /dev/null +++ b/src/sequifier/io/prepared_policy_data.py @@ -0,0 +1,21 @@ +"""The small data contract shared by policy phase preparation.""" + +from collections.abc import Sized +from typing import TYPE_CHECKING, Protocol + +from sequifier.io.policy_dataset import PolicySequenceDataset + +if TYPE_CHECKING: + from sequifier.rl.policy import CategoricalPolicy + + +class PreparedPolicyData(Protocol): + @property + def training(self) -> Sized: ... + + @property + def contract(self) -> list: ... + + def fitting_dataset(self) -> PolicySequenceDataset: ... + + def validate(self, actor: "CategoricalPolicy") -> None: ... diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index c4f098c7..4af98339 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -7,18 +7,18 @@ from sequifier.config.train_config import PolicyRunConfig from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism -from sequifier.io.preference_dataset import PreferenceData from sequifier.rl.checkpoint import load_policy, save_policy -from sequifier.rl.offline_data import OfflineData +from sequifier.rl.phase_artifacts import PhaseArtifacts from sequifier.rl.preparation import ( + RunInputs, build_actor, - input_identity, preflight_environments, preflight_optimization, prepare_data, resolve_normalization, validate_data, validate_declared_policy, + validate_output_paths, ) from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_checkpoint import RunCheckpoint @@ -47,7 +47,9 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: raise TypeError("policy training requires PolicyRunConfig") configure_determinism(config.seed, config.enforce_determinism) plan = _plan_contract(config) - data_identity = input_identity(config) + inputs = RunInputs.resolve(config) + data_identity = inputs.identity + validate_output_paths(config, inputs) checkpoint = Path(config.checkpoint_path) recovery = RunCheckpoint.load(checkpoint, config, plan, data_identity) resuming = recovery is not None @@ -58,13 +60,13 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: ) representation = recovery.representation validate_declared_policy(config.policy, representation.policy, "resume.policy") - data = prepare_data(config, representation.policy) + data = prepare_data(config, representation.policy, inputs) actor = build_actor( config, representation.policy, representation.values["statistics"] ) recovery.validate_actor(actor, config.units) else: - data = prepare_data(config, config.policy) + data = prepare_data(config, config.policy, inputs) if config.initialize_from: source = load_policy( config.initialize_from, @@ -83,13 +85,7 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: actor.network.load_state_dict(source.network.state_dict(), strict=True) else: first = data[0] - fitting = ( - first.training.fitting_dataset() - if isinstance(first, PreferenceData) - else first.training - if isinstance(first, OfflineData) - else None - ) + fitting = first.fitting_dataset() if first is not None else None fitted = ( fitting.fit_categorical_config() if fitting is not None @@ -112,7 +108,7 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: representation.values["statistics"], "run.normalization", ) - validate_data(config, data, actor) + validate_data(data, actor) data_contract = [item.contract if item is not None else None for item in data] if recovery is not None: require_equal(data_contract, recovery.data_contract, "resume.partitions") @@ -190,21 +186,7 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: optimization = preflight_optimization(config, actor, data) restore_rng_state(actor.device, recovery.random if resuming else saved_random) - published = set() - - def publish_completed(): - for result, weights in zip(recovery.results, recovery.selected_actors): - if result.index in published: - continue - portable = deepcopy(actor).eval() - portable.network.load_state_dict(weights, strict=True) - directory = ( - Path(config.phase_directory) / f"{result.index:03d}-{result.name}" - ) - save_policy(portable, directory / "actor.pt", metadata=result.as_dict()) - write_json_atomic(directory / "report.json", result.as_dict()) - published.add(result.index) - + artifacts = PhaseArtifacts(config.phase_directory, representation) if not resuming: # Persist the complete fitted representation before the first update. recovery.save(checkpoint, actor.device) @@ -212,11 +194,11 @@ def publish_completed(): phase_index = recovery.phase_index current_state = recovery.phase_state phase = config.phases[phase_index] - require_equal( + inputs.revalidate( data_identity, - input_identity(config), f"phases.{phase.name}.source_identity", ) + validate_output_paths(config, inputs) incoming_weight_hash = weight_fingerprint(actor.network.state_dict()) phase_actor = deepcopy(actor).eval() reference, reference_identity = None, None @@ -295,10 +277,8 @@ def commit(state): ), }, ) - recovery.finish_phase(actor, result) + recovery.finish_phase(actor, result, artifacts) recovery.save(checkpoint, actor.device) - publish_completed() - publish_completed() report = { "format": "sequifier.policy-run-report", "version": 1, diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index 84061c47..81f433a9 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -40,6 +40,17 @@ def use_config(self, config): for transitions in self.transitions: transitions.use_config(config) + def fitting_dataset(self): + return self.training + + def validate(self, actor): + self.use_config(actor.config) + for part in (self.training, self.validation): + part.validate_normalization(actor.interface.selected_columns_statistics) + if self.transitions is not None: + for transitions in self.transitions: + transitions.validate_normalization(actor) + def prepare_offline(run, phase, config, catalog, cache): # The cache belongs to one run: budgets and optimizers do not affect partitions. diff --git a/src/sequifier/rl/phase_artifacts.py b/src/sequifier/rl/phase_artifacts.py new file mode 100644 index 00000000..0460b84e --- /dev/null +++ b/src/sequifier/rl/phase_artifacts.py @@ -0,0 +1,78 @@ +"""Commit completed phase exports before their run checkpoint handoff.""" + +import json +from pathlib import Path + +from sequifier.fingerprints import fingerprint +from sequifier.rl.checkpoint import save_policy +from sequifier.rl.run_state import ( + require_equal, + sync_file, + weight_fingerprint, + write_json_atomic, +) +from sequifier.trajectory.sources import file_checksum + + +class PhaseArtifacts: + def __init__(self, directory, representation): + self.directory = Path(directory) + self.representation = representation + + def paths(self, result): + directory = self.directory / f"{result.index:03d}-{result.name}" + return ( + directory / "actor.pt", + directory / "report.json", + directory / "manifest.json", + ) + + def _manifest(self, result): + actor, report, _ = self.paths(result) + return { + "format": "sequifier.policy-phase", + "version": 1, + "representation_fingerprint": self.representation.identity, + "result_fingerprint": fingerprint(result.as_dict()), + "selected_weight_hash": result.selected_weight_hash, + "actor": {"path": actor.name, "sha256": file_checksum(actor)}, + "report": {"path": report.name, "sha256": file_checksum(report)}, + } + + def validate(self, result, record): + """Check a committed export without loading earlier models into memory.""" + _, _, manifest = self.paths(result) + require_equal(str(manifest), record["path"], "resume.phase_manifest.path") + require_equal( + record["sha256"], file_checksum(manifest), "resume.phase_manifest.sha256" + ) + require_equal( + self._manifest(result), + json.loads(manifest.read_text(encoding="utf-8")), + "resume.phase_artifacts", + ) + + def commit(self, actor, result): + """Publish actor, report, then manifest; the run checkpoint commits last. + + Files not referenced by a committed run checkpoint are staged exports. + A retry can reuse a complete staged export or replace an incomplete one. + """ + require_equal( + result.selected_weight_hash, + weight_fingerprint(actor.network.state_dict()), + "handoff.selected_actor", + ) + actor_path, report, manifest = self.paths(result) + if manifest.is_file(): + record = {"path": str(manifest), "sha256": file_checksum(manifest)} + try: + self.validate(result, record) + return record + except (OSError, ValueError, KeyError, TypeError): + pass + save_policy(actor, actor_path, metadata=result.as_dict()) + sync_file(actor_path) + write_json_atomic(report, result.as_dict()) + write_json_atomic(manifest, self._manifest(result)) + return {"path": str(manifest), "sha256": file_checksum(manifest)} diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index d690afa2..bf82093a 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -1,13 +1,12 @@ """Read-only policy run preparation and whole-plan preflight validation.""" -import hashlib +from dataclasses import dataclass from pathlib import Path import torch from sequifier.io.policy_dataset import PolicyDataCatalog -from sequifier.io.policy_window import PolicyWindowView -from sequifier.io.preference_dataset import PreferenceData +from sequifier.io.prepared_policy_data import PreparedPolicyData from sequifier.model.factory import build_transformer_network from sequifier.rl.environment import ( adapter_identity, @@ -15,41 +14,57 @@ require_exact_snapshot, require_group_start, ) -from sequifier.rl.offline_data import OfflineData from sequifier.rl.policy import CategoricalPolicy, policy_execution_config from sequifier.rl.ppo_evaluation import require_evaluation_timer from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_state import capture_rng_state, require_equal, restore_rng_state -from sequifier.trajectory.sources import source_paths - - -def file_hash(path): - digest = hashlib.sha256() - with Path(path).open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def input_identity(config): - protected = set() - for source in config.sources.values(): - protected.update(source_paths(source)) - protected.update( - Path(path).resolve() - for path in ( - source.manifest, - source.metadata_config_path, - source.static_data_path, - ) - if path +from sequifier.trajectory.sources import ( + ResolvedSequenceSource, + file_checksum, + resolve_source, +) + + +@dataclass(frozen=True) +class RunInputs: + """Resolved sources and verified file identities owned by one run.""" + + sources: dict[str, ResolvedSequenceSource] + checksums: dict[Path, str] + + @classmethod + def resolve(cls, config): + checksums = {} + sources = { + name: resolve_source(source, checksums=checksums) + for name, source in config.sources.items() + } + for path in config.preferences.values(): + path = Path(path) + if path not in checksums: + checksums[path] = file_checksum(path) + return cls(sources, checksums) + + @property + def identity(self): + return {str(path.resolve()): value for path, value in self.checksums.items()} + + def revalidate(self, expected, field): + """Rehash every dependency, including manifests, at each phase boundary.""" + require_equal( + expected, + {str(path.resolve()): file_checksum(path) for path in self.checksums}, + field, ) - protected.update(Path(path) for path in config.preferences.values()) - data_identity = {str(path): file_hash(path) for path in sorted(protected)} + + +def validate_output_paths(config, inputs): + """Keep every output and archive separate from inputs and other outputs.""" + protected = {path.resolve() for path in inputs.checksums} if config.initialize_from: - protected.add(Path(config.initialize_from)) + protected.add(Path(config.initialize_from).resolve()) protected.update( - Path(phase.reference.path) + Path(phase.reference.path).resolve() for phase in config.phases if getattr(phase, "reference", None) is not None and phase.reference.path ) @@ -60,9 +75,12 @@ def input_identity(config): ] for i, phase in enumerate(config.phases): directory = Path(config.phase_directory) / f"{i:03d}-{phase.name}" - outputs += [directory / "actor.pt", directory / "report.json"] + outputs += [ + directory / name for name in ("actor.pt", "report.json", "manifest.json") + ] + outputs = [path.resolve() for path in outputs] archives = [ - Path(phase.archive_directory) + Path(phase.archive_directory).resolve() for phase in config.phases if getattr(phase, "archive_directory", None) ] @@ -93,7 +111,6 @@ def input_identity(config): raise ValueError( f"archive {directory} conflicts with input/output {other}" ) - return data_identity def resolve_normalization(policy, declaration): @@ -154,8 +171,8 @@ def build_actor(config, fitted, statistics): return actor -def prepare_data(config, policy): - catalog = PolicyDataCatalog(config.sources, policy, config.episode_cache_size) +def prepare_data(config, policy, inputs) -> list[PreparedPolicyData | None]: + catalog = PolicyDataCatalog(inputs.sources, policy, config.episode_cache_size) prepared = {} result = [] for phase in config.phases: @@ -167,43 +184,13 @@ def prepare_data(config, policy): return result -def validate_data(config, data, actor): - policy = actor.config +def validate_data(data, actor): validated = set() for item in data: - if id(item) in validated: + if item is None or id(item) in validated: continue validated.add(id(item)) - if isinstance(item, OfflineData): - item.use_config(policy) - for index, part in enumerate((item.training, item.validation)): - part.validate_normalization(actor.interface.selected_columns_statistics) - if item.transitions is not None: - transitions = item.transitions[index] - for episode in part.episodes: - step = len(episode.actions) - 1 - successor = transitions._successor(episode, step) - if successor is not None: - inputs = PolicyWindowView(policy).successor_inputs( - episode, step, successor - ) - features, _ = actor.model_inputs(inputs.to(actor.device)) - if any( - t.is_floating_point() and not torch.isfinite(t).all() - for t in features.values() - ): - raise ValueError( - f"episode {episode.episode_id}: non-finite normalized IQL successor" - ) - elif isinstance(item, PreferenceData): - item.training.use_config(policy) - item.validation.use_config(policy) - for partition in (item.training, item.validation): - fitting = partition.fitting_dataset() - fitting.use_config(policy) - fitting.validate_normalization( - actor.interface.selected_columns_statistics - ) + item.validate(actor) def preflight_environments(config): diff --git a/src/sequifier/rl/registry.py b/src/sequifier/rl/registry.py index fd185581..1874b999 100644 --- a/src/sequifier/rl/registry.py +++ b/src/sequifier/rl/registry.py @@ -4,6 +4,7 @@ from typing import Callable from sequifier.io.preference_dataset import prepare_preferences +from sequifier.io.prepared_policy_data import PreparedPolicyData from sequifier.rl.dpo import run_dpo_phase from sequifier.rl.grpo import run_grpo_phase from sequifier.rl.iql import run_iql_phase @@ -15,7 +16,7 @@ @dataclass(frozen=True) class BuiltinAlgorithm: runtime: Callable - prepare_data: Callable | None = None + prepare_data: Callable[..., PreparedPolicyData] | None = None state_version: int = 1 diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 152527c6..1ed76907 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -1,4 +1,4 @@ -"""Self-contained policy recovery with one authoritative committed phase boundary.""" +"""Policy recovery with artifact-backed, authoritative committed phase boundaries.""" import math from dataclasses import dataclass, field @@ -6,6 +6,7 @@ import torch +from sequifier.rl.phase_artifacts import PhaseArtifacts from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_state import ( ActorRepresentation, @@ -30,7 +31,7 @@ class RunCheckpoint: random: dict reference_contract: dict = field(default_factory=dict) results: list[PolicyPhaseResult] = field(default_factory=list) - selected_actors: list[dict] = field(default_factory=list) + completed_artifacts: list[dict] = field(default_factory=list) phase_state: dict | None = None @property @@ -58,9 +59,9 @@ def load(cls, path, config, plan, data_identity): if ( not isinstance(payload, dict) or payload.get("artifact_type") != "sequifier_policy_run" - or payload.get("format_version") != 3 + or payload.get("format_version") != 4 ): - raise ValueError("policy resume requires a version-3 policy run checkpoint") + raise ValueError("policy resume requires a version-4 policy run checkpoint") try: require_equal(plan, payload["plan"], "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") @@ -79,7 +80,7 @@ def load(cls, path, config, plan, data_identity): payload["random"], payload["reference_contract"], results, - payload["selected_actors"], + payload["completed_artifacts"], payload["phase_state"], ) require_equal( @@ -93,28 +94,22 @@ def load(cls, path, config, plan, data_identity): raise ValueError(f"malformed policy run checkpoint: {exc}") from exc def validate(self, config): - if len(self.results) > len(config.phases) or len(self.selected_actors) != len( - self.results - ): + if len(self.results) > len(config.phases) or len( + self.completed_artifacts + ) != len(self.results): raise ValueError("checkpoint completed handoffs disagree with the plan") validate_finite_state(self.incoming_actor, "incoming_actor") - validate_finite_state(self.selected_actors, "selected_actors") # Parsing the owned representation also validates the persisted policy schema. self.representation.policy - for index, (result, weights) in enumerate( - zip(self.results, self.selected_actors) - ): + artifacts = PhaseArtifacts(config.phase_directory, self.representation) + for index, result in enumerate(self.results): phase = config.phases[index] require_equal( (index, phase.name, phase.algorithm, phase.handoff), (result.index, result.name, result.algorithm, result.handoff), "resume.completed_phase", ) - require_equal( - result.selected_weight_hash, - weight_fingerprint(weights), - "resume.selected_actor.weights", - ) + artifacts.validate(result, self.completed_artifacts[index]) validate_finite_state(result.as_dict(), "completed_phase") incoming_identity = weight_fingerprint(self.incoming_actor) require_equal( @@ -173,7 +168,7 @@ def validate(self, config): def validate_actor(self, actor, units): self.representation.validate_actor(actor, units, "resume.representation") expected = actor.network.state_dict() - weights = [self.incoming_actor, *self.selected_actors] + weights = [self.incoming_actor] if self.phase_state is not None: state = self.phase_state weights += [ @@ -201,7 +196,7 @@ def save(self, path, device, state=None): path, { "artifact_type": "sequifier_policy_run", - "format_version": 3, + "format_version": 4, "plan": self.plan, "representation": self.representation.canonical, "representation_fingerprint": self.representation.identity, @@ -212,22 +207,18 @@ def save(self, path, device, state=None): "initialization": self.initialization, "random": self.random, "incoming_actor": self.incoming_actor, - "selected_actors": self.selected_actors, + "completed_artifacts": self.completed_artifacts, "completed_results": [result.as_dict() for result in self.results], "phase_state": self.phase_state, }, ) - def finish_phase(self, actor, result): + def finish_phase(self, actor, result, artifacts): require_equal(self.phase_index, result.index, "handoff.phase_index") + record = artifacts.commit(actor, result) weights = clone_weights(actor.network) - require_equal( - result.selected_weight_hash, - weight_fingerprint(weights), - "handoff.selected_actor", - ) self.results.append(result) - self.selected_actors.append(weights) + self.completed_artifacts.append(record) self.incoming_actor = weights self.phase_state = None diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 99945958..c28a2025 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -3,6 +3,7 @@ import hashlib import json import math +import os import uuid from copy import deepcopy from dataclasses import dataclass @@ -48,6 +49,7 @@ def atomic_save(path, payload): try: torch.save(payload, temporary) temporary.replace(path) + sync_file(path) finally: temporary.unlink(missing_ok=True) @@ -61,10 +63,24 @@ def write_json_atomic(path, payload): json.dumps(payload, indent=2, allow_nan=False) + "\n", encoding="utf-8" ) temporary.replace(path) + sync_file(path) finally: temporary.unlink(missing_ok=True) +def sync_file(path): + """Flush an atomically published file before committing a reference to it.""" + path = Path(path) + with path.open("rb") as handle: + os.fsync(handle.fileno()) + if os.name != "nt": + descriptor = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + def capture_rng_state(device): return { "local": RandomStateManager(device).capture_local().state_dict(), diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index c49f1f60..1f49ed46 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -7,7 +7,7 @@ from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.io.sequence import SequenceEpisode, TransitionColumns from sequifier.trajectory.schema import Trajectory, validate_trajectory -from sequifier.trajectory.sources import resolve_source +from sequifier.trajectory.sources import ResolvedSequenceSource, resolve_source from sequifier.trajectory.store import load_trajectory from sequifier.trajectory.tables import table_episodes @@ -66,14 +66,23 @@ def load(self) -> SequenceEpisode: ) -def source_episodes(source: SequenceSourceConfig, policy: PolicySchemaConfig): - resolved = resolve_source(source) - source, paths = resolved.source, resolved.paths +def source_episodes( + source: SequenceSourceConfig | ResolvedSequenceSource, policy: PolicySchemaConfig +): + resolved: ResolvedSequenceSource = ( + resolve_source(source) if isinstance(source, SequenceSourceConfig) else source + ) + source_config = resolved.source if resolved.kind == "trajectory": return [ TrajectoryReference( - path, policy, source.action_semantics, source.episode_namespace + path, + policy, + source_config.action_semantics, + source_config.episode_namespace, ) - for path in paths + for path in resolved.paths ] - return table_episodes(source, policy, paths, prepared=resolved.kind == "prepared") + return table_episodes( + source_config, policy, resolved.paths, prepared=resolved.kind == "prepared" + ) diff --git a/src/sequifier/trajectory/sources.py b/src/sequifier/trajectory/sources.py index 8771320e..997c3fd3 100644 --- a/src/sequifier/trajectory/sources.py +++ b/src/sequifier/trajectory/sources.py @@ -21,13 +21,27 @@ def file_checksum(path): return digest.hexdigest() -def _source_files(source): +def _checksum(path, checksums): + path = Path(path) + if path not in checksums: + checksums[path] = file_checksum(path) + return checksums[path] + + +def _source_files(source, checksums): manifest = None if source.manifest is None: paths = [Path(path) for path in source.paths] else: manifest_path = Path(source.manifest) - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + content = manifest_path.read_bytes() + checksum = hashlib.sha256(content).hexdigest() + if manifest_path in checksums and checksums[manifest_path] != checksum: + raise ValueError( + f"source manifest changed during resolution: {manifest_path}" + ) + checksums[manifest_path] = checksum + manifest = json.loads(content.decode("utf-8")) if not isinstance(manifest, dict) or manifest.get("version") != 1: raise ValueError("source manifest must have version 1") sequence_manifest = manifest.get("format") == "sequifier.sequence-data" @@ -43,7 +57,7 @@ def _source_files(source): ): raise ValueError("each manifest entry requires a nonempty path") path = manifest_path.parent / entry["path"] - if sequence_manifest and file_checksum(path) != entry.get("sha256"): + if sequence_manifest and _checksum(path, checksums) != entry.get("sha256"): raise ValueError(f"sequence shard checksum disagrees with {path}") if ( "episode_id" in entry @@ -51,14 +65,13 @@ def _source_files(source): ): raise ValueError(f"manifest episode ID disagrees with {path}") paths.append(path) - paths = [path.resolve() for path in paths] - if not paths or len(set(paths)) != len(paths): + if not paths or len({path.resolve() for path in paths}) != len(paths): raise ValueError("sources require nonempty, unique paths") return paths, manifest def source_paths(source: SequenceSourceConfig) -> list[Path]: - return _source_files(source)[0] + return [path.resolve() for path in _source_files(source, {})[0]] @dataclass(frozen=True) @@ -66,12 +79,29 @@ class ResolvedSequenceSource: source: SequenceSourceConfig paths: tuple[Path, ...] kind: str + checksums: dict[Path, str] -def resolve_source(source: SequenceSourceConfig) -> ResolvedSequenceSource: +def resolve_source( + source: SequenceSourceConfig, *, checksums: dict[Path, str] | None = None +) -> ResolvedSequenceSource: + checksums = {} if checksums is None else checksums if source.split_index is not None: source = resolve_sequence_source(source, Path.cwd()) - paths, manifest = _source_files(source) + # Retain manifest-relative locations so revalidation also detects retargeted links. + paths, manifest = _source_files(source, checksums) + dependencies = set(paths) + dependencies.update( + Path(path) + for path in ( + source.manifest, + source.metadata_config_path, + source.static_data_path, + ) + if path + ) + verified = {path: _checksum(path, checksums) for path in dependencies} + resolved_paths = tuple(path.resolve() for path in paths) if manifest is not None and manifest.get("format") == "sequifier.sequence-data": recorded_columns = PolicyTableColumns.model_validate(manifest["columns"]) if ( @@ -90,7 +120,7 @@ def resolve_source(source: SequenceSourceConfig) -> ResolvedSequenceSource: ) kind = source.format if kind == "auto": - suffixes = {path.suffix.lower() for path in paths} + suffixes = {path.suffix.lower() for path in resolved_paths} if suffixes <= {".json"}: kind = "trajectory" elif suffixes <= {".csv", ".parquet"}: @@ -101,4 +131,4 @@ def resolve_source(source: SequenceSourceConfig) -> ResolvedSequenceSource: raise ValueError( "select a source format explicitly for mixed or unsupported files" ) - return ResolvedSequenceSource(source, tuple(paths), kind) + return ResolvedSequenceSource(source, resolved_paths, kind, verified) From afd1839efb130709b6fd9bef53a7fded70f0d779 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 9 Nov 2026 19:22:30 +0100 Subject: [PATCH 21/71] cleanup --- documentation/configs/infer.md | 5 +- documentation/configs/train.md | 6 +- .../configs/rollout-ppo-toy.yaml | 8 +- .../example_inputs/configs/train-bc-toy.yaml | 1 - .../configs/train-iql-from-table.yaml | 3 - .../configs/train-multiphase-control.yaml | 2 - .../configs/train-policy-from-table.yaml | 2 +- .../configs/train-return-conditioned.yaml | 3 +- .../example_inputs/ppo_toy_control.py | 4 +- ...dpo-grpo-multiphase-implementation-plan.md | 390 ------------- .../plans/offline-iql-implementation-plan.md | 222 -------- ...nforcement-learning-implementation-plan.md | 528 ------------------ .../plans/scoped-ppo-implementation-plan.md | 271 --------- documentation/policy-learning.md | 109 ++-- documentation/rollout.md | 2 +- pyproject.toml | 2 +- release-notes/unreleased-policy-training.md | 18 +- .../artifacts/backbone_repository.py | 30 +- src/sequifier/artifacts/manifests.py | 19 +- src/sequifier/artifacts/model_artifact.py | 13 +- src/sequifier/artifacts/run_checkpoint.py | 26 +- src/sequifier/config/infer_config.py | 2 - src/sequifier/config/policy_infer_config.py | 13 +- src/sequifier/config/policy_optimization.py | 21 + src/sequifier/config/policy_plan.py | 117 +--- src/sequifier/config/rollout_config.py | 2 +- src/sequifier/config/train_config.py | 15 - src/sequifier/consolidated-docs.md | 120 ++-- src/sequifier/infer.py | 15 +- src/sequifier/io/atomic.py | 41 ++ src/sequifier/io/iql_dataset.py | 20 - src/sequifier/io/policy_dataset.py | 14 +- src/sequifier/make.py | 293 +--------- src/sequifier/rl/checkpoint.py | 4 + src/sequifier/rl/control.py | 43 +- src/sequifier/rl/dpo.py | 63 +-- src/sequifier/rl/environment.py | 123 +--- src/sequifier/rl/grpo.py | 129 ++--- src/sequifier/rl/grpo_update.py | 19 +- src/sequifier/rl/inference.py | 4 +- src/sequifier/rl/iql.py | 115 +--- src/sequifier/rl/learner.py | 259 ++++----- src/sequifier/rl/phase_artifacts.py | 78 --- src/sequifier/rl/ppo.py | 137 ++--- src/sequifier/rl/ppo_buffer.py | 2 +- src/sequifier/rl/ppo_evaluation.py | 2 +- src/sequifier/rl/ppo_update.py | 49 +- src/sequifier/rl/preparation.py | 10 +- src/sequifier/rl/registry.py | 2 +- src/sequifier/rl/run_checkpoint.py | 65 ++- src/sequifier/rl/run_state.py | 233 ++++---- src/sequifier/rl/runtime.py | 319 +++++------ src/sequifier/rl/training.py | 63 +-- src/sequifier/rl_api.py | 13 +- src/sequifier/rollout/contracts.py | 13 +- src/sequifier/rollout/coordinator.py | 58 +- src/sequifier/rollout_api.py | 2 - src/sequifier/runtime/random_state.py | 34 +- src/sequifier/templates/infer-policy.yaml | 10 + .../sequifier/templates/train-dpo.yaml | 0 .../sequifier/templates/train-grpo.yaml | 0 src/sequifier/templates/train-iql.yaml | 45 ++ src/sequifier/templates/train-policy.yaml | 42 ++ .../sequifier/templates/train-ppo.yaml | 2 - src/sequifier/trajectory/schema.py | 48 +- src/sequifier/trajectory/store.py | 24 +- 66 files changed, 1136 insertions(+), 3211 deletions(-) delete mode 100644 documentation/plans/dpo-grpo-multiphase-implementation-plan.md delete mode 100644 documentation/plans/offline-iql-implementation-plan.md delete mode 100644 documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md delete mode 100644 documentation/plans/scoped-ppo-implementation-plan.md create mode 100644 src/sequifier/io/atomic.py delete mode 100644 src/sequifier/rl/phase_artifacts.py create mode 100644 src/sequifier/templates/infer-policy.yaml rename documentation/example_inputs/configs/train-dpo-control.yaml => src/sequifier/templates/train-dpo.yaml (100%) rename documentation/example_inputs/configs/train-grpo-control.yaml => src/sequifier/templates/train-grpo.yaml (100%) create mode 100644 src/sequifier/templates/train-iql.yaml create mode 100644 src/sequifier/templates/train-policy.yaml rename documentation/example_inputs/configs/train-ppo-toy.yaml => src/sequifier/templates/train-ppo.yaml (96%) diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index 2847d48c..ddb52701 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -271,6 +271,9 @@ ORT installation. Outputs stay indexed by outer item coordinates. Use the selected actor export rather than the policy run checkpoint. Use raw CSV/Parquet observation rows and optional recorded action histories; alignment, vocabulary, and normalization come from the artifact. `model_type: generative` and `model_type: policy` are accepted. Environment -execution uses the existing `rollout` section with `mode: policy`. See the +execution uses a policy `rollout` section with `environment` (a version-3 adapter +supporting `seeded_reset`), `trajectory_path`, `seed_index`, and optional +`max_episode_steps`. Set `autoregressive: true` and `generation_steps`; policy +rollout does not configure forecast bindings. See the [policy guide](../policy-learning.md#python-and-inference) and [inference example](../example_inputs/configs/infer-policy.yaml). diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 9b5592f7..adba68db 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -790,7 +790,7 @@ selection monitors and exact/non-exact continuation. Examples: - [BC](../example_inputs/configs/train-bc-toy.yaml) - [Return-conditioned imitation](../example_inputs/configs/train-return-conditioned.yaml) - [IQL](../example_inputs/configs/train-iql-from-table.yaml) -- [PPO](../example_inputs/configs/train-ppo-toy.yaml) -- [DPO](../example_inputs/configs/train-dpo-control.yaml) -- [GRPO](../example_inputs/configs/train-grpo-control.yaml) +- [PPO](../../src/sequifier/templates/train-ppo.yaml) +- [DPO](../../src/sequifier/templates/train-dpo.yaml) +- [GRPO](../../src/sequifier/templates/train-grpo.yaml) - [Imitation → IQL → DPO → GRPO → PPO](../example_inputs/configs/train-multiphase-control.yaml) diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index 13fcfb92..301efdde 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -9,12 +9,10 @@ deterministic: true autoregressive: true generation_steps: 8 rollout: - mode: policy - adapter: - factory: sequifier.rl.control:ControlRolloutAdapter + environment: + factory: sequifier.rl.control:CategoricalControlAdapter config: {horizon: 8} - bindings: - - {input: cue, source: environment, key: cue} + implementation_id: sequifier.categorical-control.v1 trajectory_path: outputs/ppo-rollout.json seed_index: 0 max_episode_steps: 8 diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml index f154972c..6fb71dd1 100644 --- a/documentation/example_inputs/configs/train-bc-toy.yaml +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -37,4 +37,3 @@ phases: sources: [demonstrations.recorded] epochs: 30 batch_size: 64 - actor_learning_rate: 0.0003 diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index 57074ab2..52a936a7 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -52,8 +52,5 @@ phases: inverse_temperature: 3.0 max_weight: 100.0 target_update_rate: 0.005 - actor_learning_rate: 0.0003 - value_learning_rate: 0.0003 - critic_learning_rate: 0.0003 evaluation_interval: 100 checkpoint_interval: 100 diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index ad46a13a..021911bb 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -71,8 +71,6 @@ phases: gamma: 0.99 gae_lambda: 0.95 clip_epsilon: 0.2 - actor_learning_rate: 0.0003 - value_learning_rate: 0.0003 evaluation_interval: 2 checkpoint_interval: 2 evaluation: diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml index 636a537c..1bd6d7e1 100644 --- a/documentation/example_inputs/configs/train-policy-from-table.yaml +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -37,4 +37,4 @@ phases: epochs: 80 sources: [demonstrations.recorded] batch_size: 64 - actor_learning_rate: 0.005 + optimization: {actor: {learning_rate: 0.005}} diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index b40f0015..fcad048b 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -43,5 +43,4 @@ phases: epochs: 80 sources: [demonstrations.recorded] batch_size: 64 - gradient_clip: 1.0 - actor_learning_rate: 0.005 + optimization: {actor: {learning_rate: 0.005}} diff --git a/documentation/example_inputs/ppo_toy_control.py b/documentation/example_inputs/ppo_toy_control.py index 25128ca0..9f918257 100644 --- a/documentation/example_inputs/ppo_toy_control.py +++ b/documentation/example_inputs/ppo_toy_control.py @@ -1,5 +1,7 @@ """Train the categorical control example with the canonical Python policy API.""" +from importlib.resources import files + from sequifier.config.train_config import load_train_config from sequifier.rl_api import ( PolicyRunConfig, @@ -11,7 +13,7 @@ def main(): config = load_train_config( - "documentation/example_inputs/configs/train-ppo-toy.yaml", {}, False + str(files("sequifier").joinpath("templates", "train-ppo.yaml")), {}, False ) if not isinstance(config, PolicyRunConfig): raise TypeError("the example requires a policy run config") diff --git a/documentation/plans/dpo-grpo-multiphase-implementation-plan.md b/documentation/plans/dpo-grpo-multiphase-implementation-plan.md deleted file mode 100644 index ee0ae73f..00000000 --- a/documentation/plans/dpo-grpo-multiphase-implementation-plan.md +++ /dev/null @@ -1,390 +0,0 @@ -# DPO, GRPO, and multi-phase policy training implementation plan - -Status: proposed, 2026-11-09. This plan builds on the policy-learning changes -since `2d01d407cd2134c8197b1a18f0034d01486f18a9`, the -[IQL plan](offline-iql-implementation-plan.md), and the -[PPO plan](scoped-ppo-implementation-plan.md). It is an implementation plan, -not a claim that these features exist. All policy-learning changes since that -commit are unreleased and may be redesigned or removed. There is no requirement -to migrate their configs, APIs, artifacts, checkpoints, or adapter protocols. -The verification work described below is for a subsequent implementation; no -tests are to be written or run as part of producing this plan. - -## 1. Outcome and scope - -Support ordered, same-run combinations of `policy_imitation`, `iql`, `ppo`, -`dpo`, and `grpo` phases through the training CLI and Python API. Each phase -has an explicit budget, data/environment contract, evaluation, and selection -rule. A phase produces a selected actor and hands it to the next phase; the -final selected actor is a portable Sequifier model usable for policy inference -and rollout through the redesigned interfaces. - -The first release targets this policy boundary: one fixed -categorical action, scalar/categorical observation and static fields, causal -fixed-length history, local single-process execution, and one selected policy -interface. DPO compares alternatives from the same initial history; GRPO samples -multiple trajectories from the same initial condition. This is a control and -sequence-policy implementation, not a claim of LLM-tokenizer, text-prompt, -continuous-action, vectorized, distributed, or arbitrary reward-model support. -The design includes imitation, return-conditioned imitation, IQL, and PPO. -Return-conditioned policies are available for imitation; IQL, PPO, DPO, and GRPO initially require an -unconditioned actor, with an explicit preflight error for incompatible plans. - -A multi-phase plan is sequential, not a mixture of concurrently trained -algorithms. Supervised forecasting phases are not part of this policy plan: -reject mixing them with policy phases rather than implying an unspecified -conversion. Cross-run composition uses the same portable actor artifact contract. - -## 2. Product and execution contracts - -### 2.1 Authoring and dispatch - -Add a canonical `PolicyRunConfig` with run-wide model, policy interface, -context length, device, seed, output paths, source catalog, and a discriminated -list of phase configurations. Offline phases reference source names; online -phases declare an environment, evaluation, and frozen normalization. Specify -budgets in their natural units: epochs, offline updates, PPO environment steps, -or GRPO groups. Validate unique phase names and all references before training. -Do not overload supervised `ResolvedTrainingPhase`, which assumes epoch-based -execution and supports only supervised/imitation algorithms. - -Use `PolicyRunConfig` as the sole policy-training configuration, with a single -result type containing ordered phase results and the final actor. Route both CLI -and Python entry points through the same validation and coordinator. Delete or -rewrite the unreleased `PolicyTrainConfig`, `PPOTrainConfig`, their separate -dispatch paths, and redundant public entry points as needed. Design one coherent -CLI and `rl_api` around the new contracts; no old policy syntax, default, result -shape, output path, or import needs an adapter. Keep the pre-existing supervised -training path independent, and reject mixed supervised/policy plans. - -Replace duplicated closed-set branching with a **built-in explicit registry** -that maps algorithm identifiers to phase config type and runtime factory. -Pydantic's static discriminated union still explicitly lists built-in phase -classes; the registry is not a promise of third-party dynamic config plugins. -One registration inventory drives dispatch and a consistency check verifies -that it agrees with the union. Algorithm-specific configuration and validation -remain in their own phase models; avoid a generic dictionary of arbitrary -options or an algorithm-independent `update()` abstraction that conceals -required inputs. Update typing, `rl_api`, docs and CLI examples together. - -### 2.2 Frozen actor representation and phase handoff - -Resolve one immutable actor representation before the first phase: architecture, -interface, context length, feature ordering/types, categorical vocabularies, -action IDs and ordering, normalization and units, and return conditioning. -The first offline phase may fit codecs and statistics on its training partition -only; persist the resulting complete representation before any update. An -online-first run must declare a frozen representation or initialize from a -compatible artifact. A later phase must reuse the frozen representation: never -refit from its own data or silently substitute identity statistics. Preflight -all explicitly declared phase representations and artifact contracts against it; -validate data-dependent details when the first phase has fitted them and before -starting later phases. Reject incompatible architecture, codec, normalization, -units, context, or conditioning with an actionable field-level diagnostic. - -Each phase receives a deep-copied **selected** actor from the previous phase, -not a mutable handle to the previous trainer. The first phase starts from the -run's `initialize_from` actor if supplied, otherwise from a seeded fresh actor. -Each phase declares `handoff: best | last` (default `best`), its supported -monitor and direction, and a deterministic tie rule: retain the earliest best. -`best` requires a finite metric and an evaluation schedule capable of producing -one; fail rather than silently use `last`. `last` means the most recently -completed optimizer transaction. Save both selected provenance and metric. -The final export is the final phase's selected actor, not the globally best -actor across incomparable metrics. - -Optimizers, schedulers, samplers, critics and mutable environment state are -phase-local and start fresh at a phase boundary. Do not transfer optimizer state -or reuse old on-policy samples between algorithms. Warm-starting critics or -optimizer state is outside this release. Use explicit phase indices and names in -paths and reports to prevent collisions. Protect all inputs, including -preference manifests and reference artifacts, from output overwrites. - -### 2.3 Reference policies - -DPO and, in this plan, GRPO require a frozen reference actor. Each declares -`reference: {source: phase_start}` or `{source: artifact, path: ...}`; -`phase_start` is the default. A reference is a deep copy of the incoming actor -*before* any update, in evaluation mode, with gradients disabled. For a fresh -first phase it is the initialized actor. Validate the full representation and -architecture against the trainable actor and fingerprint the actual weights, -not merely the path. Never silently retarget a reference at the end of an -epoch or phase. Reference weights or an immutable content-addressed artifact -and its verified hash must survive resume even if the original file changes or -disappears; use embedded checkpoint weights in the first implementation. - -DPO's reference enters the chosen/rejected log-ratio. GRPO's reference is used -only for its explicitly configured KL penalty; it is distinct from the frozen -**old collection policy** whose scores define the importance ratio. GRPO still -records a reference identity when its KL coefficient is zero so its run -contract is unambiguous. - -## 3. DPO data and objective - -### 3.1 Preference contract - -Add a versioned, inspectable preference-pair manifest and an offline dataset -view in `trajectory/` and `io/`. A pair has a unique ID, a declared common -initial history/context identity, and chosen and rejected alternatives, each -referencing an episode or a contiguous decision span in a supported -sequence source. Define span bounds as zero-based start-inclusive/end-exclusive; -require at least one scored action per side. Alternative histories must have -identical canonical initial observations, static inputs, valid masks and prior -action history at the branch point, under the frozen codec. After branching, -each alternative uses its own actual observations and prior actions. The pair -record need not invent rewards, successor states, or terminal flags for DPO; -design sequence sources to support reward-free preference data as well as -reward-bearing trajectories. -Never fabricate a reward-bearing `Trajectory` merely to satisfy its validator. -Reject missing actions, unsupported labels, ambiguous alignment, non-finite -features, duplicate pair IDs, overlapping chosen/rejected identities that make a -preference meaningless, and incompatible initial contexts. Permit unequal -alternative lengths; record exact decision masks and length diagnostics. - -Split before fitting representation. Form connected components across shared -context/group IDs, episode IDs, and duplicate content, then put each component -entirely in train or validation. Explicit validation sources must satisfy the -same no-overlap checks. If too few independent components remain, fail with a -clear error. Avoid materializing all windows: reuse `PolicyWindowView` and -bounded episode caches; persist source identities, span definitions, content -fingerprints, split membership and codec identity for exact resume. Comparison -semantics are supplied by the data: do not infer preference from total reward. - -### 3.2 Sequence likelihood and optimization - -Add a differentiable sequence-scoring helper over `CategoricalPolicy`: score -the recorded executable action at each selected decision using the causal -window valid at that decision, then sum FP32 log probabilities per alternative. -Padding and unselected context positions contribute zero. Ensure start/special -tokens are never executable labels. Chunk windows to bound memory while -preserving gradients for trainable scores. Reference scoring uses no gradients, -identical input transformations, and evaluation mode. Stochastic network layers -remain disabled during both reference and trainable scoring for reproducible -likelihood ratios; gradients are still enabled on the trainable actor. - -For each pair, with `s_theta(y)` the summed action log likelihood and `s_ref` -the frozen reference score, implement canonical sigmoid DPO: - -```text -margin = (s_theta(chosen) - s_theta(rejected)) - - (s_ref(chosen) - s_ref(rejected)) -loss = -logsigmoid(beta * margin) -``` - -Require finite `beta > 0`; average losses over pairs. Do not silently switch to -per-token averaging, add a supervised objective, or treat a trajectory reward -as a preference. A pair contributes equally even when alternatives have unequal -lengths; report lengths, margins and preference accuracy to expose that choice. -Support a phase-local optimizer/scheduler, gradient accumulation weighted by -actual pair count, clipping, deterministic sampler, checkpoint interval, and -held-out DPO loss as the default minimizing selection metric. Define evaluation -frequency and save/checkpoint boundaries in optimizer updates. Reference scores -may be cached only against content, span, codec and reference weight hashes; -a cache miss or mismatch must recompute, never silently reuse stale scores. - -## 4. GRPO collection and objective - -### 4.1 Reproducible group starts - -Define an online environment contract with an explicit group-start capability. It must produce an opaque, serializable start token and initialize -independent episodes from that token, yielding the same initial observation, -static inputs and units. Specify whether the token captures simulator randomness: -independent rollout action/environment RNG streams must be derived from run -seed, group index and sample index, while the initial condition remains the -same. An adapter may implement this by restoring a captured initial snapshot or -by a deterministic scenario/reset factory, but must explicitly declare the -capability. A repeated seed alone is insufficient. Validate initial histories -and units across a group and reject unsupported adapters before collection. -Use one coherent adapter protocol for online algorithms, with group-start as an -explicit capability required by GRPO. Redesign the current unreleased adapter -protocol and bridge as needed; do not infer capabilities from method names alone. - -One group contains `group_size >= 2` independent trajectories from the same -start token, generated sequentially in the first implementation under one -frozen old actor version. Use distinct action RNG streams. Each episode ends at -a real terminal or truncation, or at an explicit per-sample `max_steps` cutoff; -the scalar score is the finite sum of observed rewards up to that boundary, -with a declared fixed reward scale. A cutoff is an explicitly accepted -finite-horizon score, not an invented terminal or a bootstrap. If an adapter -fails, a score is non-finite, or a requested complete group cannot fit within -the configured memory/step budget, abort the whole uncommitted group and never -update from a partial group. Close adapters in `finally`. `total_groups` is the -hard authored budget; optional `max_env_steps` must be checked before starting -a group against the conservative `group_size * max_steps` bound. Report actual -executed actions as well as groups, including discarded actions if a failure is -reported. No automatic retry that silently changes data. - -Store group/start identity, episode and step IDs, action histories/masks, -rewards, cutoff reason, fixed old per-action log probabilities, reference -identity, score, and actor generation in a bounded sealed group buffer. Retain -only enough causal frames to reconstruct each action history. Verify old scores -against the current actor before the first update; enforce one generation and -consume each sealed buffer only once. Keep archival optional and distinct from -training input, as with PPO. Groups are collected under `no_grad` and no actor -updates happen until all groups for an iteration have been sealed. - -### 4.2 Chosen GRPO variant - -Specify this exact first variant rather than treating all GRPO variants as -interchangeable. Collect `groups_per_update` complete groups. For group scores -`R_i`, compute FP32 population mean and standard deviation, and assign the same -advantage to every valid decision of trajectory `i`: - -```text -A_i = (R_i - mean(R)) / (std_population(R) + advantage_epsilon) -r_it = exp(log pi_theta(a_it|h_it) - log pi_old(a_it|h_it)) -K_it = exp(log pi_ref(a_it|h_it) - log pi_theta(a_it|h_it)) - - (log pi_ref(a_it|h_it) - log pi_theta(a_it|h_it)) - 1 -loss = mean_over_trajectories(mean_over_valid_decisions( - -min(r_it * A_i, clamp(r_it, 1-epsilon, 1+epsilon) * A_i) - + kl_coefficient * K_it)) -``` - -This sampled reverse-KL estimator is nonnegative per decision, but its -expectation is a KL only under the corresponding sampling distribution; after -multiple optimization epochs the samples are from `pi_old`, so report it as a -sampled reference penalty, not an exact current-policy KL. Scores from the old -actor and fixed reference remain detached for every optimization epoch. -`advantage_epsilon > 0` is explicit; identical group scores yield zero -advantages, though a nonzero reference penalty can still update the actor. This -variant weights each trajectory equally irrespective of length and each group -equally because group sizes are fixed. Minibatches contain whole trajectories; -gradient accumulation weights by actual trajectory count, including the final -partial minibatch. Compute ratio, penalty and loss in FP32 with explicit finite -checks and fail before applying an invalid update. `kl_coefficient >= 0` and -`0 < epsilon < 1` are phase fields. Do not reuse PPO's old-policy approximate -KL as the reference penalty; an optional old-policy KL early-stop is separately -named and documented. GRPO has no value model, GAE, or value optimizer. - -Evaluation uses a separate adapter and independent deterministic evaluation -seeds/scenarios, never the training group RNG. Define score mean as the default -maximizing selection metric, evaluation interval in completed updates, and an -explicit final evaluation so `handoff: best` is meaningful. Report score -spread, zero-variance groups, sample lengths/cutoffs, KL penalty, clip fraction, -policy version, optimizer steps and real environment steps. - -## 5. Runtime, checkpoints, and artifacts - -Introduce a policy run coordinator responsible for validating the plan, -preparing the immutable actor representation, invoking a phase runtime, -committing phase results, and advancing to the next phase. Phase runtimes own -their natural lifecycle and algorithm-specific state. A narrow runtime -contract should accept a phase context and return a selected actor, typed phase -result, provenance and resumable state; it need not impose identical update -loops on imitation, IQL, PPO, DPO and GRPO. Factor shared actor loading and -compatibility, reference freezing, optimizer/scheduler construction, RNG -capture, safe atomic writes and report conventions only where their semantics -actually match. Keep PPO's collection-policy version separate from generic -phase/run IDs. Refactor imitation, IQL and PPO directly into the target runtime; simplify or -replace their current interfaces and implementation where this improves the -architecture. Specify their objectives, selection rules and defaults as part of -the new contracts rather than preserving incidental current behavior. - -Create a versioned multi-phase run checkpoint envelope containing the resolved -plan and immutable fingerprint, actor representation, run RNG state, current -phase index and status, completed phase results and selected actors, current -phase's algorithm identifier/version and opaque typed state, phase-local RNGs, -and the last committed transaction. Each algorithm owns validation of its -state: DPO includes actor/reference, optimizer/scheduler, sampler, dataset/split -identity, clocks and best selection; GRPO includes actor/reference, optimizer, -scheduler, group/seed counters, evaluation state and clocks. Write to a -temporary file and atomically replace only after a completed optimizer update -or phase handoff. No partial group or partially applied update is a checkpoint -boundary. On resume, check plan, data, model, reference hashes and capabilities -before mutating outputs. Resume at the current phase without rerunning completed -phases or reselecting their actor; a committed handoff must be idempotent. - -For GRPO, checkpoints are taken only after updates and with no live group. -Subsequent groups are derived from persisted group counters/start tokens and RNG -streams. Exact continuation requires the adapter's documented group-start -reproducibility and compatible implementation identity; validate both. If the -adapter cannot guarantee it, reject exact resume rather than claim equivalence. -An explicitly selected non-exact restart may begin new groups with new derived -seeds, report the discontinuity and preserve completed updates, but may never -reuse a partial group. PPO must likewise declare whether resuming restores -the environment exactly or resets it; exact continuation requires a validated -adapter snapshot, and a reset must be reported as a discontinuity. - -Use the new checkpoint envelope as the only policy-training checkpoint format. -Remove the unreleased algorithm-specific readers and migration paths where they -are no longer needed; do not add legacy format detection or conversion. -Portable inference artifacts remain actor-only. The run report records phase -order/config fingerprints, source/reference provenance, chosen metric and actor, -phase clocks, continuation mode and final artifact path. Checkpoint payloads can -contain training-only networks and references; inference loading must not depend -on them. - -## 6. Dependency-ordered delivery - -1. **Target contracts:** define the policy run/phase schemas, algorithm - registry, adapter capabilities, actor representation, phase-scoped paths and - structured results. Remove redundant policy configuration and dispatch. - Reject mixed supervised/policy plans and incompatible representations early. -2. **Coordinator and handoff:** implement the phase runtime contract, immutable - phase context, selected actor handoff and checkpoint envelope. Restructure - imitation, IQL and PPO to fit directly into the target architecture; define - their objectives and selection behavior explicitly. Demonstrate imitation → - IQL and imitation → PPO. -3. **DPO inputs and scoring:** implement pair manifest, source adapters, - component-safe splits, causal sequence log scoring and frozen reference - loading. Validate representation and prevent data leakage before training. -4. **DPO runtime:** add loss, optimizer, metrics, evaluation and resume; wire - the phase through CLI/Python and export. Exercise imitation → DPO and DPO → - another compatible algorithm as well as single-phase DPO. -5. **GRPO environment and buffers:** implement the shared online environment - contract, explicit group-start capability, reproducible independent rollout - streams, bounded complete-group collection and old/reference scoring. -6. **GRPO runtime:** add the specified group-normalized objective, actor-only - optimizer, separate evaluation and checkpoint/resume. Exercise imitation → - GRPO, DPO → GRPO, and a longer mixed plan with phase handoffs. -7. **Product integration:** document the new schemas, limitations, checkpoint - behavior, monitor meanings and sample configs. Update CLI help, public Python - exports and release notes; ensure final actors work with the redesigned - inference and rollout interfaces. Remove superseded unreleased policy code, - examples and documentation instead of maintaining parallel interfaces. - -## 7. Acceptance and verification for implementation - -The implementation is complete only when all items below have evidence. These -are future verification requirements, not tests executed for this plan. Use the -repository's actual test ownership and commands discovered at implementation -time; do not assume an external test checkout exists. - -- **Algorithm coverage:** single-phase BC, return-conditioned imitation, IQL, - PPO, DPO and GRPO work under the new contracts. Verify their stated objectives - and selection behavior; unsupported combinations fail before training. The - supervised training path predating this policy work continues to function. -- **DPO data and math:** hand-computed pair scores, causal masking, unequal - lengths, objective and gradient direction match the equations; actor equal to - reference gives zero margin and loss `log(2)`. Reference parameters never - receive gradients. Reject invalid pairs and train/validation leakage. -- **GRPO data and math:** identical group starts and independent reproducible - streams are established; analytic group normalization, clipping for both - advantage signs, reference penalty, per-trajectory weighting and zero-spread - groups match the declared variant. No critic is allocated, stale policy - versions and incomplete groups are rejected, and buffers obey memory limits. -- **Lifecycle:** every supported phase transition transfers exactly the chosen - actor and frozen codec; optimizers, samplers, reference policies and critics - have the stated ownership. Best/last behavior, evaluation failures, source - overlaps and output path collisions are explicit. A deterministic toy control - task demonstrates usable DPO and GRPO learning without making a universal - learning-quality guarantee. -- **Resume:** interruption at each committed update and phase boundary produces - equivalent subsequent state under the supported exact-resume contract; - changed data, reference weights, codec, config or adapter identity is - rejected. Non-exact environment reset is labeled, and partial groups are - never consumed. Only the new policy checkpoint format is required. -- **End-to-end:** run representative single- and multi-phase YAML configs via - CLI and Python, check phase reports and counters, then load the final ordinary - actor for inference and rollout through the redesigned interfaces. Verify deterministic reproducibility - on a supported local device and document any backend-specific limits. - -## 8. Deliberately deferred - -LLM/prompt-completion support, continuous or multi-component action spaces, -remote/vectorized environments, distributed/mixed-precision execution, -learned reward models, automatic preference generation, online preference -collection, asynchronous groups, dynamic external algorithm plugins, -supervised-to-policy phases, cross-phase optimizer/critic inheritance, and -mid-group exact checkpointing require their own contracts. Their omission must -not be represented as general support for those workflows. diff --git a/documentation/plans/offline-iql-implementation-plan.md b/documentation/plans/offline-iql-implementation-plan.md deleted file mode 100644 index 885ef61c..00000000 --- a/documentation/plans/offline-iql-implementation-plan.md +++ /dev/null @@ -1,222 +0,0 @@ -# Offline IQL implementation plan for the current policy-learning slice - -Status: implementation added, verification deferred at user request, 2026-11-09. -This is a scoped follow-up to -[the rollout and reinforcement-learning plan](rollout-and-reinforcement-learning-implementation-plan.md), -especially sections 3, 5.1, 5.3–5.5, and 8. The target is an **offline IQL path -as complete as the current scalar, single-categorical-action BC and -return-conditioned imitation path**: train from the existing supported sequence -sources, select and save an actor, resume training, load it for inference, and -execute it with the existing single-environment coordinator. The original plan's -continuous/joint actions, media, vector/remote environments, DDP, PPO, and -mixed-algorithm runs remain later work. - -## 1. Baseline in the diff since `2d01d407cd2134c8197b1a18f0034d01486f18a9` - -The diff provides the dependencies for this slice, but no IQL learner: - -- `sequifier train` accepts one `policy_imitation` phase. It trains categorical - BC or return-conditioned imitation using the ordinary model, optimization, - evaluation, and format-v2 run-checkpoint services. The best actor is a normal - portable model artifact. `sequifier infer` can make raw table decisions and - execute a policy through the shared rollout coordinator. -- CSV/Parquet, prepared scalar histories, and version-1 JSON rollout logs adapt - to `SequenceEpisode`. Optional `TransitionColumns` already represent rewards, - boundary flags, successors, and bootstrap flags. Episode splitting, training-only - fitting, bounded episode caching, and causal windows exist. -- Rollout collection has a persistent **single** local adapter, explicit input - bindings, terminal action execution, scalar reward, final successor, and atomic - one-episode JSON output. It is not a general vector simulator or an evaluation - service. -- `CategoricalPolicy` and `CategoricalActionDistribution` support one categorical - action. There is no V, Q, target-Q, transition sampler, IQL optimizer, or - multi-network checkpoint. Policy training's config and command dispatch reject - IQL and more than one phase. - -No repository test suite is present in this checkout; the release notes say the -tests moved to `../sequifier-test/tests/sequifier-refactor-unit`. Verification -below must discover and run that suite where available, and add focused tests in -the owning test repository or an agreed in-repository location. - -## 2. Scope and contract decisions - -1. **Action and model scope.** One fixed categorical action vocabulary, existing - scalar/categorical observation and static fields, finite context, one actor - interface, and independent V/Q1/Q2 modules. The actor is unconditioned - (`return_conditioned: false`). A return-conditioned actor is rejected by IQL - until an explicit conditioning or distillation policy is implemented. The - ordinary BC actor can warm-start IQL after exact schema, vocabulary, - normalization, architecture, and context-length checks; critics start fresh. -2. **Data scope.** A fixed offline training partition and separate held-out - partition from the current `SequenceSourceConfig` formats. IQL requires a - reward, action, known termination/truncation and bootstrap decision at every - sampled step, plus a genuine successor whenever bootstrap is allowed. BC's - ability to use reward-free data stays intact. Collection cutoffs and fragments - are eligible when their history and successor are known; unknown boundaries - are rejected for IQL. No new online data is added during the phase. -3. **Runtime scope.** One `iql` phase via `sequifier train`, with Python entry - point parity, single process on the currently supported policy devices, - actor-only inference/rollouts, and exact offline resume from complete update - boundaries. No same-run BC→IQL phase scheduling is required for this slice: - BC artifact warm-start is the explicit bridge. Do not imply the full - multi-phase or distributed claims of the parent plan are complete. -4. **Reward and evaluation scope.** Existing sources contain one scalar reward. - Preserve it in raw units; expose an optional fixed, checkpointed scale only - if needed. Select checkpoints using a declared held-out metric. Include - held-out losses/diagnostics and a repeatable local rollout evaluation on a - small control fixture, so learning quality is measured independently of - Bellman loss. A general training-time rollout monitor is not part of this - increment. - -## 3. Dependency-ordered work packages - -### A. Lock transition and successor-history semantics - -Extend `io/sequence.py` and `trajectory/importers.py` with an IQL eligibility -validator. Require transition columns to be complete for selected IQL steps; -reject absent rewards or boundary/bootstrap flags, non-finite reward, ambiguous -action timing, bootstrap after termination, bootstrap with a missing final -successor, and a reset observation passed as a successor. Preserve the current -ability to record both `terminated` and `truncated`, with termination taking -precedence. Make validation errors include source, episode ID, and step. - -Add an IQL transition view in `io/` that indexes decisions across episodes without -materializing all context windows. It returns `h_t`, recorded `a_t`, scalar -`r_t`, `h_next`, bootstrap mask, and valid/padding masks. Reuse -`PolicyWindowView` for `h_t`; add a successor-window method that appends the -**actual** successor observation and uses `a_t` as its previous action. For an -interior step, the explicit successor must equal the next row when both are -present. For a terminal step without a successor, return a masked placeholder -and never evaluate V on it. Do not cross episode or fragment boundaries by -looking up the next dataset item. Static context and trained codecs are shared -between both windows. Fit vocabularies and normalization on training episodes -only, then freeze them for validation and inference. - -Sampling is uniform over eligible transitions, with a persisted RNG state and -dataset identity (content hash, episode/split IDs, codec hash). Keep the existing -bounded episode cache and construct only the two context-sized windows per -sample. Continue to guard against duplicate paths, IDs, content, and collection -groups across train/validation partitions. - -**Gate A:** hand-authored episodes prove first/interior/final alignment, truncation -bootstrap from the final state, terminal no-bootstrap, fragment context, and -rejection of missing/contradictory capabilities. Changing `a_t`, reward, or future -observation cannot change actor/V inputs at time `t`. - -### B. Add actor, V, and twin-Q model primitives - -In `rl/models.py`, compose V, Q1, and Q2 from the existing ingestion/backbone -building blocks. V accepts only history. Each Q encodes history, then fuses the -**recorded current action** through a separate categorical action encoder before -its scalar head. Q1 and Q2 have independent learned parameters. Actor, V, Q1, -and Q2 share immutable codecs/specifications but no trainable parameters or -mutable normalization buffers. Make parameter ownership and optimizer assignment -explicit and validate disjointness before training. - -Expose a small policy runtime boundary in `rl/policy.py`/ -`rl/distributions.py`: deterministic/stochastic `act` and -`evaluate_actions` return the same action IDs/domain labels and joint log -probability for the recorded categorical action. Keep special/start IDs excluded -from actions. This is a one-component instance of the parent plan's contract; -continuous, ordered, and joint distributions are deferred. Keep probability -math in FP32 and reject invalid action labels. - -**Gate B:** shape, action-codec, finite-output, parameter-disjointness, and -gradient-ownership tests. Actor/V are invariant to changing current action -labels and future observations; Q responds to its explicit action input. - -### C. Add the IQL phase and optimizer transactions - -Add a discriminated `IQLPhaseConfig` to `config/training_plan.py` and route it -through `config/train_config.py`, `train.py`, and `rl/learner.py`. Keep the -existing supervised and imitation defaults. Require one IQL phase for this -slice, named training/validation sources, actor/V/Q specs, `updates`, `batch_size`, -`gamma`, `expectile`, `inverse_temperature`, `max_weight`, target-Q Polyak rate, -separate optimizer/scheduler plans, clipping limits, evaluation/checkpoint -intervals, and optional BC artifact initialization. Validate all ranges, -references, action/schema compatibility, unsupported distribution modes and -distributed/FSDP settings **before** a long run starts. Do not invent epochs or -supervised targets for an IQL update budget. - -Add `rl/algorithms/iql.py` behind an algorithm runtime that owns preparation, -one update, evaluation, checkpoint state, and named counters. Reuse the existing -optimizer transaction and random-state services where their semantics fit; -extend them to assign independent V, Q, and actor optimizers and scheduler -clocks. Controller/observer events must identify algorithm, network role, -algorithm update, and optimizer step so one-optimizer integrations cannot -silently act on a different network. The supervised loss path stays untouched. - -For each sampled batch, take a consistent pre-update snapshot of detached -targets/advantages: - -```text -qbar = min(Q1_target(h_t,a_t), Q2_target(h_t,a_t)) -delta = stop_gradient(qbar) - V(h_t) -L_V = mean(|expectile - 1[delta < 0]| * delta²) -y = r_t + gamma * bootstrap_t * stop_gradient(V(h_next)) -L_Q = mean((Q1(h_t,a_t)-y)² + (Q2(h_t,a_t)-y)²) -w = min(exp(inverse_temperature * stop_gradient(qbar - V(h_t))), max_weight) -L_actor = -mean(stop_gradient(w) * log pi(a_t|h_t)) -``` - -Cap the exponent input at `log(max_weight)` for stability. Apply V, Q, and actor -optimizer transactions to those targets without gradient paths across networks. -Update target Qs by Polyak averaging **only after a successful Q step**. Define -one IQL update as all three attempted transactions and record applied/skipped -counters separately; handle AMP skips without advancing the wrong scheduler or -target. Stop on non-finite targets/losses instead of silently clipping them. - -**Gate C:** hand-computed expectile asymmetry, Bellman bootstrap, twin minimum, -actor weights/cap, and target updates match reference numbers. Only the intended -network receives gradients in each transaction. A one-batch deterministic run -produces finite metrics and saves an actor that `load_policy` can execute. - -### D. Bundle IQL state without changing portable actor artifacts - -Add an algorithm-owned checkpoint bundle in `rl/checkpoint.py` or a dedicated -module. It contains actor/V/Q1/Q2 and target-Q weights, three -optimizer/scheduler/scaler states, update and applied-step counters, sampler and -all device RNG states, frozen normalization/codecs, source/split/content hashes, -phase config, and artifact-initialization provenance. Save atomically after a -complete update. Restore strictly; missing critics or mismatched data/specs -must fail exact resume. Retain reading of existing supervised/imitation -format-v2 checkpoints, and do not reinterpret one as an IQL resume bundle. - -Export **only** the actor through the current `sequifier_model` path. Record IQL -provenance without embedding critics or optimizers. Inference, `load_policy`, -and the single-environment coordinator should execute it with the existing -codec/action mapping. Artifact warm-start and exact IQL resume remain distinct. - -**Gate D:** interrupted versus uninterrupted fixed-seed training matches -sample order, counters, and weights within device-appropriate tolerances; -corrupt/mismatched bundles fail; existing format-v2 resume and model loading -still work; actor outputs match before and after export. - -### E. Evaluate, document, and release the slice - -Write a structured IQL report with source/split IDs, exclusions, update counts, -actor NLL, V/Q ranges, Bellman residual, advantage and weight quantiles, -weight-cap fraction, best-checkpoint criterion, and artifact paths. Held-out -diagnostics must not fit codecs or update networks. Provide a deterministic -small local control environment and mixed-quality offline behavior dataset. -Compare IQL against BC on fixed seeds using rollout return and action quality; -report mean, spread, dataset-generation rules, and tolerances. The local -rollout evaluation can initially be a separate explicit evaluation command/job -that consumes each saved actor and the same adapter protocol; it need not be a -generalized training monitor. - -Add one complete `train-iql` YAML example, Python API example, and documentation -for required transition columns, cutoff/truncation handling, actor warm-start, -resume, inference, unsupported combinations, and result interpretation. Update -`documentation/policy-learning.md`, config docs, `sequifier make` templates, -consolidated docs, and release notes. Run lint/type checks and the existing -suite, plus the focused gates above and a clean CLI train → checkpoint → reload -→ infer → rollout smoke run. - -**Gate E / done:** a user can import or point to an existing supported scalar -trajectory source, train a fixed-dataset categorical IQL actor with a named -phase, resume it exactly, inspect learning and held-out behavior, load the -actor as an ordinary model artifact, and execute it through the current local -coordinator. BC and return-conditioned workflows still pass their existing -checks. This completes **this IQL slice**, not all Stage 1/2 acceptance criteria -of the parent plan. diff --git a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md b/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md deleted file mode 100644 index b05f82f4..00000000 --- a/documentation/plans/rollout-and-reinforcement-learning-implementation-plan.md +++ /dev/null @@ -1,528 +0,0 @@ -# Rollouts, return-conditioned policies, IQL, and PPO - -Status: implementation specification. Date: 2026-11-07. - -First implementation slice: opt-in CPU numeric forecast collection with explicit -bindings, a persistent local single-environment adapter, shared prediction/step -ordering, seed history and fixed preprocessing, and validated JSON trajectories. -See [local forecast rollouts](../rollout.md) for configuration and public APIs. -The broader stages below remain planned. No tests or acceptance fixtures were -written or run for this slice, as requested. - -Second implementation slice: episode-split trajectory datasets, causal -current-observation/previous-action windows, discounted return labels, a single -categorical actor, return-conditioned supervised training and BC, local actor -artifacts, and execution through the same coordinator. See -[policy learning](../policy-learning.md). This slice has not been executed or -acceptance-tested; no tests were written or run, as requested. IQL and the -remaining broader stages are still planned; RC is an independent route. - -Third implementation slice: `sequifier train` resolves a single -`policy_imitation` phase with named trajectory sources and policies, an -algorithm runtime boundary, grouped episode splitting, best-action-NLL selection, -actor-only artifacts and training reports. Synthetic demo code, demo-specific -configurations/public exports and the experimental `policy` command are removed. -Applications supply their own recorded trajectories and environment adapters. JSON episodes and the CPU categorical actor remain interim contracts. -Mixed phases, phase transfer, sharded/multimodal storage, general rollout -infrastructure, IQL/PPO, distributed RL, exact resume and exports remain planned. -No tests were written or run for this increment, as requested. - -Integration follow-up: policy alignment now wraps the canonical configurable -network, with standard portable model and resumable run artifacts. `sequifier -infer` supports raw policy decisions and shared-coordinator environment rollouts. -Matching backbone transfer, CPU/CUDA/MPS execution, bounded file-backed episode -caching, context-sized windows, and BC eligibility without return labels are -implemented. The broader stages below remain planned. No tests or training runs -were executed for this follow-up. - -Representation unification follow-up: raw CSV/Parquet demonstrations, scalar -prepared histories and JSON recordings now adapt into a common columnar sequence -representation with optional transition capabilities. Policy views use explicit -same-position causal alignment, named inputs, standard sequence batches, shared -vocabulary/statistics fitting and supervised loss calculation. Shared model, -dataset-part and phase authoring is available alongside the compact compatibility -surface. Preprocess has an explicit optional raw sequence import operation with -atomic Parquet/manifest publication. BC requires no synthetic rewards, successors, -bindings or seed histories. Policy image/depth inputs, mixed-algorithm execution, -IQL/PPO and their algorithm-specific buffers remain planned. No tests or training -runs were written or executed for this follow-up, as requested. - -Offline IQL follow-up: one fixed-dataset, scalar, single-categorical-action -`iql` phase now uses transition-capable sequence sources, independent V and twin -Q networks, target-Q backups, advantage-weighted actor extraction, a complete -learner checkpoint, and ordinary actor-only model artifacts. See the -[scoped IQL implementation plan](offline-iql-implementation-plan.md) and -[policy learning guide](../policy-learning.md). This does not complete the -broader Stage 1/2 contracts below. No tests were written or run for this -follow-up, as requested. - -Scoped PPO follow-up: the [local categorical PPO implementation plan](scoped-ppo-implementation-plan.md) -defines the next implementation slice at the current BC/return-conditioned -product boundary. It does not replace the full Stage 3 requirements below. - -Deliver this as one coordinated update, built and verified in the three dependency-ordered stages below. Completion means all three stages work together; stage boundaries are engineering checkpoints, not separate proposed releases. This document specifies new behavior and proposed API names, not functionality already present. - -## 1. Outcome and scope - -The update must enable: - -1. Autoregressive inference when some next-step inputs come from predictions and others come from external dependent-variable calculations or simulation. Support scalar features, categorical features, image views, static conditioning, and depth inputs through the existing execution schema. -2. A persistent, batched environment interface shared by inference, training-time rollout evaluation, trajectory collection, and PPO. A separate optional reward provider can supply rewards from external software. -3. Versioned trajectories that can be collected online, imported from existing data, evaluated, reward-labeled, and consumed by offline learners without temporal leakage. -4. Return-conditioned sequence modeling (RCSM), complete offline Implicit Q-Learning (IQL), and full synchronous clipped PPO with generalized advantage estimation (GAE). -5. One training plan that can run supervised/return-conditioned pretraining, offline IQL, and online PPO phases, with explicit state transfer and optimizer lifecycle rules. -6. PyTorch learning and PT/ONNX policy deployment, including consistent preprocessing and action distributions. Training algorithms and external environments remain outside the exported model graph. - -Supported RL actions: ordinary categorical variables, bounded and unbounded continuous vectors, and joint policies combining these. Include independent categorical heads and explicitly ordered autoregressive categorical heads. All learnable action components have a joint likelihood. Existing ordinary regression heads remain available to supervised modeling and dependent-variable inference; RL phases require an explicit probability distribution. - -The release includes single-process CPU/CUDA execution and synchronous DDP learning with separate environment slots per rank. Existing supervised distributed strategies must keep working. New IQL/PPO phases reject FSDP initially with a specific error until multi-network optimizer/checkpoint semantics are implemented for it; do not advertise FSDP RL support through inheritance from supervised training. - -Not part of this update: SAC/TD3/AWAC, online IQL fine-tuning, learned world models, differentiable simulators, asynchronous actors, policy-gradient methods for diffusion/flow policies, constrained/multi-objective RL, dynamic action spaces, multi-agent learning, or hard real-time robot scheduling. Hash-decoded categorical outputs remain supported in existing inference but are rejected as RL actions unless they expose a tested normalized joint probability; candidate ranking scores are insufficient. - -One RL decision executes one action vector and receives one transition. The adapter may hold that action for a configured fixed number of physics substeps and aggregate its reward. Temporal action chunks with intervening replanning or variable-duration decisions need explicit semi-Markov discounts and are deferred. Ordered action dimensions at one instant are supported and are distinct from temporal chunks. - -## 2. Repository integration points - -Current constraints established by source inspection: - -- `infer.py:get_probs_preds_from_dict` owns a NumPy generation loop, assumes each changing input exists in `preds_for_step`, and rejects future image frames. Extract orchestration rather than adding independent loops to each consumer. -- `config/infer_config.py` separately enforces identical inputs/targets in autoregression and generation-step validators; depth autoregression is rejected elsewhere. Replace these restrictions with source coverage and execution-schema checks for supported rollout paths. Preserve forward-looking, one-step generative constraints where still required. -- `model/network.py` exposes encoding/decoding primitives and `ModelOutput`. Preserve the supervised output contract; add policy/value/Q wrappers rather than changing every scalar output into an RL distribution. -- `io/batch.py` and `training/loss.py` describe supervised batches and masked supervised losses. RL needs dedicated transition/rollout batches; do not overload target masks with termination semantics. -- `training/engine.py`, `training/runtime.py`, `training/optimization.py`, and `train.py` already separate orchestration from model, data, and update services. Extend that separation with algorithm runtimes. -- `integration/specifications.py` already defines persistent `module:attribute` factories and state policies. Extract reusable factory loading/identity validation; environments are not `TrainingObserver` or `TrainingController` instances and must not be forced into those protocols. -- `evaluation/service.py` and its monitor currently describe supervised loss. Extend results/monitors with named metrics rather than placing returns in `total_loss`. -- `artifacts/run_checkpoint.py` currently stores one model and optimization state in format version 2. Introduce an explicit compatible checkpoint migration and an algorithm-owned bundle for additional networks/optimizers. -- `hyperparameter_search.py` launches `evaluation_script` once and reads named JSON metrics. Keep that contract. Share factory conventions with hooks, not its subprocess-per-evaluation execution model. - -Proposed new modules: - -```text -config/rollout_config.py Environment/reward factories, bindings, budgets -config/rl_config.py Policy specs and discriminated algorithm phases -rollout/{contracts,bindings,history,coordinator,returns}.py -rollout/{factories,dependent,rewards,remote}.py -rollout/adapters/{gymnasium,tensor_vector}.py -trajectory/{schema,store,importers,dataset,relabel}.py -rl/{policy,distributions,models,normalization,metrics}.py -rl/{runtime,checkpoint,transfer}.py -rl/algorithms/{policy_imitation,iql,ppo}.py -rl/{ppo_buffer,offline_sampling}.py -rollout_api.py Stable external adapter/collector API -rl_api.py Stable policy and learner API -``` - -Names can be adjusted during implementation, but the ownership boundaries and contracts below are requirements. - -## 3. Common contracts to settle before implementation - -### 3.1 Time and observation/action alignment - -Use the following canonical transition everywhere: - -```text -h_t = observations through o_t, actions only through a_(t-1), validity and time metadata -a_t ~ policy(h_t) -environment.step(a_t) -> o_(t+1), reward_t, terminated_t, truncated_t -transition = (h_t, a_t, reward_t, o_(t+1), boundary flags) -``` - -The policy and V must never see `a_t`, its reward, or next observation when computing their step-t outputs. Q may receive `a_t` through a separate action encoder. Introduce an explicit policy window view that puts current observations and previous actions at one decision position and supervises the current action there. Do not reinterpret ordinary `target_offset` silently. For existing forecast models, keep their training alignment and use an explicit forecast binding; do not claim that arbitrary next-item checkpoints can be converted into current-observation policies without retraining. - -For robotics, the row contains current images/proprioception and the previous motor command; the output is the next command to execute. For order-book forecasting, a forecast binding appends predicted level-4 features and adapter-computed level-2 features into the same next context position. Both use the coordinator, with different bindings. - -### 3.2 Environment and reward protocols - -Use typed dataclasses/protocols with a protocol version. Illustrative signatures: - -```python -class EnvironmentAdapter(Protocol): - def describe(self) -> EnvironmentSpec: ... - def reset(self, request: ResetRequest) -> ObservationBatch: ... - def step(self, request: StepRequest) -> TransitionBatch: ... - def close(self) -> None: ... - -class RewardProvider(Protocol): - def score(self, transitions: TransitionBatch) -> RewardBatch: ... - def finalize_episode(self, episode: EpisodeSummary) -> EpisodeOutcome: ... - def close(self) -> None: ... -``` - -`EnvironmentSpec` declares ordered fields, shapes, dtypes, units, bounds, batch/device support, partial-reset/autoreset semantics, seed support, and snapshot capabilities. `StepRequest` includes slot IDs, episode IDs, transition indices, an idempotency/request ID, actions, and relevant context. The coordinator steps only active slots. GPU adapters use fixed slots plus an active mask; adapters unable to step subsets must freeze inactive worlds or keep all slots active during collection. Never silently advance a live trajectory without recording it. - -`TransitionBatch` contains true next observations, named rewards if supplied, separate `terminated`/`truncated` masks, optional declared privileged critic observations, elapsed simulated time, and diagnostics. Return final pre-reset observations explicitly. Autoreset adapters normalize their outputs into this contract and separately expose reset observations. A state that was reset is never used as the successor of the preceding episode. - -Every hook is instantiated once per rank/run or evaluation session and closed in `finally`. Partial reset is explicit and changes only selected slots. Episode IDs are unique across ranks/restarts. A stateless dependent-variable adapter uses the same protocol, seeded from the initial history, but can simply calculate a deterministic function of predictions without a physics engine. - -Environment and reward factory specifications share `factory`, `config`, `id`, and version fields. Use `module:factory` as the CLI/config path and allow supplied instances for local Python runs. Construct instances inside workers for DDP. Adapter configuration specifies connection/launch details; no per-step process startup. - -The public value representation is domain values: original categorical labels, physical continuous units, and image arrays/tensors. One shared codec maps to/from model representation using fixed training metadata. Provide a numeric tensor fast path with declared integer vocabularies and scales; use a metadata handshake once to avoid decoding labels through Python on every GPU step. Reject changed units/vocabularies/shapes and non-finite values. Never refit preprocessing on rollout/evaluation data. - -Provide optional adapter `state_dict`/`load_state_dict` and explicit capabilities for exact snapshots, including remote simulator RNG and reward state. Exact snapshots are optional; never infer snapshot support from the existence of a Python method alone. - -### 3.3 Bindings and field ownership - -Each dynamic model-input field, image view, and structured input has exactly one next-step source: `prediction`, `environment`, or `provided` (an explicitly scheduled exogenous input). Static fields remain constant within an episode. Return conditioning is owned by the coordinator. Reject missing coverage, duplicate producers, ambiguous prediction/environment overrides, and exhausted exogenous schedules before the affected transition. - -Mappings include prediction-to-action names and codecs, input update names, and structured masks. A model can predict outputs that are only actions and never fed back, or outputs that are appended directly. Environment outputs can include full corrected state when simulator state is authoritative. This choice must be declared per field; do not automatically overwrite predicted values. - -Image frame-valid masks, attention masks, depth occupancy masks, and time coordinates advance atomically with their features. Reuse `ExecutionSchema` for shape/dtype checks. Depth updates replace a complete next-position structured value plus its mask. Prefix/preconditioning images remain static unless the policy interface explicitly models them as dynamic image views. - -### 3.4 Rewards, outcomes, and discounts - -Store raw named reward channels and episode metrics separately. Algorithms optimize one scalar reward selected by explicit fixed channel weights and transforms. Version and persist this scalarization; multiple channels do not imply a multi-objective learner. Avoid double-counting a channel produced by both the simulator and reward provider. - -Support sparse terminal rewards: the episode outcome callback runs on termination/truncation before the terminal transition is sealed and may supply an explicitly configured terminal reward adjustment. Once emitted to a learner, a transition is immutable. Scores available only after a whole collection run are evaluation metrics unless an offline relabeling job constructs a new dataset. PPO rejects unresolved delayed reward providers; it cannot update from provisional rewards and revise them later. - -Distinguish task termination from time-limit truncation and collection boundaries. Store a `bootstrap_allowed` flag validated against those boundaries: ordinary true termination disables bootstrap; time-limit truncation with a valid final observation permits it. If task semantics make a time limit genuinely terminal, encode that explicitly. Collector horizon cuts are fragments, not artificial episode ends. - -If an adapter reports both termination and truncation on the same transition, preserve both facts and give true termination precedence for bootstrap and GAE. Import validation rejects contradictory discount/next-observation claims, not the simultaneous flags by themselves. - -Use fixed gamma per decision for this release. Record physics substep count and duration, and reject variable-duration RL decisions rather than applying a misleading fixed discount. Raw rewards remain unchanged in artifacts; any learner reward scaling is a fixed, checkpointed transform. - -## 4. Stage 1 — shared rollout and trajectory infrastructure - -### 4.1 Coordinator and history storage - -Implement a coordinator independent of CLI, model format, and optimizer. It owns slot identities, bindings, codec calls, context windows, masks, environment/reward lifecycle, collection budgets, and sink writes. It calls a one-step predictor interface; PT/ONNX forecasting and RL policy runtimes implement that interface. - -Use a per-slot history ring buffer with explicit decision positions and padding. Each collection starts from either validated historical seed windows or adapter reset observations. Default for reset-only starts: left padding with false attention-valid masks, then one real observation and a declared previous-action start token/value. Longer warm-up requires an explicit source/controller and is recorded. Never fabricate valid historical camera frames or prior actions. - -Budget modes are explicit: maximum transitions, completed episodes, and per-episode maximum decisions, plus inference `generation_steps`. Prediction output budgets count emitted predictions, even when trajectories end early. Ragged results carry episode/step identity; padded tensors carry valid masks. For fixed-shape GPU collection, collect the configured decisions per slot and reset ended episodes in place. - -Terminal predictions must still be executed and rewarded. Refactor the existing generation loop's early break so the final requested action/forecast gets a transition and outcome even though no next model call is needed. In pure legacy prediction-feedback mode use a no-op adapter and avoid externally visible extra work. - -Build one coordinator path for eager PT/ONNX and media/tensor inputs. Avoid eager CPU conversion for the torch policy path. Separate output formatting from collection so existing flat prediction/probability output remains unchanged when there are no episode boundaries. Include episode IDs and masks in the new trajectory outputs. - -### 4.2 Adapters and service transport - -Ship these adapters/examples: - -- A dependent-feature adapter for level-4 to level-2 order-book feature updates, with deterministic synthetic data and no trading-system dependency. -- A native torch vector adapter for small analytic control environments, with CPU/CUDA support and partial resets. This is the reference for high-throughput learning. -- An optional Gymnasium single/vector adapter covering discrete, continuous Box, MultiDiscrete, and supported dictionary observations. Pin/test the optional dependency and normalize supported autoreset modes explicitly. -- A deterministic toy robotics adapter with RGB views and motor commands, demonstrating image/mask updates without a heavyweight simulator installation. -- A generic persistent remote adapter and example server using versioned, framed requests/responses, request IDs, deadlines, handshake schemas, and bounded payload sizes. Numeric arrays use a binary representation preserving shape/dtype; avoid JSON pixel lists. Support start/connect/close and demonstrate crash handling. No transparent retry of a non-idempotent step after an ambiguous timeout; require server deduplication or terminate collection with a clear error. - -Third-party robotics integrations implement the same adapter protocol. The release claims complete simulator integration plumbing, with tested local/vector/remote examples; it does not claim bundled production integrations for every robotics engine. Keep Gymnasium and media dependencies optional and lazy-loaded. Timeouts are specified per reset/step/reward request; a synchronous local call cannot be forcibly interrupted in-process, so adapters requiring hard deadlines must use process/service isolation. - -### 4.3 Trajectory artifact version 1 - -Define a manifest plus atomic, sharded records. Use Arrow/Parquet for tabular columns and indexed binary/tensor shards for images and depth payloads; do not require all media in RAM. Include a final committed manifest, shard checksums, counts, codec/schema hashes, and immutable source provenance. Incomplete shards are recoverable but never silently treated as a finished dataset. - -Required logical fields: - -| Scope | Fields | -|---|---| -| Dataset | Format version, field/units/action specs, preprocessing/codec hash, reward definition and scalarization, task ID, split IDs, simulator/adapter versions, collection policy artifact ID | -| Episode | Globally unique ID, slot/rank, seed/reset conditions, static context, start/end status, outcome metrics, simulated duration | -| Transition | Episode ID, t, current observation reference, action, next/final observation reference, named rewards, terminal/truncation/bootstrap flags, time, validity, behavior policy version | -| Optional learner data | Joint old log-probability, value estimate, pre-transform continuous action, action masks, privileged critic observations, collection generation | -| Fragment | Starting history references, continuation/end reason, preceding action, whether complete episode outcome/return is known | - -Store observations once and reference adjacent frames where practical. References preserve true terminal successor observations even when the next stored record is a reset. Serialize action component order, allowed categories, bounds, transforms, and masks as part of compatibility hashes. Sinks support configurable flush intervals and bounded queues; backpressure cannot drop transitions. - -Add strict importers for existing tabular/PT histories with explicit mappings for episode, observation, action, reward, and boundary fields. Reject gaps, conflicting terminal flags, missing successors for bootstrapped records, and ambiguous action timing. Never assume every file is one complete episode. For final terminal steps without observable successors, permit an explicit missing-next mask only when no bootstrap is used. Split by whole episode (and optionally task/collection group), not windows; detect duplicate episode IDs across train/evaluation splits. - -Offline reward relabeling produces a new manifest with parent dataset ID and new reward version. Episode metrics are not automatically copied into every step as rewards. Add return/transition validation utilities and inspectable trajectory summaries. - -### 4.4 Inference, evaluation, and configuration - -Add shared rollout config referenced by `infer`, new `sequifier rollout --config-path`, and training rollout evaluation sources. `infer` keeps its existing arguments and output behavior when rollout config is absent. `rollout` can start from environments without a file-backed seed dataset. - -Replace input/target equality validation with binding coverage only when a rollout binding is present; legacy feedback paths keep their existing checks until routed through an equivalent generated binding. Keep BERT/embedding and multi-position forecast restrictions explicit. No relaxing a validator without implementing the associated image/depth history behavior. - -Training-time rollouts run against an immutable evaluation snapshot, separate environment slots and RNG streams, and no learner gradients. Restore model mode after evaluation. Metrics include channel returns, scalarized return, success rate when defined, episode length, completed/censored counts, simulated time, and collection/policy/transport timing. Aggregate weighted sums/counts across ranks; report incomplete episodes separately. - -Extend evaluation sources to a discriminated supervised/rollout union and monitors to select source + metric + direction. Preserve legacy total-loss selectors by normalization. Training schedules use epoch/batch triggers for supervised phases and update/environment-step triggers for RL phases. Hyperparameter search can run rollout evaluation using `evaluation_inference_config` pointing to an inference config with rollout settings, or consume native named evaluation metrics; keep legacy `evaluation_script` JSON output intact. Pruning uses the configured comparable metric, not an accidental PPO loss. Disallow incompatible per-trial metric schedules. - -Replace loss-specific best-model bookkeeping in `RunState` with a typed monitor identity/value/direction and a migration for existing `best_validation_loss` state. Update best-checkpoint selection, export naming/selection, early stopping, visualization, and search consumers together. Continue to emit legacy supervised loss metrics alongside the new named metric stream so existing integrations remain usable. - -### 4.5 Stage 1 acceptance - -- An order-book model with level-4 outputs and level-4/level-2 inputs generates multiple steps; synthetic adapter outputs prove that the next model call sees the derived features. -- Camera observations returned after a motor command become the next policy input, including masks, on CPU and the optional GPU path. -- Different episode lengths, partial resets, terminal final observations, truncations, and collection fragments survive collection → storage → loading unchanged. -- PT and ONNX forecast rollouts have matched deterministic results within documented tolerances; all existing inference tests pass. -- A terminal reward is recorded for the final emitted action, adapters close on success/error, and remote timeout/retry behavior is tested. -- Rollout evaluation can select a best checkpoint and return custom hyperparameter-search metrics. - -## 5. Stage 2 — return conditioning and complete offline IQL - -### 5.1 Policy distributions and model wrappers - -Implement these before either IQL policy extraction or PPO: - -```python -PolicyRuntime.act(history, *, deterministic, generator) -> PolicyStep -PolicyRuntime.evaluate_actions(history, actions, auxiliary) -> ActionEvaluation -``` - -`PolicyStep` includes domain action, internal action, joint log-probability, optional value, action mask, and any continuous latent needed for numerically stable likelihood evaluation. `ActionEvaluation` evaluates exactly the same recorded joint action. Its joint log-probability sums over action components, never averages them. - -Distributions: - -- Categorical and independent MultiDiscrete components, excluding special tokens and respecting state-dependent masks. Reject an all-masked component. Store masks used during collection. -- Ordered autoregressive categorical actions: sample in declared component order; evaluate recorded actions by teacher forcing that same prefix; sum conditional log-probabilities. No sampling a new prefix when scoring a recorded action. -- Unbounded diagonal Gaussian continuous actions with learned/clamped log standard deviation. -- Affine tanh-transformed diagonal Gaussian for bounded continuous actions. Account for the tanh and affine Jacobians and retain pre-tanh actions for PPO. Never externally clip a sampled Gaussian and then score it as if unclipped. Offline actions exactly at bounds require an explicit, reported inverse-transform epsilon convention or rejection; no silent arbitrary clamping. -- Joint mixed actions: explicit factorization and component order. Initially continuous blocks are independent conditional on history; any dependence on sampled categorical actions must be explicitly represented and identically evaluated. - -Use FP32 probability math even with lower-precision networks. Analytic entropy is available for independent categorical/unbounded Gaussian policies; for transformed or autoregressive distributions use a documented current-policy Monte Carlo estimator, not entropy evaluated on stale PPO actions. Temperature, masks, normalization, and action bounds are fixed within a PPO collection/update generation. - -Freeze dropout and stochastic augmentation for RL action collection and likelihood evaluation, while retaining autograd during optimization (`eval()` does not mean `no_grad()`). Freeze mutable observation-normalization state within a PPO iteration. Use deterministic per-record transforms where any image preprocessing is required. Before the first optimizer update, old/new log-probabilities must match numerically. - -Compose separate actor, V, Q1, and Q2 networks using existing ingestion/backbone/encode primitives. Default to no shared trainable parameters between actor, V, and Q networks; this prevents unintended optimizer interaction. Q encodes history first, then combines its representation with the recorded current action using an action encoder. V receives history only. Q1/Q2 may be implemented as a validated twin module but must have independent learned estimates. Share immutable preprocessing specifications, not mutable normalization buffers. - -Allow explicitly separate actor and critic observation schemas for asymmetric simulator critics. A privileged critic input must be available in collection, its genuine successor state, offline data when used for IQL, and resume snapshots. It must never enter the actor codec, actor encoder, or exported actor artifact. Validate and test this isolation. Default all networks to the ordinary observation history to keep dataset-only workflows straightforward. - -Initialize actor from a compatible supervised policy artifact; initialize critics independently or from explicitly copied encoders. Register all parameters in the parameter catalog, freezing/optimizer routing, initialization provenance, and checkpoint bundle. Ordinary point-regression actor transfer can initialize Gaussian means only under an explicit conversion policy, with new standard-deviation parameters recorded. - -### 5.2 Return-conditioned sequence modeling - -Implement a named algorithm runtime `policy_imitation` using the policy-aligned window view. This is Decision-Transformer-style return conditioning using Sequifier's multivariate rows; do not claim identical interleaved return/state/action token architecture. - -For complete recorded episodes compute: - -```text -G_t = reward_t + gamma * G_(t+1), with G_T = 0 -``` - -Gamma is explicit in `(0, 1]`, default 1 for this mode. Compute rewards with the configured fixed scalarization; persist return scale and definition. For incomplete/truncated episodes, require a declared finite-horizon objective or exclude them from exact return-to-go training. Never present an unknown future tail as a known zero return. IQL may still consume bootstrappable fragments from the same dataset. - -Add `desired_return` as a dynamic input at the current decision position; actions remain targets, with only previous actions in features. Use the existing supervised optimization services for masked action likelihood loss. Optional deterministic regression loss can be exposed only for the standalone return-conditioned learner; such an artifact is not directly PPO/IQL-compatible until converted into a distributional actor. Default to the shared distributional policy for seamless phase transfer. - -Inference requires an initial desired return in raw configured reward units and updates it after each observed reward: - -```text -desired_return_(t+1) = (desired_return_t - reward_t) / gamma -``` - -Apply scaling only at the encoder boundary and reset the budget per episode. Optional clamps must be authored configuration and logged; do not infer attainable targets from evaluation episodes. Unknown future realized returns are never used online. - -Compare with behavior cloning and success-filtered behavior cloning on a controlled dataset with conflicting actions and known return differences. Test gamma/scaling arithmetic, episode boundaries, and absence of future-action/reward leakage. - -### 5.3 IQL data and update implementation - -Add an `iql` runtime that samples fixed-dataset transitions plus sufficient history for both h_t and h_(t+1). Sampling a transition includes up to the configured context length of history and one successor observation; never reconstruct successor context by concatenating arbitrary independently sampled windows. Respect masks, previous-action alignment, static context, and image/depth payload references. - -Implement the canonical losses using scalarized rewards, learned V, twin Qs, and slowly updated target Qs. At each update, compute detached targets/advantages from a consistent pre-update snapshot: - -```text -q_bar = min(Q1_target(h_t, a_t), Q2_target(h_t, a_t)) -delta = stop_gradient(q_bar) - V(h_t) -L_V = mean(abs(expectile - indicator(delta < 0)) * delta^2) - -y = reward_t + gamma * bootstrap_allowed_t * stop_gradient(V(h_next)) -L_Q = mean((Q1(h_t,a_t)-y)^2 + (Q2(h_t,a_t)-y)^2) - -advantage = stop_gradient(q_bar - V(h_t)) -weight = min(exp(inverse_temperature * advantage), max_weight) -L_actor = -mean(stop_gradient(weight) * log policy(a_t | h_t)) -``` - -Evaluate the exponential stably by capping its input at `log(max_weight)`. Reject invalid expectile/temperature/target-update settings. Target Qs are frozen copies, updated by Polyak averaging only after a successful critic optimizer step. Define one algorithm update as the configured V/Q/actor transactions and record individual update counters. Recompute forward graphs as necessary; do not backpropagate through detached targets, target networks, or the weight calculation. If AMP skips a transaction, dependent counters/target updates follow the actual applied steps; checkpoint only after the complete update boundary. - -Keep initial scope faithful to offline IQL: a fixed training dataset, an independently held-out evaluation environment/dataset, uniform transition sampling, and no fresh data injected during the phase. Expose expectile, inverse temperature, maximum weight, gamma, Polyak factor, batch size, gradient clipping, and separate optimizer/scheduler plans. Reward transforms are fixed. Log Q/V ranges, Bellman residuals, advantage/weight distributions, weight-cap frequency, actor likelihood, update counts, and held-out rollout performance. - -RC actor to IQL transfer requires a declared conditioning policy: either retain desired-return conditioning with identical semantics for all actor/critic contexts, or initialize a separately specified unconditioned actor via an explicit conversion/distillation step. Never silently remove an input. The straightforward default workflow is behavior-cloned unconditioned actor → IQL; return-conditioned training is a separate supported route, not a mandatory IQL prerequisite. - -### 5.4 Training-plan changes - -Replace resolved phase branching with a discriminated union normalized from authored config: - -- `supervised`: existing sources, epochs, and source scheduling, unchanged defaults. -- `policy_imitation`: trajectory sources, policy interface, epochs, return definition. -- `iql`: fixed trajectory sources, updates, actor/V/Q specifications, algorithm settings. -- `ppo`: environment source, policy/value specifications, environment-step budget, rollout and update settings (Stage 3). - -Legacy phases without an algorithm normalize to supervised. Give online phases their own source type and budget; they must not require dummy `dataset_training` files, criteria, epochs, or validation loaders. Adjust singleton normalization, canonical search config, authored/resolved validation, phase compilation, docs, and CLI routing together. Search spaces can address new discriminated config fields after ordinary canonical composition rules. - -Introduce `AlgorithmRuntime` with explicit collect/prepare/update/evaluate/checkpoint boundaries and named counters. Reuse optimization transaction mechanics, random-state handling, distributed strategy, metrics, and integration events. Keep supervised `LossService` intact; algorithm loss functions return their own structured result with accounting counts. Do not implement RL as observer callbacks changing gradients behind the engine's back. - -Extend update event identities with algorithm, network/optimizer role, algorithm-update index, and environment-step count. Keep existing supervised identity fields/behavior compatible. Controller directives apply only to the named optimizer transaction; observer/controller state is saved once per declared instance. Validate execution requirements per algorithm so a controller assuming one optimizer cannot silently change the wrong network. Define scheduler clocks explicitly per optimizer update, not in fabricated online epochs. - -Phase transitions retain actor weights and compatible codecs, but reset optimizer moments by default when algorithms change. Copy/reset value state only under explicit compatible specs. Record transfer provenance and validate action/observation schema, context length, return conditioning, and bounds before copying. Artifact warm-start and exact run resume are different operations. - -### 5.5 Stage 2 acceptance - -- Return-conditioned policies change action choices with requested return on a deterministic fixture; online return-budget arithmetic matches offline labels. -- IQL losses, gradient ownership, expectile asymmetry, weight caps, and target updates match hand-computed examples. -- A tiny tabular/analytic control task demonstrates learned value ordering and policy improvement against a deliberately mixed-quality behavior dataset over fixed seeds; include tolerances and report variance rather than asserting monotonic improvement per update. -- Actor/V outputs are invariant to replacing current-action labels or future observations; Q changes appropriately with its explicit action input. -- Distribution sampling/scoring round trips, categorical prefixes, action masks, and bounded Gaussian Jacobians pass numerical tests. -- Exact offline resume reproduces uninterrupted sampling and updates within device-appropriate deterministic tolerances; actor-only exports reproduce pre-export policy outputs. - -## 6. Stage 3 — full synchronous PPO - -### 6.1 Collection and rollout buffer - -Freeze a behavior policy version per iteration. Collect `num_envs × rollout_length` decisions per rank using the Stage 1 coordinator. Run collection without autograd; store observations/history references, sampled actions and latent values, joint old log-probabilities, old V estimates, scalar/raw rewards, final observations, terminal/truncation flags, masks, and policy version. - -A completed episode resets immediately in its slot and starts a new history. A collection cut preserves live context for the next iteration. PPO may change the policy between fragments of the same episode; each transition retains its behavior version and only the current iteration's buffer is optimized. Treat environment-step count as the total across ranks. Validate a total budget compatible with fixed-size synchronous collection or explicitly define a final smaller masked rollout; never silently overshoot the authored budget. - -Preallocate device rollout tensors/rings for numeric/image payloads and avoid duplicating full history windows per transition. Retain enough pre-rollout context to reconstruct every training history exactly. If storage is insufficient, fail before collecting, with a memory estimate. Permit CPU/pinned staging where configured; benchmark transport separately. Seal the buffer before calculating targets or updating. Reuse each buffer only for its configured PPO optimization epochs, then discard it from on-policy training (optional trajectory archival is independent). - -### 6.2 Bootstrap and GAE - -Evaluate V on each genuine successor/final observation using the corresponding successor history, with the same frozen value snapshot used during collection. Let b_t indicate whether bootstrap is permitted; let c_t indicate whether GAE recursion continues into the next stored transition of this same episode and collection segment: - -```text -td_t = reward_t + gamma * b_t * V_old(h_next) - V_old(h_t) -A_t = td_t + gamma * lambda * c_t * A_(t+1) -value_target_t = stop_gradient(A_t + V_old(h_t)) -``` - -True terminal: b=0, c=0. Time-limit truncation with valid final state: b=1, c=0. Ordinary transition inside the segment: b=1, c=1. Last live transition at a collection cut: b=1, c=0, so it bootstraps without accessing the next collection's advantages. Never bootstrap using reset observations or carry GAE across episode IDs. - -Compute GAE in FP32. Normalize advantages once over globally valid transitions per iteration using DDP sums/counts (configurable off); do not normalize rewards implicitly or re-normalize each minibatch. Keep returns and old values fixed through optimization. - -### 6.3 PPO loss and optimization - -Evaluate each recorded action on the exact same observation history, mask, codecs, and frozen preprocessing state. Joint ratio and losses: - -```text -ratio = exp(logp_new - logp_old) -L_policy = -mean(min(ratio * A, - clip(ratio, 1-epsilon, 1+epsilon) * A)) -L_value = 0.5 * mean((V_new - value_target)^2) -L_total = L_policy + value_coefficient * L_value - - entropy_coefficient * entropy_current -``` - -Expose optional value clipping around `V_old` as a documented maximum-of-clipped/unclipped loss, disabled by default. Implement optimizer gradient clipping and optional target-KL early stopping. Estimate KL consistently (for example `mean((ratio-1)-log_ratio)`) and aggregate globally; every rank stops at the same boundary. Log clipping fraction, ratio tails, entropy, KL, gradient norm, explained variance, value loss, returns, episode success, steps/sec, and timings. Detect non-finite ratios/losses and fail the iteration; do not hide them with arbitrary ratio clamping outside PPO's specified clipped objective. - -Use separate actor and value modules/optimizers by default, with no shared parameters. Combined mathematical loss can be split into separate transactions; both optimize the same sealed targets. Support configurable optimization epochs, minibatch size, and gradient accumulation with correct valid-sample weighting. Randomly sample decision indices and reconstruct complete fixed-length causal contexts; do not shuffle time positions inside a context. This is a finite-context transformer policy, not a persistent recurrent state requiring truncated BPTT. Any future hidden-state/KV-cache optimization must prove equivalence after parameter updates and resets. - -The complete PPO schema includes collection horizon/environment count, total budget, optimization epochs, minibatch/accumulation sizes, gamma, lambda, clipping epsilon, optional value-clip range, value/entropy coefficients, optional target KL, advantage normalization, actor/value optimizer and scheduler plans, gradient limits, action standard-deviation initialization/bounds, evaluation intervals, and checkpoint/resume policy. Validate positivity/ranges, actor/value parameter disjointness, batch coverage, divisibility or explicit remainder masks, and supported execution/distribution combinations before instantiating expensive simulators. Do not hide these controls in free-form adapter configuration. - -Use the shared distributions for discrete, continuous, ordered categorical, and mixed policies. A density-free deterministic regression head is invalid for PPO. Default evaluation uses mode/mean actions and no exploration; report separately if stochastic evaluation is requested. Deployment returns the same bounded/domain actions as collection. - -### 6.4 Distributed synchronization and simulator execution - -Each DDP rank has its own simulator batch, unique episode/slot IDs, and derived seeds based on run, rank, slot, and reset count. Actor/value parameters are synchronized before collection. All ranks perform compatible collect/update schedules, use globally weighted valid-sample losses, and agree on KL stopping, errors, pruning, and checkpoint boundaries. Avoid rank-zero-only environment collection feeding a nominal DDP learner. - -For IQL, partition/sample offline data with deterministic rank-specific samplers and synchronized update counts; target networks update from synchronized Q parameters. For all new runtimes, distinguish valid transition counts from padded context lengths in loss denominators and metric reductions. - -Remote adapters use batched RPC calls and one connection/session per declared rank namespace. A slow service may reduce throughput but must not silently mix policy generations. Support GPU tensor adapters without mandatory CPU synchronization per field. Simulator APIs, transport, rewards, and preprocessing have separate timings so performance problems are diagnosable. - -### 6.5 Checkpoints and resume - -Version the algorithm checkpoint payload and retain a reader/migration for existing supervised format-v2 artifacts. Save model specs/weights for actor, V, Q/target Q where relevant; separate optimizer/scheduler/scaler states; algorithm counters; dataset samplers; policy version; random streams; normalization; codec/action spec hashes; phase transfer provenance; and environment/reward state capabilities. - -For PPO, write durable checkpoints at complete iteration boundaries after optimizer updates, target computation, and metric accounting have finished. Exact resume additionally requires snapshots of live simulator states, histories, reset counters, reward state, and all rank RNG streams. If the adapter cannot snapshot, provide an explicit `reset_environments` resume mode: preserve learner state, reset environments/history, discard incomplete episode metrics, and label the run as a continuation without trajectory-equivalent resume. `exact` mode must fail rather than claim reproducibility. Mid-collection/mid-minibatch resume is not required in this release; a crash restarts from the last committed boundary. - -Keep actor-only model artifacts small and independent of critic state. Learner checkpoints contain the complete training bundle. Validate every network and optimizer assignment during restore; no missing-critic fallback in exact resume. Atomic manifests must not point to partially written rank state. - -### 6.6 Stage 3 acceptance - -- Hand-computed PPO clipping tests cover positive/negative advantages, joint action likelihoods, optional value clipping, entropy, and KL stop behavior. -- GAE tests cover true terminals, truncations, collection cuts, autoresets, nonuniform episode lengths, and missing invalid successor observations. -- The identity-ratio invariant holds before any update, including image inputs, ordered action heads, and checkpoint round trips. -- Discrete and bounded continuous control environments show reliable learning over multiple seeded runs. Keep short deterministic CPU learning checks in CI; run longer multi-seed quality benchmarks separately and save results. -- An RGB toy control policy completes collection → PPO update → evaluation → checkpoint → reload → rollout. A structured/depth observation fixture proves the same schema path. -- DDP results agree with single-process global-batch loss calculations within tolerance; one rank hitting KL/error/termination does not hang others. -- Exact-snapshot adapters reproduce an uninterrupted continuation; non-snapshot adapters correctly reject exact resume and pass explicit reset-mode continuation. -- Remote vector environments can train PPO using the same algorithm runtime as in-process tensor environments. - -## 7. Authoring surface and operational workflow - -All names below are proposed new fields. Implement schema-validated examples with the complete existing model configuration in documentation/examples; these snippets show the new surface only. - -```yaml -rollouts: - robot: - environment: - id: toy_robot - factory: examples.toy_robot:make_environment - config: {num_envs: 32} - reward: - scalarization: {task_success: 1.0, effort: -0.01} - bindings: - actions: {motor_command: motor_command} - inputs: - camera: {source: environment, field: camera} - joint_position: {source: environment, field: joint_position} - previous_command: {source: prediction, field: motor_command} - -training_plan: - phases: - - name: offline_improvement - algorithm: iql - updates: 100000 - trajectory_sources: [recorded_robot_training] - policy: robot_policy - value: robot_value - critics: robot_q_twins - gamma: 0.99 - expectile: 0.7 - inverse_temperature: 3.0 - max_weight: 100.0 - target_update_rate: 0.005 - - name: simulated_refinement - algorithm: ppo - environment_source: robot - policy: robot_policy - value: robot_value - total_environment_steps: 1048576 - rollout_length: 128 - optimization_epochs: 4 - minibatch_size: 512 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 -``` - -Stage 2 adds named trajectory sources and policy/value/critic registries to the canonical config. Existing model interface definitions are referenced where suitable; avoid embedding unrelated interface authoring syntax in each algorithm config. Model registry validation must catch accidental parameter sharing and incompatible copied interfaces. `rollouts` is a shared named registry consumed by inference, rollout collection, and evaluation. Expose consistent factory resolution relative to the project root/import environment, with explicit error messages for missing modules. - -Commands: - -```text -sequifier rollout --config-path configs/collect.yaml -sequifier preprocess --config-path configs/import-trajectories.yaml -sequifier train --config-path configs/train-return-conditioned.yaml -sequifier train --config-path configs/train-iql-ppo.yaml -sequifier infer --config-path configs/infer-with-dependent-features.yaml -``` - -Extend preprocess with an explicit trajectory import/relabel operation union rather than treating transition metadata as ordinary selected features. A Python API supports the same operations and direct local adapter instances. Update `sequifier docs`, `make` templates, consolidated docs generation, documentation/configs, and release notes. Publish examples for dependent scalar variables, RGB control, offline return conditioning, IQL warm-start, and PPO with local and remote simulators. - -Clarify numerical versus behavioral compatibility: ONNX exports expose policy parameters with action-spec/codec metadata, and the shared runtime performs sampling/scoring. Exported stochastic trajectories need not match PT draw-for-draw across backend RNGs; distribution parameters and deterministic actions must match within tested tolerances. Store ordered categorical decoding dependencies and implement equivalent stepwise ONNX sampling, or reject the particular export at preflight. Never label an exported logits tensor as a complete bounded-action policy without its transforms. - -## 8. Implementation order within the single update - -| Work package | Depends on | Concrete deliverable | -|---|---|---| -| A. Contracts and schema | None | Typed rollout/trajectory/action specs, phase union design, alignment and boundary fixtures | -| B. Coordinator and adapters | A | Dependent-variable inference, tensor/Gym/remote adapters, history/mask handling | -| C. Trajectory data | A/B | Atomic store/import/relabel, episode splits, history-aware loading | -| D. Evaluation integration | B/C | Rollout metrics, monitors, search/pruning support, inference regressions | -| E. Policy primitives | A | Distributions, actor/V/Q wrappers, codecs, log-probability tests, actor exports | -| F. Learner orchestration | C/E | Algorithm runtime, config union, optimizer/checkpoint bundle and migration | -| G. Return conditioning | C/E/F | Return labels, training, inference budget updates and examples | -| H. Offline IQL | C/E/F | Complete learner, targets, weighted actor extraction, resume and evaluation | -| I. PPO | B/D/E/F | Collection buffer, GAE, PPO optimization, DDP and resume semantics | -| J. Integrated release validation | G/H/I | End-to-end examples, numerical/learning/performance checks, docs and migration notes | - -Build contracts and small mathematical fixtures first; add learning tests once the corresponding algorithm exists. Do not combine algorithm validation with heavy simulator installation. Stage gates should be reviewable commits within the same update, with the release gated on J. - -## 9. Verification, performance, and release criteria - -Preserve all existing supported preprocessing, supervised learning, inference, artifact, and export behavior. Run the existing required suite and targeted tests for every changed subsystem. New tests must prove mathematical/boundary behavior rather than mirror private implementation details. - -Maintain three validation tiers: - -1. Fast CPU tests: schema validation, temporal alignment, codec round trips, distributions/gradients, return arithmetic, IQL/PPO/GAE formulas, partial resets, artifact migration, remote failure handling. -2. Integration tests: CLI/Python parity, file/tensor/media sources, actor export, exact/reset-mode resume, training phase transfer, rollout monitors/hyperparameter search, two-process CPU DDP and optional CUDA DDP. -3. Reproducible learning/performance jobs: fixed datasets/tasks/seeds, BC and return-conditioned and IQL comparisons, PPO discrete/continuous/image tasks, local versus remote adapter equivalence, and measured throughput/memory. - -Before optimization, record an agreed baseline on specified hardware with context length, model size, image resolution, simulator count, and DDP topology fixed. Measure policy inference, environment/reward stepping, transfer, serialization, update time, peak memory, and total valid transitions/sec separately. Require no accidental per-field GPU-to-CPU copies in the native tensor collection path and bounded memory over long runs. Set quantitative throughput regression limits after the baseline is measured; do not invent a hardware-independent performance promise. - -The update is complete when all Stage 1/2/3 acceptance criteria pass, the old test suite remains green, the five documented workflows run from clean optional-dependency installations, checkpoints/exports have tested compatibility, and unsupported algorithm/model combinations fail before long collection or training begins. Publish benchmark configuration and results, including seeds and uncertainty, without claiming general robotics SOTA from toy tasks. - -## 10. Algorithm references and interpretation - -- [Decision Transformer](https://arxiv.org/abs/2106.01345): return-conditioned causal action modeling. The proposed multivariate-row representation is a Sequifier-specific implementation choice. -- [Implicit Q-Learning](https://arxiv.org/abs/2110.06169): expectile value learning, Q backups, and advantage-weighted policy extraction. The separate-network, runtime, and storage designs above are proposed integration decisions. -- [Proximal Policy Optimization](https://arxiv.org/abs/1707.06347): clipped policy objective and repeated minibatch optimization of recent rollouts. -- [Generalized Advantage Estimation](https://arxiv.org/abs/1506.02438): value-based advantage estimation. The explicit adapter boundaries, checkpoint contracts, and terminal-observation handling are requirements of this implementation plan. diff --git a/documentation/plans/scoped-ppo-implementation-plan.md b/documentation/plans/scoped-ppo-implementation-plan.md deleted file mode 100644 index 5e524f9a..00000000 --- a/documentation/plans/scoped-ppo-implementation-plan.md +++ /dev/null @@ -1,271 +0,0 @@ -# Scoped PPO implementation plan for the current policy-learning path - -Status: proposed implementation plan, 2026-11-09. This is a follow-up to the -[broader rollout and reinforcement-learning plan](rollout-and-reinforcement-learning-implementation-plan.md), -especially sections 3, 4, 5.1, 5.4, and 6. The target is **online PPO at the -current behavior-cloning and return-conditioned imitation product boundary**: -one scalar-observation, single-categorical-action policy can be trained through -`sequifier train`, evaluated in a local environment, checkpointed, exported as -the ordinary actor model, loaded by `sequifier infer`, and executed through the -same rollout machinery. This does not claim completion of full Stage 3. - -## 1. Baseline and scope - -The diff since `2d01d407cd2134c8197b1a18f0034d01486f18a9` adds opt-in -single-environment scalar forecast rollouts; validated version-1 JSON episode -records; common columnar episode imports from CSV/Parquet, prepared scalar data, -and JSON recordings; causal policy windows; one categorical distributional -actor; BC and return-conditioned imitation; ordinary actor artifacts and -format-v2 imitation run checkpoints; and policy inference/rollouts. The latest -commit adds one fixed-dataset IQL phase with V and twin Q/target Q networks, -transition sampling, weighted actor extraction, an IQL learner checkpoint, and -actor-only export. The IQL phase is single-process and single-categorical; it -does not add an online environment loop or PPO infrastructure. - -The current `collect_rollout` resets one seed-initialized episode, stops at an -episode end or budget, and always closes the adapter. `PolicyExecutor` builds -causal windows and samples actions, but its current execution path returns -domain action labels and records log-probabilities as side metadata. It does not -retain value estimates or a reusable live environment. `PolicyTrainConfig` -requires `dataset_training` and one offline phase; `load_train_config` rejects -PPO. The existing `HistoryCritic` can serve as a separate history-only V, while -the IQL Q networks and offline transition sampler are not PPO components. - -**Included:** one persistent in-process adapter with one environment slot; -scalar real or categorical observation/static fields already supported by the -policy codec; one fixed categorical action vocabulary; fixed-length causal -context; single-process CPU and supported local torch devices; synchronous -collect/update iterations; local deterministic rollout evaluation; explicit -resume behavior; Python and CLI entry points; actor-only portable artifacts. -An unconditioned BC or IQL actor may initialize PPO after strict compatibility -checks. PPO from scratch uses an explicit, frozen observation codec and -normalization declaration, including a deliberate identity-normalization -choice; it never fits statistics from its changing on-policy data. - -**Deferred:** vector/Gym/remote adapters, media/depth, continuous or joint -actions, DDP/FSDP, reward providers and delayed rewards, mixed phases or -same-run phase transfer, generalized rollout monitors/search, ONNX policy -execution, and sharded trajectory storage. Return-conditioned actors are -rejected for this PPO slice; dropping the desired-return input would silently -change the actor contract. Existing BC/return-conditioned/IQL behavior remains -available through their current routes. - -## 2. Contracts to settle first - -1. **Decision timing.** At decision `t`, actor and V read observations through - `o_t` and actions only through `a_(t-1)`. Execute `a_t`, then record its - reward, genuine `o_(t+1)`, `terminated`, and `truncated`. The successor V - window includes `o_(t+1)` and `a_t` as its previous action. Never use an - autoreset observation as that successor. Continue to permit both boundary - flags, with termination taking precedence for bootstrap. -2. **Environment lifecycle.** Add an opt-in online local adapter protocol (or - versioned extension to `EnvironmentAdapter`) whose `reset(seed)` returns the - initial observation and static values, and whose `step(action)` returns the - true successor, scalar finite reward, and boundary flags. One instance lives - from training start through all collect/update iterations and closes in - `finally`. Keep the existing seed-initialized, version-1 inference adapter - callable unchanged; provide a narrow compatibility wrapper where possible. - Reset only after a recorded boundary, before the next decision. A local - adapter may optionally declare and implement exact state snapshots; merely - having `state_dict` methods is insufficient. -3. **History and identity.** Move causal history maintenance into a per-slot - state object shared by `PolicyExecutor`, collection, and evaluation. It - stores the current observation, previous action/start token, static values, - attention mask, episode ID, step ID, and reset count. A collection horizon - cuts a live episode without resetting it. Derive seeds from run seed, - evaluation/training stream, and reset count; do not mix evaluation resets - into training RNG state. Assert that a policy version is constant within a - collected buffer; a live episode may span versions across buffers. -4. **Frozen representation.** Pin the observation/action schema, action-value - order, normalization statistics, context length, and deterministic input - transforms before the first reset. Use domain values at the adapter edge and - the existing codec for model tensors. Reject schema/shape/unit drift and - non-finite values before insertion. Disable dropout and stochastic - augmentation for collection and PPO scoring while retaining gradients in - scoring. Make `evaluate_actions` and `act` use FP32 probability math. -5. **Rewards and boundaries.** This slice optimizes the adapter's one scalar - reward with a fixed, checkpointed scale (default 1). Reject provisional or - delayed rewards. A true terminal has no bootstrap; a time-limit truncation - bootstraps only from its genuine final state; a collection cut is a fragment - and bootstraps without a reset. Reject a requested bootstrap when the true - successor is unavailable. - -**Gate A:** hand-authored resets, first/interior/final decisions, truncations, -cuts, and episode restarts prove alignment. Replacing current action, reward, -or future observations cannot change step-`t` actor/V outputs. Existing -seed-based rollout behavior and version-1 JSON loading still work. - -## 3. Dependency-ordered implementation packages - -### B. Persistent collection and on-policy data - -Build a `PPOCollector` over the shared history/adapter contract. At each -iteration freeze an actor/value snapshot and collect exactly `rollout_length` -decisions, except a declared final shorter iteration when `total_env_steps` is -not divisible by the horizon. The budget counts executed actions, including -terminal actions. A completed episode resets immediately for the next decision; -a live episode and its history continue into the next iteration. Collection -runs under `no_grad` and records one policy version for every decision. - -Add a sealed, bounded `PPORolloutBuffer` with decision references to a compact -observation/action history ring rather than a copied context window per step. -Each entry holds domain/encoded action, old joint log-probability, old V, -reward, true successor/final observation reference, termination/truncation, -bootstrap eligibility, episode/step identity, and policy version. Retain the -pre-buffer context needed to reconstruct the earliest decision window. Store -old V of each true successor or calculate it from the frozen collection value -network before the buffer is sealed. Memory use is bounded by the authored -horizon and context; estimate required memory before collection. Random -minibatch access reconstructs complete causal windows and preserves their -within-window time order. The buffer is immutable after target calculation and -cannot be reused for a later policy generation. - -Keep optional trajectory archival separate from the training buffer. Extend -the JSON episode/fragment record or introduce a small versioned collection -record so fragments retain preceding action/history, policy version, cutoff -reason, old log-probability/value when requested, and true final successor. -Write completed fragments atomically; preserve readers for existing version-1 -records. Archival is never silently fed back into an on-policy update. - -**Gate B:** collect across multiple short episodes and across a live episode -cut; verify exact executed-step budget, terminal reward, no lost/duplicated -actions, bounded storage, reproducible sampled actions, and reconstructed -windows equal collection windows before optimization. A buffer from an older -policy version is rejected. - -### C. Value model, GAE, and PPO math - -Reuse `CategoricalPolicy` and `CategoricalActionDistribution` for the actor. -Instantiate an independent history-only `HistoryCritic` for V; do not create -IQL's Q/target-Q networks. Validate disjoint actor/V trainable parameter IDs -and identical frozen observation codecs. Provide a single decision API that -returns domain action, encoded action, old log-probability, entropy, and V -from the same causal input. Re-scoring that exact action before the first -optimizer step must reproduce old log-probability and V within tolerance. - -Compute GAE in FP32 over the sealed buffer. With `b_t` meaning bootstrap is -allowed and `c_t` meaning the next stored decision is in the same episode and -same collection segment: - -```text -delta_t = reward_t + gamma * b_t * V_old(h_next) - V_old(h_t) -A_t = delta_t + gamma * lambda * c_t * A_(t+1) -target_t = stop_gradient(A_t + V_old(h_t)) -``` - -Set `(b,c)` to `(0,0)` at true termination, `(1,0)` at a time-limit truncation -with valid final observation, `(1,0)` at a live collection cut, and `(1,1)` -only for an ordinary transition followed by the next decision of this episode -within the buffer. Normalize advantages once over valid decisions for an -iteration when configured, never per minibatch. Keep old log-probabilities, -old values, advantages, and targets fixed for all optimization epochs. - -Implement clipped policy loss, independent value loss, and current-policy -categorical entropy. Value clipping around `V_old` is optional and off by -default. Calculate ratio from joint `logp_new - logp_old`; check it and all -losses for non-finite values. Implement optional target-KL early stopping and -gradient clipping. Use independent actor/V optimizers and schedulers with -explicit per-optimizer-step clocks. Correctly weight a smaller last minibatch -and gradient accumulation by valid decision count. Never construct a fake -supervised target or pass PPO loss through supervised `LossService`. - -**Gate C:** hand-computed positive/negative advantage clipping, value clipping, -entropy, KL, and GAE match reference values. Gradient ownership is disjoint. -The identity ratio is one before any update, including after a checkpoint -round trip. A one-iteration run has finite losses and actually changes actor -and V weights. - -### D. Online phase, execution, and evaluation - -Add a discriminated `PPOPhaseConfig` and an online environment source to -`config/training_plan.py`/`config/train_config.py`. Route it through -`load_train_config`, `train.py`, `rl/learner.py`, and a `rl/ppo.py` algorithm -runtime. It must not require `dataset_training`, offline `sources`, `epochs`, -validation loaders, or fabricated `global_training` batch counts. Preserve -legacy supervised and imitation defaults. For this slice require exactly one -PPO phase; mixed-phase scheduling remains a separate parent-plan item. - -The typed PPO schema includes `total_env_steps`, `rollout_length`, -`optimization_epochs`, minibatch and accumulation sizes, gamma, GAE lambda, -policy clip epsilon, optional value clip, value/entropy coefficients, optional -target KL, advantage normalization, actor/V optimizer and scheduler plans, -learning rates, gradient limits, evaluation/checkpoint intervals, adapter -factory/config, separate evaluation adapter factory/config, reset/seed policy, -normalization source, and resume mode. Validate ranges, field ownership, -unsupported distributions/devices/distribution modes, budget/remainder rules, -paths, and parameter ownership before a long collection starts. An optional -`initialize_from` actor must match schema, vocab, normalization, model -architecture, and context length exactly. Do not treat this as exact resume. - -Run evaluation on a separate local adapter instance with fixed seeds and a -frozen actor snapshot. Default to deterministic mode actions; allow explicitly -named stochastic evaluation. Report mean episode return, spread, lengths, -completed/censored counts, and success only when the adapter defines it. -Select the exported best actor by a declared evaluation metric/direction, -defaulting to mean completed-episode return; never select by PPO loss. If no -episode completes, report that the selection metric is unavailable rather than -treating censored return as complete. The evaluation budget and timeout must be -explicit so a nonterminating task cannot hang training. Expose the same -evaluator to Python callers and to the PPO CLI workflow; this does not yet -require generalized supervised/rollout monitor or -hyperparameter-search integration. - -**Gate D:** a clean PPO CLI run can start from a reset-only local environment, -evaluate without changing training RNG/history, choose a best actor by return, -write a report, and load that actor through the existing policy inference and -single-environment rollout APIs. Existing BC, return-conditioned, and IQL -configs still parse and dispatch as before. - -### E. Checkpoint, resume, and operational validation - -Write an algorithm-owned versioned PPO checkpoint at complete iteration -boundaries. Include actor/V specs and weights; both optimizer/scheduler states; -global environment steps, iteration and policy version; best actor/metric; -codec/statistics and config fingerprints; actor initialization provenance; -training/evaluation seed streams and all torch RNG states; and the live -history/episode counters. Checkpoint writes are atomic. Do not put V or -optimizers in the portable `sequifier_model` actor artifact. Keep reading the -existing supervised format-v2 and IQL checkpoints through their own routes. - -Support two declared resume modes. `exact` requires an adapter capability -declaration and successfully restored simulator/RNG state plus history; an -unsupported adapter fails preflight. `reset_environments` restores learner -state but resets the adapter and causal history, drops incomplete-episode -metrics, and marks the continuation as non-trajectory-equivalent. For a crash -inside collection or optimization, restart from the last committed iteration; -do not pretend the partial buffer was resumed. Validate incompatible config, -action codec, model, reward scale, or adapter identity before restore. - -Add a structured training report with counts, return/evaluation curves, -policy/value loss, entropy, clip fraction, approximate KL, explained variance, -gradient norms, ratio tails, invalid/censored episode counts, and separate -collection/update/evaluation timing. Add a minimal deterministic categorical -control adapter and PPO config to `documentation/example_inputs`, a `sequifier -make` template, Python API example, policy guide, config docs, consolidated -docs, and release notes. Document the exact scoped limitations and the two -resume semantics. - -**Gate E / done for this slice:** the fixed-seed control task improves over -its random policy over multiple seeds; a short deterministic CPU job stays in -CI. A clean train → evaluate → checkpoint → reload → infer → rollout workflow -passes. Interrupted versus uninterrupted exact-snapshot runs match within -device-appropriate tolerance; non-snapshot adapters reject exact resume and -continue in reset mode. Run the existing repository and external test suite -where available, and test the old supervised/BC/return-conditioned/IQL paths. -Record seeds, task rules, hardware, context length, model size, throughput, -and peak memory. Do not mark the broad parent plan complete from this slice. - -## 4. Implementation order and review boundaries - -| Package | Depends on | Reviewable result | -|---|---|---| -| A. Online contracts and causal state | Existing policy/rollout path | Reset/step/boundary fixtures, shared history state, frozen codec | -| B. Collector and buffer | A | Persistent single-slot collection, exact windows, versioned fragments | -| C. Value, GAE, PPO update | B | Independent V, numerical formulas, identity ratio, optimizer transactions | -| D. Config and evaluation | A–C | One online PPO phase, CLI/Python parity, return-based best actor | -| E. Checkpoints and validation | A–D | Resume modes, artifact/export, learning and regression evidence | - -The parent plan's work packages B/D/E/F/I/J remain broader than these scoped -packages. In particular, passing the gates above adds a usable local -categorical PPO path; it does not establish vector/remote/multimodal or -distributed PPO acceptance. diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 0635ab19..240622d1 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -17,8 +17,8 @@ vectorized environments, and reward models are outside this contract. ## Authoring a run Start with the [multi-phase example](example_inputs/configs/train-multiphase-control.yaml), -[single-phase DPO](example_inputs/configs/train-dpo-control.yaml), or -[single-phase GRPO](example_inputs/configs/train-grpo-control.yaml). +[single-phase DPO](../src/sequifier/templates/train-dpo.yaml), or +[single-phase GRPO](../src/sequifier/templates/train-grpo.yaml). Generate their curated data with: ```shell @@ -112,19 +112,30 @@ update always evaluates. Missing finite evaluation for `best` fails explicitly. export uses the last phase's selected actor, with metric, transaction and weight hash recorded; metrics from different phases are never compared globally. -Imitation, DPO and GRPO share these actor optimization fields: +Each phase configures trainable roles under `optimization`. Imitation, DPO and +GRPO have `actor`; PPO has `actor` and `value`; IQL has `actor`, `value` and +`critic` (the twin Q networks share that optimizer). Omitted roles use defaults. +For example: + +```yaml +optimization: + actor: {learning_rate: 0.005, optimizer: {name: AdamW}, gradient_clip: 1.0} +``` + +Every role defaults to `learning_rate: 0.0003`, `optimizer: {name: Adam}`, +`scheduler: {name: ConstantLR, factor: 1.0, total_iters: 1}`, and +`gradient_clip: 1.0`. Unknown roles fail validation. + +Imitation, DPO and GRPO share these batching and scheduling fields: | Field | Default | | --- | --- | | `batch_size`, `accumulation_steps` | 32, 1; DPO counts pairs, GRPO counts whole trajectories | -| `actor_learning_rate`, `actor_optimizer` | 0.0003, `{name: Adam}` | -| `actor_scheduler` | `{name: ConstantLR, factor: 1.0, total_iters: 1}` | -| `gradient_clip` | 1.0 | | `evaluation_interval`, `checkpoint_interval` | 1; optimizer updates for imitation/DPO, completed collection/optimization updates for GRPO | Optimizers support Adam, AdamW, SGD and RMSprop. Their constructor arguments belong alongside `name` (or in `arguments`); `lr` and parameters belong to the -phase. Schedulers advance on completed optimizer steps; plateau/compound +role configuration. Schedulers advance on completed optimizer steps; plateau/compound schedulers are unsupported. `OneCycleLR.total_steps` must equal the authored maximum optimizer-step budget. Accumulation weights by the actual number of examples, including final partial batches. Network stochastic layers are disabled @@ -170,10 +181,11 @@ Each update samples transitions uniformly with replacement. Held-out unweighted `validation_action_nll` is the minimizing selection monitor; Bellman residual, values, Q and weight diagnostics are reported separately. -`actor_`, `value_`, and `critic_learning_rate` default to 0.0003; each role has -its own optimizer/scheduler. `gradient_clip` defaults to 1.0, evaluation and -checkpoint intervals to 100 updates. Critics are fresh even after another IQL -phase and are absent from the portable actor. +The `optimization.actor`, `optimization.value` and `optimization.critic` roles +have independent optimizers and schedulers. All gradients are validated and +clipped once before any optimizer steps. Evaluation and checkpoint intervals +default to 100 updates. Critics are fresh even after another IQL phase and are +absent from the portable actor. ## DPO preferences @@ -235,7 +247,7 @@ a still-present artifact with changed weights is rejected on resume. ## Online adapters -PPO and GRPO share the version-3 `OnlineEnvironmentAdapter` contract. Configure +PPO, GRPO, evaluation and policy rollout share the version-3 `OnlineEnvironmentAdapter` contract. Configure `environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. Factories must create fresh independent adapters for evaluation. An adapter explicitly declares: @@ -278,9 +290,14 @@ establish this capability. Exact GRPO continuation additionally requires the adapter's documented `reproducible_group_start` guarantee. The bundled `CategoricalControlAdapter` demonstrates a deterministic scenario token. -`SeededAdapterBridge` can adapt an existing seeded forecast-rollout adapter with -an explicit initial-state factory; it declares no group-start or exact-snapshot -capability. Forecast rollout bindings keep their separate version-1 contract. +Policy inference from supplied observation rows additionally requires +`EnvironmentCapabilities(seeded_reset=True)` and +`reset_from_observation(initial: OnlineResetResult, seed: int) -> OnlineResetResult`. +It must initialize the simulator without executing an action and return exactly +the supplied observation, static inputs and units. This is distinct from the +ordinary `reset(seed)` used during training. The bundled control adapter +implements both. Forecast rollout bindings retain their separate version-1 +contract. ## Online categorical PPO @@ -298,8 +315,8 @@ Defaults: `optimization_epochs: 4`, `minibatch_size: 64`, `accumulation_steps: 1 `value_coefficient: 0.5`, `entropy_coefficient: 0.01`, `normalize_advantages: true`, `reward_scale: 1`. Optional positive `target_kl` stops optimization against the old collection policy; `value_clip` is optional. -`actor_learning_rate`/`value_learning_rate` default to 0.0003, with independent -optimizers/schedulers and `actor_gradient_clip`/`value_gradient_clip` of 1.0. +The `optimization.actor` and `optimization.value` roles have independent +optimizers/schedulers and the shared learning-rate and clipping defaults. `allow_partial_rollout: true` executes exactly the remaining action budget; false requires divisibility. `max_buffer_bytes` defaults to 536870912. `archive_directory` optionally writes separate versioned collection records; @@ -381,32 +398,29 @@ conflicts are checked separately before output mutation and at phase boundaries. Prepared data supplies its fitting view and frozen-representation validation; algorithm-specific transition checks remain with the IQL dataset. -The only policy-training checkpoint is `sequifier_policy_run`, format version 4. -Completed phase results determine the phase boundary; an optional live phase -state records the latest committed transaction. Only the incoming actor and -current phase's recovery tensors are embedded; earlier selected actors live in -their phase artifacts rather than being serialized with every checkpoint. -It contains the resolved plan, immutable representation and its fingerprint, data -and adapter identities, run RNG, completed results, phase manifest paths and -checksums, and optional versioned phase state. DPO/GRPO reference weights are -embedded in live phase state. Optimizer, scheduler, sampler, selection, evaluation -and phase counters remain phase-local. Only version-4 checkpoints are supported. - -Writes use a temporary file and atomic replacement at completed updates or -handoffs. No partial group is checkpointable. Handoffs flush the selected -`actor.pt`, then `report.json`, then a small `manifest.json` recording their SHA-256 -checksums, the selected weight hash, and result/representation fingerprints. Only -then is the run checkpoint committed with the manifest identity and incoming -actor. The checkpoint is the authoritative phase boundary: exports left by an -interrupted, uncommitted handoff can be reused if valid or replaced on retry. -A committed handoff advances the phase index exactly once; resume never reruns -completed phases or reselects their actor. - -Completed phase artifacts and manifests must be retained alongside the checkpoint -for recovery; the checkpoint alone is no longer self-contained. Resume verifies -their identities and fails on missing or modified files before output mutation, -without loading or rewriting earlier models. Representation and -data/adapter/reference mismatches also fail before output mutation. +The policy-training checkpoint is `sequifier_policy_run`, format version 5. +Completed results determine the phase boundary; optional version-2 phase state +records the latest committed transaction. It contains the resolved plan, frozen +representation, source/partition and adapter identities, run RNG, completed +results and phase actor paths/checksums. Live phase state contains role-keyed +networks and optimizers/schedulers/step counts, sampler, selection, metrics and +algorithm-specific progress. DPO/GRPO references are embedded. Earlier selected +actors remain in their phase artifacts. Earlier unreleased checkpoints are not +supported. + +Writes use temporary files, flush them, and atomically replace their destinations. +At a handoff, the selected `actor.pt` is published with its result metadata, then +the run checkpoint commits its checksum and result. The checkpoint is the +authoritative phase boundary. An interrupted, uncommitted export is replaced on +retry; a committed handoff advances exactly once. Completed phases never rerun +or reselect their actor. + +Retain phase actor artifacts alongside the checkpoint. Resume checks their +checksums and result identities before output mutation, without loading or +rewriting earlier models. Phase `report.json` files are derived from committed +results and regenerated on resume; they are not recovery dependencies. No +separate phase manifest is needed. Representation and data/adapter/reference +mismatches still fail before output mutation. `resume.mode: exact` requires validated PPO simulator/RNG snapshots or GRPO's reproducible group-start capability and matching implementation identity. PPO @@ -444,8 +458,13 @@ paths. `rl_api` also exports the phase schemas, preference records, online capabilities, `PolicyWindowView`, and model loading/execution helpers. Use `sequifier infer` with the final artifact for categorical sequence inference -or rollout. `PolicyExecutor` and `collect_policy_rollout` build the same causal -histories; return-conditioned actors require an explicit desired return. +or rollout. `collect_policy_rollout` uses the same causal `PolicyHistory` and +`decide` path as online training and evaluation. Return-conditioned actors require +an explicit desired return. `PolicyRolloutConfig` declares `environment`, +`trajectory_path`, `seed_index` and optional `max_episode_steps`; the environment +must support seeded reset. Successors always come from the environment, so no +prediction bindings are configured. A true terminal may omit its successor; +truncation and collection cutoffs must retain it. See [policy inference](configs/infer.md), [rollout](rollout.md), and the [Python PPO workflow](example_inputs/ppo_toy_control.py). diff --git a/documentation/rollout.md b/documentation/rollout.md index 1f0181b3..93af3d0a 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -1,7 +1,7 @@ # Local numeric forecast rollouts For trajectory-backed categorical policy training and return-conditioned -execution through the same coordinator, see [policy learning](policy-learning.md). +execution through the online environment protocol, see [policy learning](policy-learning.md). The first rollout slice collects one seed-initialized episode or episode fragment on CPU. It supports real scalar features and real static conditioning, with diff --git a/pyproject.toml b/pyproject.toml index 311a61eb..86049ea8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ Repository = "https://github.com/0xideas/sequifier" sequifier = "sequifier.sequifier:main" [tool.setuptools.package-data] -sequifier = ["consolidated-docs.md"] +sequifier = ["consolidated-docs.md", "templates/*.yaml"] [project.optional-dependencies] images = [ diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index 428391d7..b07cde53 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -12,22 +12,22 @@ - GRPO adds explicit reproducible group-start capabilities, independent RNG streams, bounded complete groups, population-normalized advantages, clipped ratios and a separately named sampled frozen-reference penalty, without a critic. -- Version-3 local online adapters serve PPO and GRPO. Exact continuation requires +- Version-3 local online adapters serve PPO, GRPO and policy rollout. Explicit + seeded reset preserves supplied initial observations during inference. Exact continuation requires validated snapshots or documented reproducible group starts; non-exact restarts report a discontinuity. Failed/partial groups are never reused. -- One atomic version-3 `sequifier_policy_run` checkpoint embeds phase references, selected - actors, source/representation hashes, sampler/RNG state and committed progress. - Actor state dictionaries are stored directly; unreleased version-2 checkpoints - are not supported. - The former unreleased single-algorithm configuration/checkpoint paths are removed. +- One atomic version-5 `sequifier_policy_run` checkpoint owns phase progress, + role-keyed optimizer/network state, references, source identities and RNG. + Phase actors are durable exports referenced by checksum; reports are derived + and regenerated on resume. No phase manifests or old-checkpoint migration. +- Shared phase sessions handle selection, checkpoints and optimization. Each + phase configures optimizer roles under `optimization`; gradient validation + and clipping precede all optimizer steps in a transaction. - Actor-only portable artifacts remain usable for inference, rollout and cross-run initialization. Default output is `models//actor.pt`; phase reports and selected actors use indexed directories. - Templates, examples, public exports and guides use the canonical schema. The pre-existing supervised training runtime remains independent. -- Behavioral tests cover portable actors, partition sharing, all five algorithms' - interrupted continuation, phase handoff, and artifact republication. Offline - continuation also covers accumulation, epoch boundaries, and sparse evaluation. Scope: one categorical action, scalar/categorical inputs, causal fixed histories, local single-process FP32. Return conditioning supports imitation only. Text/LLM, diff --git a/src/sequifier/artifacts/backbone_repository.py b/src/sequifier/artifacts/backbone_repository.py index 19b48c3c..8ae40fc8 100644 --- a/src/sequifier/artifacts/backbone_repository.py +++ b/src/sequifier/artifacts/backbone_repository.py @@ -1,7 +1,6 @@ import contextlib import hashlib import json -import os import uuid from collections.abc import Iterator from datetime import datetime, timezone @@ -11,6 +10,7 @@ import torch from torch import Tensor, nn +from sequifier.io.atomic import atomic_save, write_json_atomic from sequifier.typechecking import beartype ARTIFACT_TYPE = "sequifier_backbone" @@ -75,22 +75,6 @@ def _read_json(path: Path) -> dict[str, Any]: return value -@beartype -def _write_json_atomic(path: Path, value: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - with temporary_path.open("w") as file: - json.dump(value, file, sort_keys=True, indent=2) - file.write("\n") - file.flush() - os.fsync(file.fileno()) - os.replace(temporary_path, path) - finally: - with contextlib.suppress(OSError): - temporary_path.unlink() - - @beartype def latest_pointer(repository_path: Path) -> dict[str, Any] | None: path = repository_path / "latest.json" @@ -278,18 +262,12 @@ def publish_revision( revision_path = revisions_path / revision_id revision_path.mkdir(exist_ok=False) weights_path = revision_path / "weights.pt" - temporary_weights_path = revision_path / f".weights.{uuid.uuid4().hex}.tmp" state_dict = { key: value.detach().cpu().clone() for key, value in backbone.state_dict().items() } _validate_state_dict(backbone, state_dict) - try: - torch.save(state_dict, temporary_weights_path) - os.replace(temporary_weights_path, weights_path) - finally: - with contextlib.suppress(OSError): - temporary_weights_path.unlink() + atomic_save(weights_path, state_dict) architecture = canonical_architecture(backbone_config.architecture) manifest = { @@ -303,8 +281,8 @@ def publish_revision( "source_run_id": source_run_id, "source_epoch": source_epoch, } - _write_json_atomic(revision_path / "manifest.json", manifest) - _write_json_atomic( + write_json_atomic(revision_path / "manifest.json", manifest) + write_json_atomic( repository_path / "latest.json", { "revision_id": revision_id, diff --git a/src/sequifier/artifacts/manifests.py b/src/sequifier/artifacts/manifests.py index e3786629..890f67fb 100644 --- a/src/sequifier/artifacts/manifests.py +++ b/src/sequifier/artifacts/manifests.py @@ -1,25 +1,10 @@ -import contextlib -import json -import os -import uuid from pathlib import Path from typing import Any +from sequifier.io.atomic import write_json_atomic from sequifier.typechecking import beartype @beartype def write_manifest(path: str | Path, manifest: dict[str, Any]) -> None: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - temporary_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - with temporary_path.open("w") as file: - json.dump(manifest, file, indent=2, sort_keys=True, default=str) - file.write("\n") - file.flush() - os.fsync(file.fileno()) - os.replace(temporary_path, path) - finally: - with contextlib.suppress(OSError): - temporary_path.unlink() + write_json_atomic(path, manifest, sort_keys=True, default=str, allow_nan=True) diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index 240e54e2..e0d01349 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -2,9 +2,6 @@ from __future__ import annotations -import contextlib -import os -import uuid from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any, Mapping @@ -18,6 +15,7 @@ canonicalize_state_dict, validate_model_state_contract, ) +from sequifier.io.atomic import atomic_save from sequifier.model.factory import build_transformer_network MODEL_ARTIFACT_FORMAT_VERSION = 2 @@ -119,14 +117,7 @@ def build_model_artifact( def save_model_artifact(artifact: ModelArtifact, path: str | Path) -> Path: destination = Path(path).expanduser().resolve() - destination.parent.mkdir(parents=True, exist_ok=True) - temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp") - try: - torch.save(artifact.state_dict(), temporary) - os.replace(temporary, destination) - finally: - with contextlib.suppress(OSError): - os.remove(temporary) + atomic_save(destination, artifact.state_dict()) return destination diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 745daf66..48e64e5d 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -2,9 +2,6 @@ from __future__ import annotations -import contextlib -import os -import uuid from collections.abc import Mapping from dataclasses import asdict, dataclass from pathlib import Path @@ -13,6 +10,7 @@ import torch from sequifier.artifacts.model_artifact import ModelArtifact +from sequifier.io.atomic import atomic_save RUN_CHECKPOINT_FORMAT_VERSION = 2 @@ -142,25 +140,9 @@ def write_run_checkpoint( checkpoint: RunCheckpoint, output_path: Path, latest_path: Path ) -> None: payload = checkpoint.state_dict() - temporary = output_path.with_name(f".{output_path.name}.{uuid.uuid4().hex}.tmp") - try: - torch.save(payload, temporary) - os.replace(temporary, output_path) - finally: - with contextlib.suppress(OSError): - os.remove(temporary) - if output_path == latest_path: - return - latest_path.parent.mkdir(parents=True, exist_ok=True) - latest_temporary = latest_path.with_name( - f".{latest_path.name}.{uuid.uuid4().hex}.tmp" - ) - try: - torch.save(payload, latest_temporary) - os.replace(latest_temporary, latest_path) - finally: - with contextlib.suppress(OSError): - os.remove(latest_temporary) + atomic_save(output_path, payload) + if output_path != latest_path: + atomic_save(latest_path, payload) def checkpoint_path(training_config: Any) -> Path: diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 2e90d33e..d5c0c219 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -1023,8 +1023,6 @@ class ResolvedInferenceConfig(_InferenceConfigBase[str, list[str], dict[str, str def validate_numeric_rollout(self): if self.rollout is None: return self - if self.rollout.mode != "forecast": - raise ValueError("rollout.mode: policy requires a policy model interface") if not self.autoregressive or self.generation_steps is None: raise ValueError( "rollout requires autoregressive: true and generation_steps" diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 53a7e017..fde452be 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -6,10 +6,17 @@ from sequifier.artifacts.model_artifact import ModelArtifact, read_execution_artifact from sequifier.config.policy_config import PolicyConfigBase -from sequifier.config.rollout_config import RolloutConfig +from sequifier.config.policy_plan import OnlineEnvironmentConfig from sequifier.helpers import normalize_path +class PolicyRolloutConfig(PolicyConfigBase): + environment: OnlineEnvironmentConfig + trajectory_path: str = Field(min_length=1) + seed_index: int = Field(ge=0) + max_episode_steps: int | None = Field(default=None, gt=0) + + class PolicyInferenceConfig(PolicyConfigBase): _artifacts: dict[str, ModelArtifact] = PrivateAttr(default_factory=dict) @@ -31,15 +38,13 @@ class PolicyInferenceConfig(PolicyConfigBase): previous_action_column: str | None = None sequence_id_column: str = "sequenceId" item_position_column: str = "itemPosition" - rollout: RolloutConfig | None = None + rollout: PolicyRolloutConfig | None = None autoregressive: bool = False generation_steps: int | None = Field(default=None, gt=0) @model_validator(mode="after") def validate_execution(self): if self.rollout is not None: - if self.rollout.mode != "policy": - raise ValueError("policy artifacts require rollout.mode: policy") if not self.autoregressive or self.generation_steps is None: raise ValueError( "policy rollout requires autoregressive: true and generation_steps" diff --git a/src/sequifier/config/policy_optimization.py b/src/sequifier/config/policy_optimization.py index 73bee2ce..6fa6537f 100644 --- a/src/sequifier/config/policy_optimization.py +++ b/src/sequifier/config/policy_optimization.py @@ -2,6 +2,8 @@ from dataclasses import dataclass +from pydantic import BaseModel, ConfigDict, Field + from sequifier.config.components import ComponentSpec from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class @@ -14,6 +16,25 @@ def default_scheduler(): return ComponentSpec(name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1}) +class OptimizationConfig(BaseModel): + """One authored optimizer/scheduler configuration per trainable role.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + optimizer: ComponentSpec = Field(default_factory=default_optimizer) + scheduler: ComponentSpec = Field(default_factory=default_scheduler) + learning_rate: float = Field(default=3e-4, gt=0) + gradient_clip: float = Field(default=1.0, gt=0) + + def resolve(self, step_budget): + return OptimizationPlan( + self.optimizer, + self.scheduler, + self.learning_rate, + self.gradient_clip, + step_budget, + ) + + @dataclass(frozen=True) class OptimizationPlan: optimizer: ComponentSpec diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index fb55c16c..7b6fb08a 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -1,18 +1,13 @@ """Built-in policy phase schemas; budgets and objectives are algorithm-specific.""" import math -from typing import Annotated, Literal +from typing import Annotated, ClassVar, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from sequifier.config.components import ComponentSpec from sequifier.config.identifiers import identifier from sequifier.config.policy_config import CategoricalPolicyConfig -from sequifier.config.policy_optimization import ( - OptimizationPlan, - default_optimizer, - default_scheduler, -) +from sequifier.config.policy_optimization import OptimizationConfig class PhaseConfig(BaseModel): @@ -20,12 +15,23 @@ class PhaseConfig(BaseModel): name: str handoff: Literal["best", "last"] = "best" representation: CategoricalPolicyConfig | None = None + optimization_roles: ClassVar[tuple[str, ...]] = ("actor",) + optimization: dict[str, OptimizationConfig] = Field(default_factory=dict) + evaluation_interval: int = Field(default=1, gt=0) + checkpoint_interval: int = Field(default=1, gt=0) def optimizer_step_budget(self, training_size=None): raise NotImplementedError("built-in phase must declare its optimizer budget") def optimization_plans(self, training_size=None): - raise NotImplementedError("built-in phase must declare optimization roles") + unknown = self.optimization.keys() - set(self.optimization_roles) + if unknown: + raise ValueError(f"unsupported optimization roles: {sorted(unknown)}") + budget = self.optimizer_step_budget(training_size) + return { + role: self.optimization.get(role, OptimizationConfig()).resolve(budget) + for role in self.optimization_roles + } @model_validator(mode="after") def validate_optimization(self): @@ -38,27 +44,9 @@ def phase_name(cls, value): return identifier(value, "Policy phase name") -class ActorOptimizationConfig(PhaseConfig): +class BatchedPhaseConfig(PhaseConfig): batch_size: int = Field(default=32, gt=0) accumulation_steps: int = Field(default=1, gt=0) - actor_learning_rate: float = Field(default=3e-4, gt=0) - actor_optimizer: ComponentSpec = Field(default_factory=default_optimizer) - actor_scheduler: ComponentSpec = Field(default_factory=default_scheduler) - gradient_clip: float = Field(default=1.0, gt=0) - evaluation_interval: int = Field(default=1, gt=0) - checkpoint_interval: int = Field(default=1, gt=0) - - def optimization_plans(self, training_size=None): - budget = self.optimizer_step_budget(training_size) - return { - "actor": OptimizationPlan( - self.actor_optimizer, - self.actor_scheduler, - self.actor_learning_rate, - self.gradient_clip, - budget, - ) - } class OfflinePhaseConfig(PhaseConfig): @@ -66,7 +54,7 @@ class OfflinePhaseConfig(PhaseConfig): validation_sources: list[str] | None = Field(default=None, min_length=1) -class PolicyImitationPhaseConfig(ActorOptimizationConfig, OfflinePhaseConfig): +class PolicyImitationPhaseConfig(BatchedPhaseConfig, OfflinePhaseConfig): algorithm: Literal["policy_imitation"] epochs: int = Field(gt=0) monitor: Literal["validation_action_nll"] = "validation_action_nll" @@ -80,7 +68,7 @@ def optimizer_step_budget(self, training_size=None): class IQLPhaseConfig(OfflinePhaseConfig): - model_config = ConfigDict(extra="forbid") + optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value", "critic") monitor: Literal["validation_action_nll"] = "validation_action_nll" direction: Literal["min"] = "min" algorithm: Literal["iql"] @@ -92,48 +80,12 @@ class IQLPhaseConfig(OfflinePhaseConfig): inverse_temperature: float = Field(default=3.0, gt=0, allow_inf_nan=False) max_weight: float = Field(default=100.0, ge=1, allow_inf_nan=False) target_update_rate: float = Field(default=0.005, gt=0, le=1, allow_inf_nan=False) - actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - critic_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - actor_optimizer: ComponentSpec = Field(default_factory=default_optimizer) - value_optimizer: ComponentSpec = Field(default_factory=default_optimizer) - critic_optimizer: ComponentSpec = Field(default_factory=default_optimizer) - actor_scheduler: ComponentSpec = Field(default_factory=default_scheduler) - value_scheduler: ComponentSpec = Field(default_factory=default_scheduler) - critic_scheduler: ComponentSpec = Field(default_factory=default_scheduler) - gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) evaluation_interval: int = Field(default=100, gt=0) checkpoint_interval: int = Field(default=100, gt=0) def optimizer_step_budget(self, training_size=None): return self.updates - def optimization_plans(self, training_size=None): - budget = self.optimizer_step_budget(training_size) - return { - "actor": OptimizationPlan( - self.actor_optimizer, - self.actor_scheduler, - self.actor_learning_rate, - self.gradient_clip, - budget, - ), - "value": OptimizationPlan( - self.value_optimizer, - self.value_scheduler, - self.value_learning_rate, - self.gradient_clip, - budget, - ), - "critic": OptimizationPlan( - self.critic_optimizer, - self.critic_scheduler, - self.critic_learning_rate, - self.gradient_clip, - budget, - ), - } - class OnlineEnvironmentConfig(BaseModel): model_config = ConfigDict(extra="forbid") @@ -204,7 +156,7 @@ class PolicyResumeConfig(BaseModel): class PPOPhaseConfig(PhaseConfig): - model_config = ConfigDict(extra="forbid") + optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value") algorithm: Literal["ppo"] name: str = Field(default="online_ppo", min_length=1) environment: OnlineEnvironmentConfig @@ -226,16 +178,6 @@ class PPOPhaseConfig(PhaseConfig): target_kl: float | None = Field(default=None, gt=0, allow_inf_nan=False) normalize_advantages: bool = True reward_scale: float = Field(default=1.0, gt=0, allow_inf_nan=False) - actor_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - value_learning_rate: float = Field(default=3e-4, gt=0, allow_inf_nan=False) - actor_optimizer: ComponentSpec = Field(default_factory=default_optimizer) - value_optimizer: ComponentSpec = Field(default_factory=default_optimizer) - actor_scheduler: ComponentSpec = Field(default_factory=default_scheduler) - value_scheduler: ComponentSpec = Field(default_factory=default_scheduler) - actor_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) - value_gradient_clip: float = Field(default=1.0, gt=0, allow_inf_nan=False) - evaluation_interval: int = Field(default=1, gt=0) - checkpoint_interval: int = Field(default=1, gt=0) archive_directory: str | None = Field(default=None, min_length=1) archive_scores: bool = False max_buffer_bytes: int = Field(default=536870912, gt=0) @@ -259,25 +201,6 @@ def validate_plans(self): ) return self - def optimization_plans(self, training_size=None): - budget = self.optimizer_step_budget(training_size) - return { - "actor": OptimizationPlan( - self.actor_optimizer, - self.actor_scheduler, - self.actor_learning_rate, - self.actor_gradient_clip, - budget, - ), - "value": OptimizationPlan( - self.value_optimizer, - self.value_scheduler, - self.value_learning_rate, - self.value_gradient_clip, - budget, - ), - } - class ReferenceConfig(BaseModel): model_config = ConfigDict(extra="forbid") @@ -291,7 +214,7 @@ def location(self): return self -class DPOPhaseConfig(ActorOptimizationConfig): +class DPOPhaseConfig(BatchedPhaseConfig): algorithm: Literal["dpo"] preferences: str validation_preferences: str | None = None @@ -311,7 +234,7 @@ class GRPOEvaluationConfig(EvaluationEpisodesConfig): direction: Literal["max"] = "max" -class GRPOPhaseConfig(ActorOptimizationConfig): +class GRPOPhaseConfig(BatchedPhaseConfig): scoring_chunk_size: int = Field(default=64, gt=0) algorithm: Literal["grpo"] environment: OnlineEnvironmentConfig diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index cb60c27c..9d9941ca 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -32,7 +32,7 @@ def validate_factory(cls, value: str) -> str: class RolloutConfig(BaseModel): model_config = ConfigDict(extra="forbid") - mode: Literal["forecast", "policy"] = "forecast" + mode: Literal["forecast"] = "forecast" adapter: AdapterConfig bindings: list[InputBinding] trajectory_path: str = Field(min_length=1) diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 12f43db3..e73e591b 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -868,21 +868,6 @@ def resolve(self): and archive is not None ): archive_path = (root / archive).resolve() - for artifact in ( - self.model_path, - self.report_path, - self.checkpoint_path, - self.phase_directory, - ): - artifact_path = Path(artifact) - if ( - archive_path == artifact_path - or archive_path in artifact_path.parents - or artifact_path in archive_path.parents - ): - raise ValueError( - "archive_directory must be separate from artifacts" - ) phase.archive_directory = str(archive_path) return self diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 0b8718f9..65994511 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -1743,9 +1743,9 @@ selection monitors and exact/non-exact continuation. Examples: - [BC](../example_inputs/configs/train-bc-toy.yaml) - [Return-conditioned imitation](../example_inputs/configs/train-return-conditioned.yaml) - [IQL](../example_inputs/configs/train-iql-from-table.yaml) -- [PPO](../example_inputs/configs/train-ppo-toy.yaml) -- [DPO](../example_inputs/configs/train-dpo-control.yaml) -- [GRPO](../example_inputs/configs/train-grpo-control.yaml) +- [PPO](../../src/sequifier/templates/train-ppo.yaml) +- [DPO](../../src/sequifier/templates/train-dpo.yaml) +- [GRPO](../../src/sequifier/templates/train-grpo.yaml) - [Imitation → IQL → DPO → GRPO → PPO](../example_inputs/configs/train-multiphase-control.yaml) @@ -2022,7 +2022,10 @@ ORT installation. Outputs stay indexed by outer item coordinates. Use the selected actor export rather than the policy run checkpoint. Use raw CSV/Parquet observation rows and optional recorded action histories; alignment, vocabulary, and normalization come from the artifact. `model_type: generative` and `model_type: policy` are accepted. Environment -execution uses the existing `rollout` section with `mode: policy`. See the +execution uses a policy `rollout` section with `environment` (a version-3 adapter +supporting `seeded_reset`), `trajectory_path`, `seed_index`, and optional +`max_episode_steps`. Set `autoregressive: true` and `generation_steps`; policy +rollout does not configure forecast bindings. See the [policy guide](../policy-learning.md#python-and-inference) and [inference example](../example_inputs/configs/infer-policy.yaml). @@ -2364,8 +2367,8 @@ vectorized environments, and reward models are outside this contract. ## Authoring a run Start with the [multi-phase example](example_inputs/configs/train-multiphase-control.yaml), -[single-phase DPO](example_inputs/configs/train-dpo-control.yaml), or -[single-phase GRPO](example_inputs/configs/train-grpo-control.yaml). +[single-phase DPO](../src/sequifier/templates/train-dpo.yaml), or +[single-phase GRPO](../src/sequifier/templates/train-grpo.yaml). Generate their curated data with: ```shell @@ -2459,19 +2462,30 @@ update always evaluates. Missing finite evaluation for `best` fails explicitly. export uses the last phase's selected actor, with metric, transaction and weight hash recorded; metrics from different phases are never compared globally. -Imitation, DPO and GRPO share these actor optimization fields: +Each phase configures trainable roles under `optimization`. Imitation, DPO and +GRPO have `actor`; PPO has `actor` and `value`; IQL has `actor`, `value` and +`critic` (the twin Q networks share that optimizer). Omitted roles use defaults. +For example: + +```yaml +optimization: + actor: {learning_rate: 0.005, optimizer: {name: AdamW}, gradient_clip: 1.0} +``` + +Every role defaults to `learning_rate: 0.0003`, `optimizer: {name: Adam}`, +`scheduler: {name: ConstantLR, factor: 1.0, total_iters: 1}`, and +`gradient_clip: 1.0`. Unknown roles fail validation. + +Imitation, DPO and GRPO share these batching and scheduling fields: | Field | Default | | --- | --- | | `batch_size`, `accumulation_steps` | 32, 1; DPO counts pairs, GRPO counts whole trajectories | -| `actor_learning_rate`, `actor_optimizer` | 0.0003, `{name: Adam}` | -| `actor_scheduler` | `{name: ConstantLR, factor: 1.0, total_iters: 1}` | -| `gradient_clip` | 1.0 | | `evaluation_interval`, `checkpoint_interval` | 1; optimizer updates for imitation/DPO, completed collection/optimization updates for GRPO | Optimizers support Adam, AdamW, SGD and RMSprop. Their constructor arguments belong alongside `name` (or in `arguments`); `lr` and parameters belong to the -phase. Schedulers advance on completed optimizer steps; plateau/compound +role configuration. Schedulers advance on completed optimizer steps; plateau/compound schedulers are unsupported. `OneCycleLR.total_steps` must equal the authored maximum optimizer-step budget. Accumulation weights by the actual number of examples, including final partial batches. Network stochastic layers are disabled @@ -2517,10 +2531,11 @@ Each update samples transitions uniformly with replacement. Held-out unweighted `validation_action_nll` is the minimizing selection monitor; Bellman residual, values, Q and weight diagnostics are reported separately. -`actor_`, `value_`, and `critic_learning_rate` default to 0.0003; each role has -its own optimizer/scheduler. `gradient_clip` defaults to 1.0, evaluation and -checkpoint intervals to 100 updates. Critics are fresh even after another IQL -phase and are absent from the portable actor. +The `optimization.actor`, `optimization.value` and `optimization.critic` roles +have independent optimizers and schedulers. All gradients are validated and +clipped once before any optimizer steps. Evaluation and checkpoint intervals +default to 100 updates. Critics are fresh even after another IQL phase and are +absent from the portable actor. ## DPO preferences @@ -2582,7 +2597,7 @@ a still-present artifact with changed weights is rejected on resume. ## Online adapters -PPO and GRPO share the version-3 `OnlineEnvironmentAdapter` contract. Configure +PPO, GRPO, evaluation and policy rollout share the version-3 `OnlineEnvironmentAdapter` contract. Configure `environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. Factories must create fresh independent adapters for evaluation. An adapter explicitly declares: @@ -2625,9 +2640,14 @@ establish this capability. Exact GRPO continuation additionally requires the adapter's documented `reproducible_group_start` guarantee. The bundled `CategoricalControlAdapter` demonstrates a deterministic scenario token. -`SeededAdapterBridge` can adapt an existing seeded forecast-rollout adapter with -an explicit initial-state factory; it declares no group-start or exact-snapshot -capability. Forecast rollout bindings keep their separate version-1 contract. +Policy inference from supplied observation rows additionally requires +`EnvironmentCapabilities(seeded_reset=True)` and +`reset_from_observation(initial: OnlineResetResult, seed: int) -> OnlineResetResult`. +It must initialize the simulator without executing an action and return exactly +the supplied observation, static inputs and units. This is distinct from the +ordinary `reset(seed)` used during training. The bundled control adapter +implements both. Forecast rollout bindings retain their separate version-1 +contract. ## Online categorical PPO @@ -2645,8 +2665,8 @@ Defaults: `optimization_epochs: 4`, `minibatch_size: 64`, `accumulation_steps: 1 `value_coefficient: 0.5`, `entropy_coefficient: 0.01`, `normalize_advantages: true`, `reward_scale: 1`. Optional positive `target_kl` stops optimization against the old collection policy; `value_clip` is optional. -`actor_learning_rate`/`value_learning_rate` default to 0.0003, with independent -optimizers/schedulers and `actor_gradient_clip`/`value_gradient_clip` of 1.0. +The `optimization.actor` and `optimization.value` roles have independent +optimizers/schedulers and the shared learning-rate and clipping defaults. `allow_partial_rollout: true` executes exactly the remaining action budget; false requires divisibility. `max_buffer_bytes` defaults to 536870912. `archive_directory` optionally writes separate versioned collection records; @@ -2728,32 +2748,29 @@ conflicts are checked separately before output mutation and at phase boundaries. Prepared data supplies its fitting view and frozen-representation validation; algorithm-specific transition checks remain with the IQL dataset. -The only policy-training checkpoint is `sequifier_policy_run`, format version 4. -Completed phase results determine the phase boundary; an optional live phase -state records the latest committed transaction. Only the incoming actor and -current phase's recovery tensors are embedded; earlier selected actors live in -their phase artifacts rather than being serialized with every checkpoint. -It contains the resolved plan, immutable representation and its fingerprint, data -and adapter identities, run RNG, completed results, phase manifest paths and -checksums, and optional versioned phase state. DPO/GRPO reference weights are -embedded in live phase state. Optimizer, scheduler, sampler, selection, evaluation -and phase counters remain phase-local. Only version-4 checkpoints are supported. - -Writes use a temporary file and atomic replacement at completed updates or -handoffs. No partial group is checkpointable. Handoffs flush the selected -`actor.pt`, then `report.json`, then a small `manifest.json` recording their SHA-256 -checksums, the selected weight hash, and result/representation fingerprints. Only -then is the run checkpoint committed with the manifest identity and incoming -actor. The checkpoint is the authoritative phase boundary: exports left by an -interrupted, uncommitted handoff can be reused if valid or replaced on retry. -A committed handoff advances the phase index exactly once; resume never reruns -completed phases or reselects their actor. - -Completed phase artifacts and manifests must be retained alongside the checkpoint -for recovery; the checkpoint alone is no longer self-contained. Resume verifies -their identities and fails on missing or modified files before output mutation, -without loading or rewriting earlier models. Representation and -data/adapter/reference mismatches also fail before output mutation. +The policy-training checkpoint is `sequifier_policy_run`, format version 5. +Completed results determine the phase boundary; optional version-2 phase state +records the latest committed transaction. It contains the resolved plan, frozen +representation, source/partition and adapter identities, run RNG, completed +results and phase actor paths/checksums. Live phase state contains role-keyed +networks and optimizers/schedulers/step counts, sampler, selection, metrics and +algorithm-specific progress. DPO/GRPO references are embedded. Earlier selected +actors remain in their phase artifacts. Earlier unreleased checkpoints are not +supported. + +Writes use temporary files, flush them, and atomically replace their destinations. +At a handoff, the selected `actor.pt` is published with its result metadata, then +the run checkpoint commits its checksum and result. The checkpoint is the +authoritative phase boundary. An interrupted, uncommitted export is replaced on +retry; a committed handoff advances exactly once. Completed phases never rerun +or reselect their actor. + +Retain phase actor artifacts alongside the checkpoint. Resume checks their +checksums and result identities before output mutation, without loading or +rewriting earlier models. Phase `report.json` files are derived from committed +results and regenerated on resume; they are not recovery dependencies. No +separate phase manifest is needed. Representation and data/adapter/reference +mismatches still fail before output mutation. `resume.mode: exact` requires validated PPO simulator/RNG snapshots or GRPO's reproducible group-start capability and matching implementation identity. PPO @@ -2791,8 +2808,13 @@ paths. `rl_api` also exports the phase schemas, preference records, online capabilities, `PolicyWindowView`, and model loading/execution helpers. Use `sequifier infer` with the final artifact for categorical sequence inference -or rollout. `PolicyExecutor` and `collect_policy_rollout` build the same causal -histories; return-conditioned actors require an explicit desired return. +or rollout. `collect_policy_rollout` uses the same causal `PolicyHistory` and +`decide` path as online training and evaluation. Return-conditioned actors require +an explicit desired return. `PolicyRolloutConfig` declares `environment`, +`trajectory_path`, `seed_index` and optional `max_episode_steps`; the environment +must support seeded reset. Successors always come from the environment, so no +prediction bindings are configured. A true terminal may omit its successor; +truncation and collection cutoffs must retain it. See [policy inference](configs/infer.md), [rollout](rollout.md), and the [Python PPO workflow](example_inputs/ppo_toy_control.py). diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index edb16129..2c6827da 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -1340,11 +1340,11 @@ def get_probs_preds_from_dict( all_probs_list = {col: [] for col in target_cols} all_preds_list = {col: [] for col in target_cols} - from sequifier.rollout.coordinator import coordinate_steps - metadata_for_step = metadata_np - def predict(): + if not isinstance(total_steps, int) or total_steps < 1: + raise ValueError("generation budget must be a positive integer") + for i in range(total_steps): if config.output_probabilities: probs_for_step = inferer.infer_generative( X, @@ -1364,15 +1364,11 @@ def predict(): column_data_types=column_data_types, return_probs=False, ) - return preds_for_step - - def execute(preds_for_step, i): - nonlocal X for col in target_cols: all_preds_list[col].append(preds_for_step[col]) if i == (total_steps - 1): - return False + break X_next = {} for col in X.keys(): @@ -1397,9 +1393,6 @@ def execute(preds_for_step, i): metadata_for_step["attention_valid_mask"] = np.concatenate( [shifted_mask, appended_valid], axis=1 ) - return False - - coordinate_steps(predict, execute, total_steps) final_preds = { col: np.array(preds_list).T.reshape(-1, 1).flatten() diff --git a/src/sequifier/io/atomic.py b/src/sequifier/io/atomic.py new file mode 100644 index 00000000..db03e9e2 --- /dev/null +++ b/src/sequifier/io/atomic.py @@ -0,0 +1,41 @@ +"""Atomic publication shared by model, checkpoint and trajectory writers.""" + +import json +import os +import uuid +from contextlib import contextmanager +from pathlib import Path + + +@contextmanager +def atomic_path(path, *, durable=False): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + yield temporary + if durable: + with temporary.open("rb") as stream: + os.fsync(stream.fileno()) + temporary.replace(path) + if durable and os.name != "nt": + descriptor = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + temporary.unlink(missing_ok=True) + + +def atomic_save(path, payload, *, durable=True): + import torch + + with atomic_path(path, durable=durable) as temporary: + torch.save(payload, temporary) + + +def write_json_atomic(path, payload, **options): + options = {"indent": 2, "allow_nan": False, **options} + with atomic_path(path, durable=True) as temporary: + temporary.write_text(json.dumps(payload, **options) + "\n", encoding="utf-8") diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py index d4c176d4..954f5bd4 100644 --- a/src/sequifier/io/iql_dataset.py +++ b/src/sequifier/io/iql_dataset.py @@ -1,7 +1,5 @@ """Fixed offline transitions with causal current and successor contexts.""" -import hashlib -import json from bisect import bisect_right from dataclasses import dataclass @@ -32,7 +30,6 @@ def __init__(self, policy_dataset): self.view = PolicyWindowView(policy_dataset.config) self.cumulative_lengths = policy_dataset.cumulative_lengths self.episode_ids = policy_dataset.episode_ids - digest = hashlib.sha256() for episode_index, episode in enumerate(policy_dataset.episodes): source = episode_index transitions = episode.transitions @@ -55,29 +52,12 @@ def __init__(self, policy_dataset): terminated = transitions.terminated[step] bootstrap = transitions.bootstrap_allowed[step] successor = self._successor(episode, step) - if terminated and bootstrap: - raise ValueError(f"{location}: termination forbids bootstrap") if not terminated and not bootstrap: raise ValueError( f"{location}: nonterminal transition must bootstrap" ) if bootstrap and successor is None: raise ValueError(f"{location}: bootstrap needs the true successor") - if successor is not None and set(successor) != set( - episode.observations - ): - raise ValueError( - f"{location}: successor fields differ from observations" - ) - if successor is not None and step + 1 < len(episode.actions): - if successor != episode.observation(step + 1): - raise ValueError( - f"{location}: successor breaks episode continuity" - ) - digest.update( - json.dumps(episode.model_dump(mode="json"), sort_keys=True).encode() - ) - self.data_contract = digest.hexdigest() def use_config(self, config): self.view = PolicyWindowView(config) diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index 1732b54b..829a202e 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -1,7 +1,5 @@ """Episode-safe policy views over ordinary sequences and imported rollout logs.""" -import hashlib -import json from bisect import bisect_right from collections import OrderedDict from collections.abc import Sequence @@ -54,7 +52,6 @@ def __init__( self.exclusions = {} self.all_episode_ids = [] self.episode_ids = [] - self.content_keys = [] self.demonstration_keys = [] self.group_ids = [] self.lengths = [] @@ -75,7 +72,7 @@ def __init__( seen.add(episode_id) self.all_episode_ids.append(episode_id) length = record.length - content = demonstration = record.content_key + demonstration = record.content_key if config.return_conditioned: episode = view[index] length = episode.return_length(config.returns.finite_horizon) @@ -87,12 +84,9 @@ def __init__( else "unknown_return_tail" ) continue - returns = _return_labels(episode, length, config.returns.gamma) - content = episode_content_key(episode, length=length, returns=returns) demonstration = episode_content_key(episode, length=length) eligible.append(index) self.episode_ids.append(episode_id) - self.content_keys.append(content) self.demonstration_keys.append(demonstration) self.group_ids.append(record.group_id) self.lengths.append(length) @@ -109,12 +103,6 @@ def __init__( def __len__(self): return self.cumulative_lengths[-1] if self.cumulative_lengths else 0 - @property - def data_contract(self): - return hashlib.sha256( - json.dumps([self.content_keys, self.episode_ids]).encode() - ).hexdigest() - def sampler(self, generator): return EpisodeWindowSampler(self, generator) diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 7f3d7dbd..cfe43b36 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -1,4 +1,6 @@ import os +from importlib.resources import files +from pathlib import Path from sequifier.typechecking import beartype @@ -91,268 +93,6 @@ generation_steps: 5 """ -policy_train_config_string = """project_root: . -model_name: policy -device: cpu -seed: 1010 -model: - backbone: - architecture: - dim_model: 32 - num_layers: 2 - max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} - positional_encoding_scope: global - dropout: 0.0 - interfaces: - actor: - input_columns: [OBSERVATION_COLUMN] - target_columns: [action] - policy: - observation_columns: [OBSERVATION_COLUMN] - action_values: [0.0, 1.0] - return_conditioned: true - ingestion: - type: embedding - output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -context_length: 4 -sources: - demonstrations.recorded: - paths: [PLEASE_FILL.csv] -validation_fraction: 0.2 -episode_cache_size: 8 -resume: {policy: never} -phases: -- algorithm: policy_imitation - epochs: 80 - sources: [demonstrations.recorded] - name: policy_imitation - batch_size: 64 - actor_learning_rate: 0.005 -""" - -policy_infer_config_string = """project_root: . -model_path: models/policy/actor.pt -data_path: PLEASE_FILL -model_type: generative -read_format: csv -write_format: csv -device: cpu -deterministic: true -# Omit desired_return for behavior cloning. -desired_return: 10.0 -""" - -iql_train_config_string = """project_root: . -model_name: offline-iql -device: cpu -seed: 1010 -model: - backbone: - architecture: - dim_model: 32 - num_layers: 2 - max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} - positional_encoding_scope: global - interfaces: - actor: - input_columns: [OBSERVATION_COLUMN] - target_columns: [action] - policy: - observation_columns: [OBSERVATION_COLUMN] - action_values: [0.0, 1.0] - return_conditioned: false - ingestion: - type: embedding - output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -context_length: 16 -sources: - demonstrations.recorded: - paths: [PLEASE_FILL.csv] - columns: - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN} -validation_fraction: 0.2 -resume: {policy: if_exists} -phases: -- name: offline_improvement - algorithm: iql - sources: [demonstrations.recorded] - updates: 10000 - batch_size: 64 -""" - -ppo_train_config_string = """project_root: . -model_name: online-ppo -device: cpu -seed: 1010 -context_length: 4 -model: - backbone: - architecture: - dim_model: 32 - num_layers: 1 - max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} - positional_encoding_scope: global - dropout: 0.0 - interfaces: - actor: - input_columns: [cue] - target_columns: [action] - policy: - observation_columns: [cue] - action_values: [left, right] - return_conditioned: false - ingestion: - type: embedding - output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -normalization: {source: identity} -resume: {policy: if_exists, mode: exact} -phases: -- name: online_control - algorithm: ppo - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - actor_learning_rate: 0.0003 - value_learning_rate: 0.0003 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 -""" - - -dpo_train_config_string = """project_root: . -model_name: dpo-control -device: cpu -seed: 1010 -context_length: 4 -model: - backbone: - architecture: - dim_model: 32 - num_layers: 1 - max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} - positional_encoding_scope: global - dropout: 0.0 - interfaces: - actor: - input_columns: [cue] - target_columns: [action] - policy: - observation_columns: [cue] - action_values: [left, right] - return_conditioned: false - ingestion: - type: embedding - output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -normalization: {source: identity} -phases: -- {name: preferences, algorithm: dpo, preferences: rankings, updates: 100, beta: 0.1} -sources: - decisions: - paths: [data/control-decisions.csv] - columns: - episode_id: episodeId - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {cue: next_cue} - group: contextId -preferences: {rankings: data/control-preferences.json} -""" - -grpo_train_config_string = """project_root: . -model_name: grpo-control -device: cpu -seed: 1010 -context_length: 4 -model: - backbone: - architecture: - dim_model: 32 - num_layers: 1 - max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} - positional_encoding_scope: global - dropout: 0.0 - interfaces: - actor: - input_columns: [cue] - target_columns: [action] - policy: - observation_columns: [cue] - action_values: [left, right] - return_conditioned: false - ingestion: - type: embedding - output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -normalization: {source: identity} -phases: -- name: group_improvement - algorithm: grpo - total_groups: 32 - groups_per_update: 2 - group_size: 4 - max_steps: 8 - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - evaluation: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 - metric: score_mean -""" - gitignore_string = """models/ logs/ checkpoints/ @@ -385,20 +125,15 @@ def make(args): with open(f"{project_name}/configs/infer.yaml", "w") as f: f.write(infer_config_string) - with open(f"{project_name}/configs/train-policy.yaml", "w") as f: - f.write(policy_train_config_string) - - with open(f"{project_name}/configs/infer-policy.yaml", "w") as f: - f.write(policy_infer_config_string) - - with open(f"{project_name}/configs/train-iql.yaml", "w") as f: - f.write(iql_train_config_string) - - with open(f"{project_name}/configs/train-ppo.yaml", "w") as f: - f.write(ppo_train_config_string) - - with open(f"{project_name}/configs/train-dpo.yaml", "w") as f: - f.write(dpo_train_config_string) - - with open(f"{project_name}/configs/train-grpo.yaml", "w") as f: - f.write(grpo_train_config_string) + for name in ( + "train-policy", + "infer-policy", + "train-iql", + "train-ppo", + "train-dpo", + "train-grpo", + ): + template = files("sequifier").joinpath("templates", f"{name}.yaml") + Path(project_name, "configs", f"{name}.yaml").write_text( + template.read_text(encoding="utf-8"), encoding="utf-8" + ) diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index d4d5b454..727ef972 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -14,6 +14,10 @@ from sequifier.rl.policy import CategoricalPolicy +def phase_actor_path(directory, index, name): + return Path(directory) / f"{index:03d}-{name}" / "actor.pt" + + def validate_policy_weights(policy: CategoricalPolicy) -> None: for name, value in policy.network.state_dict().items(): if value.is_floating_point() and value.dtype != torch.float32: diff --git a/src/sequifier/rl/control.py b/src/sequifier/rl/control.py index d1402b7b..813248b8 100644 --- a/src/sequifier/rl/control.py +++ b/src/sequifier/rl/control.py @@ -12,13 +12,13 @@ OnlineResetResult, OnlineStepResult, ) -from sequifier.rollout.contracts import StepResult class CategoricalControlAdapter: protocol_version = 3 implementation_id = "sequifier.categorical-control.v1" capabilities = EnvironmentCapabilities( + seeded_reset=True, exact_snapshot=True, group_start="scenario", reproducible_group_start=True, @@ -40,6 +40,18 @@ def reset(self, seed): self.ended = False return OnlineResetResult({"cue": self.cue}) + def reset_from_observation(self, initial, seed): + if ( + set(initial.observation) != {"cue"} + or initial.observation["cue"] not in {-1.0, 1.0} + or initial.static_inputs + or initial.units + ): + raise ValueError("control seed requires only a cue of -1 or +1") + self.reset(seed) + self.cue = float(initial.observation["cue"]) + return OnlineResetResult({"cue": self.cue}) + def create_group_start(self, seed): return {"cue": random.Random(seed).choice((-1.0, 1.0)), "horizon": self.horizon} @@ -94,32 +106,3 @@ def restore_state(self, state): def close(self): pass - - -class ControlRolloutAdapter: - """Version-1 bridge for the existing sequifier infer rollout machinery.""" - - protocol_version = 1 - - def __init__(self, horizon=8): - self.environment = CategoricalControlAdapter(horizon) - - def reset(self, request): - self.environment.reset(request.seed) - cue = request.seed_history["cue"][-1] - if cue not in {-1.0, 1.0}: - raise ValueError("control seed cue must be -1 or +1") - self.environment.cue = float(cue) - - def step(self, request): - if request.observation != {"cue": self.environment.cue}: - raise ValueError("control observation and environment disagree") - result = self.environment.step(request.predictions) - if result.observation is None: - raise ValueError("control must return the true successor") - return StepResult( - result.observation, result.reward, result.terminated, result.truncated - ) - - def close(self): - self.environment.close() diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index bc57139c..06d38e2e 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -6,14 +6,7 @@ from sequifier.config.policy_plan import DPOPhaseConfig from sequifier.io.preference_dataset import PreferenceData from sequifier.rl.batch_order import BatchOrder -from sequifier.rl.run_state import ( - PhaseContext, - Selection, - actor_state, - apply_gradients, - phase_events, - restore_actor_state, -) +from sequifier.rl.run_state import PhaseContext, PhaseSession from sequifier.rl.scoring import sequence_log_probabilities, sequence_score @@ -73,24 +66,24 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): ) if reference is None: raise ValueError("DPO requires a frozen reference actor") - optimizer, scheduler = context.optimization["actor"].build(actor) - restore_actor_state(context, optimizer, scheduler) - state = context.resume_state or {} - transaction, metrics = state.get("transaction", 0), list(state.get("metrics", [])) - if not 0 <= transaction <= phase.updates: + session = PhaseSession(context) + state = session.state + if not 0 <= session.transaction <= phase.updates: raise ValueError("DPO checkpoint exceeds authored update budget") - selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) batches = BatchOrder( len(data.training), phase.batch_size * phase.accumulation_steps, context.run.seed + context.index, state=state["batch_order"] if state else None, ) - if transaction != batches.transactions: + if ( + session.transaction != batches.transactions + or session.steps["actor"] != session.transaction + ): raise ValueError("DPO sampler position disagrees with optimizer transaction") - for update in range(transaction + 1, phase.updates + 1): + for update in range(session.transaction + 1, phase.updates + 1): group = batches.next() - optimizer.zero_grad(set_to_none=True) + session.zero_grad() loss_total = 0.0 for index in group: pair = data.training.pairs[index] @@ -112,39 +105,21 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): actor, episode, span.start, span.end, phase.scoring_chunk_size ): (sign * coefficient * chunk.sum()).backward() - apply_gradients( - actor, optimizer, scheduler, context.optimization["actor"].gradient_clip - ) - transaction = update - evaluate, checkpoint = phase_events( - phase, update, final=update == phase.updates - ) - if evaluate: + session.step() + session.transaction = update + final = update == phase.updates + if session.should_evaluate(final): held_out = evaluate_dpo(actor, reference, data.validation, phase) - selection.observe(actor, held_out["dpo_loss"], update) - metrics.append( + session.observe(held_out["dpo_loss"]) + session.metrics.append( { "transaction": update, "training_dpo_loss": loss_total, **{f"validation_{key}": value for key, value in held_out.items()}, } ) - if checkpoint: - context.commit( - actor_state( - context, - optimizer, - scheduler, - selection, - update, - metrics, - batch_order=batches.state_dict(), - ) - ) - return selection.finish( - context, - transaction, - {"optimizer_steps": transaction}, - metrics, + session.checkpoint(final, batch_order=batches.state_dict()) + return session.finish( + {"optimizer_steps": session.steps["actor"]}, {"data": data.contract, "reference_weight_hash": context.reference_identity}, ) diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py index 1b6f7dc5..424eea18 100644 --- a/src/sequifier/rl/environment.py +++ b/src/sequifier/rl/environment.py @@ -6,7 +6,6 @@ from typing import Literal, Mapping, Protocol from sequifier.io.values import DomainValue, numeric_scalar -from sequifier.rollout.contracts import ResetRequest, StepRequest @dataclass(frozen=True) @@ -29,6 +28,7 @@ class OnlineStepResult: @dataclass(frozen=True) class EnvironmentCapabilities: + seeded_reset: bool = False exact_snapshot: bool = False group_start: Literal["snapshot", "scenario"] | None = None reproducible_group_start: bool = False @@ -37,8 +37,13 @@ class EnvironmentCapabilities: group_start_randomness: Literal["captured", "independent"] = "independent" def __post_init__(self): - if not isinstance(self.exact_snapshot, bool) or not isinstance( - self.reproducible_group_start, bool + if any( + (not isinstance(flag, bool)) + for flag in ( + self.seeded_reset, + self.exact_snapshot, + self.reproducible_group_start, + ) ): raise ValueError("environment capability flags must be booleans") if self.group_start not in { @@ -59,6 +64,9 @@ class OnlineEnvironmentAdapter(Protocol): capabilities: EnvironmentCapabilities def reset(self, seed: int) -> OnlineResetResult: ... + def reset_from_observation( + self, initial: OnlineResetResult, seed: int + ) -> OnlineResetResult: ... def step(self, action: Mapping[str, DomainValue]) -> OnlineStepResult: ... def close(self) -> None: ... def snapshot_state(self) -> dict: ... @@ -71,21 +79,7 @@ def load_online_adapter(spec): module, attribute = spec.factory.split(":") adapter = getattr(import_module(module), attribute)(**spec.config) try: - if getattr(adapter, "protocol_version", None) != 3: - raise ValueError( - "online adapters must explicitly declare protocol_version = 3" - ) - if getattr(adapter, "implementation_id", None) != spec.implementation_id: - raise ValueError( - "environment.implementation_id differs from adapter implementation" - ) - if not isinstance( - getattr(adapter, "capabilities", None), EnvironmentCapabilities - ): - raise ValueError("online adapter requires explicit EnvironmentCapabilities") - for method in ("reset", "step", "close"): - if not callable(getattr(adapter, method, None)): - raise ValueError(f"online adapter requires {method}") + validate_online_adapter(adapter, spec) return adapter except BaseException: close = getattr(adapter, "close", None) @@ -94,6 +88,20 @@ def load_online_adapter(spec): raise +def validate_online_adapter(adapter, spec): + if getattr(adapter, "protocol_version", None) != 3: + raise ValueError("online adapters must explicitly declare protocol_version = 3") + if getattr(adapter, "implementation_id", None) != spec.implementation_id: + raise ValueError( + "environment.implementation_id differs from adapter implementation" + ) + if not isinstance(getattr(adapter, "capabilities", None), EnvironmentCapabilities): + raise ValueError("online adapter requires explicit EnvironmentCapabilities") + for method in ("reset", "step", "close"): + if not callable(getattr(adapter, method, None)): + raise ValueError(f"online adapter requires {method}") + + def require_exact_snapshot(adapter): if adapter.capabilities.exact_snapshot is not True: raise ValueError("exact PPO resume requires capabilities.exact_snapshot") @@ -159,83 +167,14 @@ def validate_step(result, units): return result -class SeededAdapterBridge: - """Expose a v1 adapter online when an explicit reset-state factory is available. - - The seed factory takes a seed and returns OnlineResetResult. All observations - must be produced by environment bindings with identical names. Snapshot - support is deliberately not inferred from the wrapped object. - """ - - protocol_version = 3 - implementation_id = "sequifier.seeded-bridge.v1" - capabilities = EnvironmentCapabilities() - - def __init__( - self, adapter, seed_factory, action_column, context_length=1, units=None +def require_seeded_reset(adapter): + if not adapter.capabilities.seeded_reset or not callable( + getattr(adapter, "reset_from_observation", None) ): - from sequifier.rollout.factories import validate_adapter - - validate_adapter(adapter) - if ( - not isinstance(context_length, int) - or isinstance(context_length, bool) - or context_length < 1 - ): - raise ValueError("bridge context_length must be positive") - self.adapter = adapter - self.seed_factory = seed_factory - self.action_column = action_column - self.context_length = context_length - self.units = units or {} - self.resets = 0 - - def reset(self, seed): - result = validate_reset(self.seed_factory(seed), self.units) - self.episode_id = f"bridge-{self.resets}" - self.resets += 1 - self.step_id = 0 - self.observation = dict(result.observation) - self.static_inputs = dict(result.static_inputs) - self.adapter.reset( - ResetRequest( - self.episode_id, - { - key: (value,) * self.context_length - for key, value in self.observation.items() - }, - (False,) * (self.context_length - 1) + (True,), - self.static_inputs, - seed, - ) - ) - return result - - def step(self, action): - if set(action) != {self.action_column}: - raise ValueError("bridge requires its one declared action column") - result = self.adapter.step( - StepRequest( - self.episode_id, - self.step_id, - self.observation, - dict(action), - self.static_inputs, - ) - ) - self.step_id += 1 - self.observation = dict(result.external_inputs) - return OnlineStepResult( - self.observation, - result.reward, - result.terminated, - result.truncated, - self.units, + raise ValueError( + "policy rollout requires seeded_reset and reset_from_observation(initial, seed)" ) - def close(self): - self.adapter.close() - def adapter_identity(adapter): from dataclasses import asdict diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 53665944..6233d0c1 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -7,6 +7,7 @@ import torch from sequifier.config.policy_plan import GRPOPhaseConfig +from sequifier.io.atomic import write_json_atomic from sequifier.rl.environment import ( adapter_identity, load_online_adapter, @@ -15,31 +16,14 @@ from sequifier.rl.grpo_collector import collect_groups from sequifier.rl.grpo_update import update_grpo from sequifier.rl.ppo_evaluation import evaluate_policy -from sequifier.rl.run_state import ( - PhaseContext, - Selection, - actor_state, - require_equal, - restore_actor_state, - restore_rng_state, - write_json_atomic, -) +from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): actor, phase = context.actor.eval(), context.phase - optimizer, scheduler = context.optimization["actor"].build(actor) - sampler = torch.Generator().manual_seed(context.run.seed + context.index) - restore_actor_state(context, optimizer, scheduler) - if context.resume_state is not None: - sampler.set_state(context.resume_state["sampler"]) - state = context.resume_state or {} - transaction, metrics = state.get("transaction", 0), list(state.get("metrics", [])) - groups, env_steps, optimizer_steps = ( - state.get("groups", 0), - state.get("env_steps", 0), - state.get("optimizer_steps", 0), - ) + session = PhaseSession(context) + state = session.state + groups, env_steps = state.get("groups", 0), state.get("env_steps", 0) evaluations, continuations = ( list(state.get("evaluations", [])), list(state.get("continuations", [])), @@ -47,23 +31,19 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): restart_stream = state.get("restart_stream", 0) if ( not 0 <= groups <= phase.total_groups - or transaction + or session.transaction != (groups + phase.groups_per_update - 1) // phase.groups_per_update ): raise ValueError("GRPO checkpoint is not a complete update/group boundary") - if groups != min(transaction * phase.groups_per_update, phase.total_groups): + if groups != min(session.transaction * phase.groups_per_update, phase.total_groups): raise ValueError("GRPO checkpoint group counter is not a complete iteration") if ( - not transaction <= optimizer_steps <= phase.optimizer_step_budget() - or not groups * phase.group_size + not groups * phase.group_size <= env_steps <= groups * phase.group_size * phase.max_steps ): - raise ValueError( - "GRPO checkpoint optimizer/environment clocks exceed authored bounds" - ) - selection = Selection(phase, "score_mean", "max", state.get("selection")) - actor.grpo_policy_version = transaction + raise ValueError("GRPO environment clock exceeds authored bounds") + actor.grpo_policy_version = session.transaction adapter = load_online_adapter(phase.environment) try: require_group_start(adapter, exact=context.run.resume.mode == "exact") @@ -74,15 +54,14 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): restart_stream += 1 continuations.append( { - "transaction": transaction, + "transaction": session.transaction, "mode": "new_groups", "restart_stream": restart_stream, "trajectory_equivalent": False, } ) - if state: - restore_rng_state(actor.device, state["random"]) + session.restore_random() def evaluate(): result = evaluate_policy(actor, phase.evaluation, forbidden_adapter=adapter) @@ -93,34 +72,31 @@ def evaluate(): ) if not torch.isfinite(torch.tensor(score)): raise ValueError("GRPO evaluation score exceeds finite FP32 range") - selection.observe(actor, score, transaction) + session.observe(score) evaluations.append( - {**result, "score_mean": score, "transaction": transaction} + {**result, "score_mean": score, "transaction": session.transaction} ) while groups < phase.total_groups: count = min(phase.groups_per_update, phase.total_groups - groups) buffer, executed = collect_groups( - context, adapter, groups, count, transaction, env_steps, restart_stream - ) - record = update_grpo( context, - buffer, - optimizer, - scheduler, - sampler, - transaction, - optimizer_steps, - ) - optimizer_steps = record["optimizer_steps"] - env_steps, groups, transaction = ( - env_steps + executed, - groups + count, - transaction + 1, + adapter, + groups, + count, + session.transaction, + env_steps, + restart_stream, ) + record = update_grpo(session, buffer) + env_steps += executed + groups += count + session.transaction += 1 + final = groups == phase.total_groups if phase.archive_directory: write_json_atomic( - Path(phase.archive_directory) / f"groups-{transaction:08d}.json", + Path(phase.archive_directory) + / f"groups-{session.transaction:08d}.json", { "format": "sequifier.grpo-archive", "version": 1, @@ -129,15 +105,12 @@ def evaluate(): "trajectories": [t.as_dict() for t in buffer.trajectories], }, ) - if ( - transaction % phase.evaluation_interval == 0 - or groups == phase.total_groups - ): + if session.should_evaluate(final): evaluate() scores = [t.score for t in buffer.trajectories] record.update( { - "transaction": transaction, + "transaction": session.transaction, "groups": groups, "env_steps": env_steps, "policy_version": actor.grpo_policy_version, @@ -155,36 +128,24 @@ def evaluate(): "buffer_bytes": buffer.estimated_bytes, } ) - metrics.append(record) - if ( - transaction % phase.checkpoint_interval == 0 - or groups == phase.total_groups - ): - context.commit( - actor_state( - context, - optimizer, - scheduler, - selection, - transaction, - metrics, - sampler=sampler.get_state(), - groups=groups, - env_steps=env_steps, - optimizer_steps=optimizer_steps, - adapter_identity=identity, - evaluations=evaluations, - continuations=continuations, - restart_stream=restart_stream, - ) - ) + session.metrics.append(record) + session.checkpoint( + final, + groups=groups, + env_steps=env_steps, + adapter_identity=identity, + evaluations=evaluations, + continuations=continuations, + restart_stream=restart_stream, + ) finally: adapter.close() - return selection.finish( - context, - transaction, - {"optimizer_steps": optimizer_steps, "groups": groups, "env_steps": env_steps}, - metrics, + return session.finish( + { + "optimizer_steps": session.steps["actor"], + "groups": groups, + "env_steps": env_steps, + }, { "reference_weight_hash": context.reference_identity, "adapter": identity, diff --git a/src/sequifier/rl/grpo_update.py b/src/sequifier/rl/grpo_update.py index 7f6d1475..da84a30a 100644 --- a/src/sequifier/rl/grpo_update.py +++ b/src/sequifier/rl/grpo_update.py @@ -2,8 +2,6 @@ import torch -from sequifier.rl.run_state import apply_gradients - def grpo_objective( log_probability, old_log_probability, reference_log_probability, advantage, phase @@ -40,10 +38,10 @@ def validate_old_scores(actor, buffer, chunk_size): ) -def update_grpo( - context, buffer, optimizer, scheduler, sampler, generation, optimizer_steps -): +def update_grpo(session, buffer): + context = session.context actor, phase = context.actor.eval(), context.phase + generation = session.transaction buffer.require_fresh(generation, context.reference_identity) if actor.grpo_policy_version != generation: raise ValueError("GRPO collection generation differs from actor") @@ -62,7 +60,7 @@ def update_grpo( size = phase.batch_size * phase.accumulation_steps epoch = -1 for epoch in range(phase.optimization_epochs): - order = torch.randperm(len(trajectories), generator=sampler).tolist() + order = torch.randperm(len(trajectories), generator=session.sampler).tolist() for start in range(0, len(order), size): indices = order[start : start + size] if phase.old_policy_target_kl is not None: @@ -86,7 +84,7 @@ def update_grpo( if old_kl > phase.old_policy_target_kl: stopped = True break - optimizer.zero_grad(set_to_none=True) + session.zero_grad() for offset in range(0, len(indices), phase.batch_size): # Whole trajectories belong to a minibatch; causal windows are # chunked within each trajectory without changing its weight. @@ -110,10 +108,7 @@ def update_grpo( trajectory.actions ) scored += 1 - apply_gradients( - actor, optimizer, scheduler, context.optimization["actor"].gradient_clip - ) - optimizer_steps += 1 + session.step() if stopped: break if not scored: @@ -122,7 +117,7 @@ def update_grpo( actor.grpo_policy_version = generation + 1 return { **{key: value / scored for key, value in totals.items()}, - "optimizer_steps": optimizer_steps, + "optimizer_steps": session.steps["actor"], "optimization_epochs": epoch + 1, "old_policy_kl_stopped": stopped, } diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index ac51b183..151aa50d 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -15,8 +15,8 @@ from sequifier.io.values import domain_scalar from sequifier.rl.checkpoint import load_policy from sequifier.rl.distributions import CategoricalActionDistribution, action_generator +from sequifier.rl.environment import load_online_adapter from sequifier.rl.runtime import collect_policy_rollout -from sequifier.rollout.factories import load_adapter from sequifier.rollout.history import numeric_scalar from sequifier.trajectory.schema import SeedHistory from sequifier.trajectory.store import save_trajectory @@ -81,7 +81,7 @@ def budget(row): desired_return = budget(row) trajectory = collect_policy_rollout( policy, - load_adapter(rollout.adapter), + load_online_adapter(rollout.environment), seed, rollout, config.generation_steps, diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index bd64632f..636531e1 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -10,14 +10,7 @@ from sequifier.io.tensors import collate_tensor_records from sequifier.rl.models import make_iql_models from sequifier.rl.offline_data import OfflineData -from sequifier.rl.run_state import ( - PhaseContext, - Selection, - actor_state, - apply_gradients, - clone_weights, - restore_actor_state, -) +from sequifier.rl.run_state import PhaseContext, PhaseSession def _select(inputs, mask): @@ -79,37 +72,19 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): ) for module in (value, q1, q2, *targets): module.eval() - modules = {"actor": actor, "value": value, "critic": torch.nn.ModuleList([q1, q2])} - plans = { - role: context.optimization[role].build(module) - for role, module in modules.items() - } - optimizers = {role: plan[0] for role, plan in plans.items()} - schedulers = {role: plan[1] for role, plan in plans.items()} - sampler = torch.Generator().manual_seed(context.run.seed + context.index) - state = context.resume_state or {} - if state: - for name, module in ( - ("value", value), - ("q1", q1), - ("q2", q2), - ("target_q1", targets[0]), - ("target_q2", targets[1]), - ): - module.load_state_dict(state["networks"][name], strict=True) - for role in ("value", "critic"): - optimizers[role].load_state_dict(state["optimizers"][role]) - schedulers[role].load_state_dict(state["schedulers"][role]) - restore_actor_state(context, optimizers["actor"], schedulers["actor"]) - if state: - sampler.set_state(state["sampler"]) - selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) - metrics = list(state.get("metrics", [])) - transaction = state.get("transaction", 0) - if not 0 <= transaction <= phase.updates: + session = PhaseSession( + context, + modules={"value": value, "critic": torch.nn.ModuleList([q1, q2])}, + frozen={"target_q1": targets[0], "target_q2": targets[1]}, + ) + if not 0 <= session.transaction <= phase.updates: raise ValueError("IQL checkpoint exceeds phase update budget") - for update in range(transaction + 1, phase.updates + 1): - indices = torch.randint(len(data[0]), (phase.batch_size,), generator=sampler) + if any(steps != session.transaction for steps in session.steps.values()): + raise ValueError("IQL optimizer clocks disagree with committed updates") + for update in range(session.transaction + 1, phase.updates + 1): + indices = torch.randint( + len(data[0]), (phase.batch_size,), generator=session.sampler + ) batch = cast( IQLBatch, collate_tensor_records([data[0][int(i)] for i in indices]) ).to(actor.device) @@ -144,29 +119,17 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): weight * actor.evaluate_actions(batch.current, batch.action) ).mean() losses = {"value": v_loss, "critic": q_loss, "actor": actor_loss} - # Validate every objective and every gradient before mutating any network. + session.zero_grad() for role, loss in losses.items(): if not torch.isfinite(loss): raise ValueError(f"non-finite IQL {role} objective") - optimizers[role].zero_grad(set_to_none=True) loss.backward() - for role, module in modules.items(): - torch.nn.utils.clip_grad_norm_( - module.parameters(), - context.optimization[role].gradient_clip, - error_if_nonfinite=True, - ) - for role, module in modules.items(): - apply_gradients( - module, - optimizers[role], - schedulers[role], - context.optimization[role].gradient_clip, - ) + session.step() _polyak(targets[0], q1, phase.target_update_rate) _polyak(targets[1], q2, phase.target_update_rate) - transaction = update - if update % phase.evaluation_interval == 0 or update == phase.updates: + session.transaction = update + final = update == phase.updates + if session.should_evaluate(final): held_out = _evaluate( actor, value, @@ -177,8 +140,8 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): phase.gamma, actor.device, ) - selection.observe(actor, held_out["action_nll"], update) - metrics.append( + session.observe(held_out["action_nll"]) + session.metrics.append( { "transaction": update, **{ @@ -193,40 +156,8 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): ), } ) - if update % phase.checkpoint_interval == 0 or update == phase.updates: - context.commit( - actor_state( - context, - optimizers["actor"], - schedulers["actor"], - selection, - update, - metrics, - sampler=sampler.get_state(), - networks={ - name: clone_weights(module) - for name, module in ( - ("value", value), - ("q1", q1), - ("q2", q2), - ("target_q1", targets[0]), - ("target_q2", targets[1]), - ) - }, - optimizers={ - role: optimizers[role].state_dict() - for role in ("value", "critic") - }, - schedulers={ - role: schedulers[role].state_dict() - for role in ("value", "critic") - }, - ) - ) - return selection.finish( - context, - transaction, - {f"{role}_optimizer_steps": transaction for role in modules}, - metrics, + session.checkpoint(final) + return session.finish( + {f"{role}_optimizer_steps": steps for role, steps in session.steps.items()}, {"data": context.data.contract}, ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 4af98339..ef2d4694 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -7,8 +7,8 @@ from sequifier.config.train_config import PolicyRunConfig from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism -from sequifier.rl.checkpoint import load_policy, save_policy -from sequifier.rl.phase_artifacts import PhaseArtifacts +from sequifier.io.atomic import write_json_atomic +from sequifier.rl.checkpoint import load_policy, phase_actor_path, save_policy from sequifier.rl.preparation import ( RunInputs, build_actor, @@ -26,13 +26,11 @@ ActorRepresentation, PhaseContext, PolicyRunResult, - capture_rng_state, clone_weights, require_equal, - restore_rng_state, weight_fingerprint, - write_json_atomic, ) +from sequifier.runtime.random_state import capture_rng_state, restore_rng_state def _plan_contract(config): @@ -41,6 +39,123 @@ def _plan_contract(config): return values +def _initialize_actor(config, data): + if config.initialize_from: + source = load_policy( + config.initialize_from, + device=config.device, + interface_name=config.policy_interface, + ) + validate_declared_policy(config.policy, source.config, "initialize_from.policy") + actor = build_actor( + config, source.config, source.interface.selected_columns_statistics + ) + ActorRepresentation.from_actor(actor, config.units).validate_actor( + source, config.units, "initialize_from" + ) + actor.network.load_state_dict(source.network.state_dict(), strict=True) + else: + first = data[0] + fitting = first.fitting_dataset() if first is not None else None + fitted = ( + fitting.fit_categorical_config() if fitting is not None else config.policy + ) + if set(fitted.categorical_columns) - fitted.categorical_n_classes.keys(): + raise ValueError( + "online-first categorical fields require frozen class counts and codecs" + ) + if config.normalization is not None or fitting is None: + statistics = resolve_normalization(fitted, config.normalization) + else: + fitting.use_config(fitted) + statistics = fitting.normalization_statistics() + actor = build_actor(config, fitted, statistics) + return actor + + +def _artifact_identity(config, path, representation, field): + actor = load_policy( + path, device=config.device, interface_name=config.policy_interface + ) + representation.validate_actor(actor, config.units, field) + return weight_fingerprint(actor.network.state_dict()) + + +def _preflight_references(config, recovery, resuming): + if resuming and config.initialize_from and Path(config.initialize_from).is_file(): + require_equal( + recovery.initialization["weight_hash"], + _artifact_identity( + config, + config.initialize_from, + recovery.representation, + "resume.initialize_from", + ), + "resume.initialize_from.weights", + ) + for index, phase in enumerate(config.phases): + reference = getattr(phase, "reference", None) + if reference is None or reference.source != "artifact": + continue + path = Path(reference.path) + if path.is_file(): + identity = _artifact_identity( + config, path, recovery.representation, f"phases.{phase.name}.reference" + ) + if str(index) in recovery.reference_contract: + require_equal( + recovery.reference_contract[str(index)], + identity, + f"phases.{phase.name}.reference.weights", + ) + recovery.reference_contract[str(index)] = identity + elif index > recovery.phase_index or ( + index == recovery.phase_index and recovery.phase_state is None + ): + raise FileNotFoundError( + f"reference artifact required before phase start: {path}" + ) + + +def _phase_reference(config, recovery, actor): + phase = config.phases[recovery.phase_index] + reference, reference_identity = None, None + declaration = getattr(phase, "reference", None) + if declaration is not None: + if recovery.phase_state is not None: + reference = deepcopy(actor) + reference.network.load_state_dict( + recovery.phase_state["reference"], strict=True + ) + elif declaration.source == "artifact": + reference = load_policy( + declaration.path, + device=config.device, + interface_name=config.policy_interface, + ) + else: + reference = deepcopy(actor) + reference.eval().requires_grad_(False) + recovery.representation.validate_actor( + reference, config.units, f"phases.{phase.name}.reference" + ) + reference_identity = weight_fingerprint(reference.network.state_dict()) + if recovery.phase_state is not None: + require_equal( + reference_identity, + recovery.phase_state["reference_identity"], + "resume.reference.weights", + ) + if declaration.source == "artifact": + require_equal( + recovery.reference_contract[str(recovery.phase_index)], + reference_identity, + "reference.artifact.weights", + ) + + return reference, reference_identity + + def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: """Validate the whole run, execute phase-local lifecycles, and export its final actor.""" if not isinstance(config, PolicyRunConfig): @@ -67,40 +182,7 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: recovery.validate_actor(actor, config.units) else: data = prepare_data(config, config.policy, inputs) - if config.initialize_from: - source = load_policy( - config.initialize_from, - device=config.device, - interface_name=config.policy_interface, - ) - validate_declared_policy( - config.policy, source.config, "initialize_from.policy" - ) - actor = build_actor( - config, source.config, source.interface.selected_columns_statistics - ) - ActorRepresentation.from_actor(actor, config.units).validate_actor( - source, config.units, "initialize_from" - ) - actor.network.load_state_dict(source.network.state_dict(), strict=True) - else: - first = data[0] - fitting = first.fitting_dataset() if first is not None else None - fitted = ( - fitting.fit_categorical_config() - if fitting is not None - else config.policy - ) - if set(fitted.categorical_columns) - fitted.categorical_n_classes.keys(): - raise ValueError( - "online-first categorical fields require frozen class counts and codecs" - ) - if config.normalization is not None or fitting is None: - statistics = resolve_normalization(fitted, config.normalization) - else: - fitting.use_config(fitted) - statistics = fitting.normalization_statistics() - actor = build_actor(config, fitted, statistics) + actor = _initialize_actor(config, data) representation = ActorRepresentation.from_actor(actor, config.units) if config.normalization is not None: require_equal( @@ -140,56 +222,15 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: clone_weights(actor.network), capture_rng_state(actor.device), ) - initialization = recovery.initialization - if resuming and config.initialize_from and Path(config.initialize_from).is_file(): - initial = load_policy( - config.initialize_from, - device=config.device, - interface_name=config.policy_interface, - ) - representation.validate_actor(initial, config.units, "resume.initialize_from") - require_equal( - initialization["weight_hash"], - weight_fingerprint(initial.network.state_dict()), - "resume.initialize_from.weights", - ) - reference_contract = recovery.reference_contract - for index, phase in enumerate(config.phases): - reference = getattr(phase, "reference", None) - if reference is None or reference.source != "artifact": - continue - path = Path(reference.path) - if path.is_file(): - ref = load_policy( - path, device=config.device, interface_name=config.policy_interface - ) - representation.validate_actor( - ref, config.units, f"phases.{phase.name}.reference" - ) - identity = weight_fingerprint(ref.network.state_dict()) - if str(index) in reference_contract: - require_equal( - reference_contract[str(index)], - identity, - f"phases.{phase.name}.reference.weights", - ) - reference_contract[str(index)] = identity - elif ( - index > recovery.phase_index - or index == recovery.phase_index - and recovery.phase_state is None - ): - raise FileNotFoundError( - f"reference artifact required before phase start: {path}" - ) + _preflight_references(config, recovery, resuming) saved_random = capture_rng_state(actor.device) optimization = preflight_optimization(config, actor, data) restore_rng_state(actor.device, recovery.random if resuming else saved_random) - artifacts = PhaseArtifacts(config.phase_directory, representation) if not resuming: # Persist the complete fitted representation before the first update. recovery.save(checkpoint, actor.device) + recovery.write_reports() while recovery.phase_index < len(config.phases): phase_index = recovery.phase_index current_state = recovery.phase_state @@ -201,39 +242,7 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: validate_output_paths(config, inputs) incoming_weight_hash = weight_fingerprint(actor.network.state_dict()) phase_actor = deepcopy(actor).eval() - reference, reference_identity = None, None - declaration = getattr(phase, "reference", None) - if declaration is not None: - if current_state is not None: - reference = deepcopy(phase_actor) - reference.network.load_state_dict( - current_state["reference"], strict=True - ) - elif declaration.source == "artifact": - reference = load_policy( - declaration.path, - device=config.device, - interface_name=config.policy_interface, - ) - else: - reference = deepcopy(phase_actor) - reference.eval().requires_grad_(False) - representation.validate_actor( - reference, config.units, f"phases.{phase.name}.reference" - ) - reference_identity = weight_fingerprint(reference.network.state_dict()) - if current_state is not None: - require_equal( - reference_identity, - current_state["reference_identity"], - "resume.reference.weights", - ) - if declaration.source == "artifact": - require_equal( - reference_contract[str(phase_index)], - reference_identity, - "reference.artifact.weights", - ) + reference, reference_identity = _phase_reference(config, recovery, phase_actor) def commit(state): representation.validate_actor( @@ -258,6 +267,7 @@ def commit(state): representation.validate_actor( actor, config.units, f"phases.{phase.name}.selected_actor" ) + actor_path = phase_actor_path(config.phase_directory, phase_index, phase.name) result = replace( result, provenance={ @@ -265,20 +275,13 @@ def commit(state): "config_fingerprint": fingerprint(phase.model_dump(mode="json")), "representation_fingerprint": representation.identity, "incoming_weight_hash": incoming_weight_hash, - "actor_path": str( - Path(config.phase_directory) - / f"{phase_index:03d}-{phase.name}" - / "actor.pt" - ), - "report_path": str( - Path(config.phase_directory) - / f"{phase_index:03d}-{phase.name}" - / "report.json" - ), + "actor_path": str(actor_path), + "report_path": str(actor_path.with_name("report.json")), }, ) - recovery.finish_phase(actor, result, artifacts) + recovery.finish_phase(actor, result) recovery.save(checkpoint, actor.device) + recovery.write_reports() report = { "format": "sequifier.policy-run-report", "version": 1, @@ -286,8 +289,8 @@ def commit(state): "plan_fingerprint": fingerprint(plan), "representation_fingerprint": representation.identity, "source_provenance": data_identity, - "reference_provenance": reference_contract, - "initialization": initialization, + "reference_provenance": recovery.reference_contract, + "initialization": recovery.initialization, "resume": config.resume.model_dump(mode="json"), "phases": [r.as_dict() for r in recovery.results], "model_path": config.model_path, diff --git a/src/sequifier/rl/phase_artifacts.py b/src/sequifier/rl/phase_artifacts.py deleted file mode 100644 index 0460b84e..00000000 --- a/src/sequifier/rl/phase_artifacts.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Commit completed phase exports before their run checkpoint handoff.""" - -import json -from pathlib import Path - -from sequifier.fingerprints import fingerprint -from sequifier.rl.checkpoint import save_policy -from sequifier.rl.run_state import ( - require_equal, - sync_file, - weight_fingerprint, - write_json_atomic, -) -from sequifier.trajectory.sources import file_checksum - - -class PhaseArtifacts: - def __init__(self, directory, representation): - self.directory = Path(directory) - self.representation = representation - - def paths(self, result): - directory = self.directory / f"{result.index:03d}-{result.name}" - return ( - directory / "actor.pt", - directory / "report.json", - directory / "manifest.json", - ) - - def _manifest(self, result): - actor, report, _ = self.paths(result) - return { - "format": "sequifier.policy-phase", - "version": 1, - "representation_fingerprint": self.representation.identity, - "result_fingerprint": fingerprint(result.as_dict()), - "selected_weight_hash": result.selected_weight_hash, - "actor": {"path": actor.name, "sha256": file_checksum(actor)}, - "report": {"path": report.name, "sha256": file_checksum(report)}, - } - - def validate(self, result, record): - """Check a committed export without loading earlier models into memory.""" - _, _, manifest = self.paths(result) - require_equal(str(manifest), record["path"], "resume.phase_manifest.path") - require_equal( - record["sha256"], file_checksum(manifest), "resume.phase_manifest.sha256" - ) - require_equal( - self._manifest(result), - json.loads(manifest.read_text(encoding="utf-8")), - "resume.phase_artifacts", - ) - - def commit(self, actor, result): - """Publish actor, report, then manifest; the run checkpoint commits last. - - Files not referenced by a committed run checkpoint are staged exports. - A retry can reuse a complete staged export or replace an incomplete one. - """ - require_equal( - result.selected_weight_hash, - weight_fingerprint(actor.network.state_dict()), - "handoff.selected_actor", - ) - actor_path, report, manifest = self.paths(result) - if manifest.is_file(): - record = {"path": str(manifest), "sha256": file_checksum(manifest)} - try: - self.validate(result, record) - return record - except (OSError, ValueError, KeyError, TypeError): - pass - save_policy(actor, actor_path, metadata=result.as_dict()) - sync_file(actor_path) - write_json_atomic(report, result.as_dict()) - write_json_atomic(manifest, self._manifest(result)) - return {"path": str(manifest), "sha256": file_checksum(manifest)} diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index e78e91db..7de83d60 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -3,8 +3,6 @@ from copy import deepcopy from pathlib import Path -import torch - from sequifier.config.policy_plan import PPOPhaseConfig from sequifier.rl.environment import ( adapter_identity, @@ -15,15 +13,7 @@ from sequifier.rl.ppo_collector import PPOCollector from sequifier.rl.ppo_evaluation import evaluate_policy from sequifier.rl.ppo_update import update_ppo -from sequifier.rl.run_state import ( - PhaseContext, - Selection, - actor_state, - clone_weights, - require_equal, - restore_actor_state, - restore_rng_state, -) +from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal from sequifier.rl.snapshots import snapshots_equal @@ -32,43 +22,18 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): value = HistoryCritic( deepcopy(actor.execution_config), actor.interface_name, None ).eval() - optimizer, scheduler = context.optimization["actor"].build(actor) - value_optimizer, value_scheduler = context.optimization["value"].build(value) - sampler = torch.Generator().manual_seed(context.run.seed + context.index) - state = context.resume_state or {} - transaction, env_steps = state.get("transaction", 0), state.get("env_steps", 0) - metrics, evaluations = ( - list(state.get("metrics", [])), - list(state.get("evaluations", [])), - ) + session = PhaseSession(context, modules={"value": value}) + state = session.state + env_steps = state.get("env_steps", 0) + evaluations = list(state.get("evaluations", [])) continuations = list(state.get("continuations", [])) - clocks = dict(state.get("clocks", {"actor": 0, "value": 0})) if ( - env_steps != min(transaction * phase.rollout_length, phase.total_env_steps) + env_steps + != min(session.transaction * phase.rollout_length, phase.total_env_steps) or not 0 <= env_steps <= phase.total_env_steps ): raise ValueError("PPO checkpoint is not a complete collection/update boundary") - if state: - if any( - not isinstance(clock, int) - or isinstance(clock, bool) - or not transaction <= clock <= phase.optimizer_step_budget() - for clock in clocks.values() - ) or set(clocks) != {"actor", "value"}: - raise ValueError("PPO optimizer clocks disagree with committed progress") - value.load_state_dict(state["value"], strict=True) - value_optimizer.load_state_dict(state["value_optimizer"]) - value_scheduler.load_state_dict(state["value_scheduler"]) - restore_actor_state(context, optimizer, scheduler) - if context.resume_state is not None: - sampler.set_state(context.resume_state["sampler"]) - selection = Selection( - phase, - phase.evaluation.metric, - phase.evaluation.direction, - state.get("selection"), - ) - actor.ppo_policy_version = transaction + actor.ppo_policy_version = session.transaction adapter = load_online_adapter(phase.environment) try: if context.run.resume.mode == "exact": @@ -93,59 +58,47 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): collector.load_state_dict(state["collector"], exact=exact) continuations.append( { - "transaction": transaction, + "transaction": session.transaction, "mode": context.run.resume.mode, "trajectory_equivalent": exact, } ) - if state: - restore_rng_state(actor.device, state["random"]) + session.restore_random() while env_steps < phase.total_env_steps: horizon = min(phase.rollout_length, phase.total_env_steps - env_steps) - buffer, episodes = collector.collect(horizon, transaction) + buffer, episodes = collector.collect(horizon, session.transaction) if phase.archive_directory: buffer.archive( Path(phase.archive_directory) - / f"collection-{transaction+1:08d}.json", + / f"collection-{session.transaction+1:08d}.json", include_scores=phase.archive_scores, ) - record = update_ppo( - actor, - value, - buffer, - phase, - {"actor": optimizer, "value": value_optimizer}, - {"actor": scheduler, "value": value_scheduler}, - sampler, - transaction, - clocks, - context.optimization, - ) - env_steps, transaction = env_steps + horizon, transaction + 1 - if ( - transaction % phase.evaluation_interval == 0 - or env_steps == phase.total_env_steps - ): + record = update_ppo(session, value, buffer) + env_steps += horizon + session.transaction += 1 + final = env_steps == phase.total_env_steps + if session.should_evaluate(final): result = evaluate_policy( actor, phase.evaluation, forbidden_adapter=adapter ) - selection.observe(actor, result["selection_value"], transaction) + session.observe(result["selection_value"]) evaluations.append( - {**result, "transaction": transaction, "env_steps": env_steps} + { + **result, + "transaction": session.transaction, + "env_steps": env_steps, + } ) - metrics.append( + session.metrics.append( { **record, - "transaction": transaction, + "transaction": session.transaction, "env_steps": env_steps, "policy_version": actor.ppo_policy_version, "episodes": episodes, } ) - if ( - transaction % phase.checkpoint_interval == 0 - or env_steps == phase.total_env_steps - ): + if session.should_checkpoint(final): environment_state = None if adapter.capabilities.exact_snapshot: require_exact_snapshot(adapter) @@ -156,37 +109,25 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): raise ValueError( "PPO snapshot must be a finite portable state dictionary" ) - context.commit( - actor_state( - context, - optimizer, - scheduler, - selection, - transaction, - metrics, - sampler=sampler.get_state(), - value=clone_weights(value), - value_optimizer=value_optimizer.state_dict(), - value_scheduler=value_scheduler.state_dict(), - clocks=clocks, - env_steps=env_steps, - evaluations=evaluations, - continuations=continuations, - collector=collector.state_dict(), - environment_state=environment_state, - adapter_identity=identity, - ) + session.checkpoint( + final, + env_steps=env_steps, + evaluations=evaluations, + continuations=continuations, + collector=collector.state_dict(), + environment_state=environment_state, + adapter_identity=identity, ) finally: adapter.close() - return selection.finish( - context, - transaction, + return session.finish( { "env_steps": env_steps, - **{f"{key}_optimizer_steps": val for key, val in clocks.items()}, + **{ + f"{role}_optimizer_steps": steps + for role, steps in session.steps.items() + }, }, - metrics, { "adapter": identity, "evaluations": evaluations, diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py index 78ea1a83..7ce40a69 100644 --- a/src/sequifier/rl/ppo_buffer.py +++ b/src/sequifier/rl/ppo_buffer.py @@ -8,10 +8,10 @@ import torch from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.io.atomic import write_json_atomic from sequifier.io.batch import SequifierInputs from sequifier.io.tensors import collate_tensor_records from sequifier.rl.history import frame_storage_bytes, history_inputs -from sequifier.rl.run_state import write_json_atomic @dataclass(frozen=True) diff --git a/src/sequifier/rl/ppo_evaluation.py b/src/sequifier/rl/ppo_evaluation.py index c9f3ce26..2e32889a 100644 --- a/src/sequifier/rl/ppo_evaluation.py +++ b/src/sequifier/rl/ppo_evaluation.py @@ -12,7 +12,7 @@ from sequifier.rl.distributions import action_generator from sequifier.rl.environment import load_online_adapter, validate_reset, validate_step from sequifier.rl.history import PolicyHistory -from sequifier.rl.run_state import capture_rng_state, restore_rng_state +from sequifier.runtime.random_state import capture_rng_state, restore_rng_state class EvaluationTimeout(TimeoutError): diff --git a/src/sequifier/rl/ppo_update.py b/src/sequifier/rl/ppo_update.py index d73aa86f..85089e4a 100644 --- a/src/sequifier/rl/ppo_update.py +++ b/src/sequifier/rl/ppo_update.py @@ -63,18 +63,9 @@ def validate_collection_scores(actor, value, buffer, batch_size): raise ValueError("PPO reconstructed values differ from collection") -def update_ppo( - actor, - value, - buffer, - phase, - optimizers, - schedulers, - sampler, - policy_version, - clocks, - optimization, -): +def update_ppo(session, value, buffer): + actor, phase = session.context.actor, session.context.phase + policy_version, clocks = session.transaction, session.steps buffer.require_version(policy_version) if actor.ppo_policy_version != policy_version: raise ValueError( @@ -83,10 +74,6 @@ def update_ppo( actor.eval() # Keep dropout off while retaining autograd. value.eval() validate_collection_scores(actor, value, buffer, phase.minibatch_size) - parameters = { - "actor": [p for p in actor.parameters() if p.requires_grad], - "value": [p for p in value.parameters() if p.requires_grad], - } totals = { key: 0.0 for key in ( @@ -101,14 +88,10 @@ def update_ppo( count = 0 norms = {"actor": [], "value": []} stopped = False - gradient_limits = { - "actor": optimization["actor"].gradient_clip, - "value": optimization["value"].gradient_clip, - } group_size = phase.minibatch_size * phase.accumulation_steps epochs = 0 for epoch in range(phase.optimization_epochs): - order = torch.randperm(len(buffer), generator=sampler).tolist() + order = torch.randperm(len(buffer), generator=session.sampler).tolist() for start in range(0, len(order), group_size): group = order[start : start + group_size] minibatches = [ @@ -130,8 +113,7 @@ def update_ppo( if kl_sum / len(group) > phase.target_kl: stopped = True break - for optimizer in optimizers.values(): - optimizer.zero_grad(set_to_none=True) + session.zero_grad() for indices in minibatches: history, actions, old_logp, old_v, advantages, target = buffer.batch( indices, actor.device @@ -159,25 +141,8 @@ def update_ppo( totals[key] += float(metrics[key].detach().sum()) ratio_chunks.append(metrics["ratio"].detach().cpu()) count += len(indices) - for role in ("actor", "value"): - norm = torch.nn.utils.clip_grad_norm_( - parameters[role], - gradient_limits[role], - error_if_nonfinite=True, - ) - norms[role].append(float(norm.detach())) - for role in ("actor", "value"): - optimizers[role].step() - schedulers[role].step() - clocks[role] += 1 - optimizers[role].zero_grad(set_to_none=True) - if any( - not torch.isfinite(parameter).all() - for parameter in parameters[role] - ): - raise ValueError( - f"PPO {role} optimizer produced non-finite parameters" - ) + for role, norm in session.step().items(): + norms[role].append(norm) epochs = epoch + 1 if stopped: break diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index bf82093a..e2134d29 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -8,6 +8,7 @@ from sequifier.io.policy_dataset import PolicyDataCatalog from sequifier.io.prepared_policy_data import PreparedPolicyData from sequifier.model.factory import build_transformer_network +from sequifier.rl.checkpoint import phase_actor_path from sequifier.rl.environment import ( adapter_identity, load_online_adapter, @@ -17,7 +18,8 @@ from sequifier.rl.policy import CategoricalPolicy, policy_execution_config from sequifier.rl.ppo_evaluation import require_evaluation_timer from sequifier.rl.registry import BUILTIN_ALGORITHMS -from sequifier.rl.run_state import capture_rng_state, require_equal, restore_rng_state +from sequifier.rl.run_state import require_equal +from sequifier.runtime.random_state import capture_rng_state, restore_rng_state from sequifier.trajectory.sources import ( ResolvedSequenceSource, file_checksum, @@ -74,10 +76,8 @@ def validate_output_paths(config, inputs): Path(config.checkpoint_path), ] for i, phase in enumerate(config.phases): - directory = Path(config.phase_directory) / f"{i:03d}-{phase.name}" - outputs += [ - directory / name for name in ("actor.pt", "report.json", "manifest.json") - ] + actor = phase_actor_path(config.phase_directory, i, phase.name) + outputs += [actor, actor.with_name("report.json")] outputs = [path.resolve() for path in outputs] archives = [ Path(phase.archive_directory).resolve() diff --git a/src/sequifier/rl/registry.py b/src/sequifier/rl/registry.py index 1874b999..56f8ab15 100644 --- a/src/sequifier/rl/registry.py +++ b/src/sequifier/rl/registry.py @@ -17,7 +17,7 @@ class BuiltinAlgorithm: runtime: Callable prepare_data: Callable[..., PreparedPolicyData] | None = None - state_version: int = 1 + state_version: int = 2 BUILTIN_ALGORITHMS = { diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 1ed76907..444ef864 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -6,17 +6,19 @@ import torch -from sequifier.rl.phase_artifacts import PhaseArtifacts +from sequifier.fingerprints import fingerprint +from sequifier.io.atomic import atomic_save, write_json_atomic +from sequifier.rl.checkpoint import phase_actor_path, save_policy from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_state import ( ActorRepresentation, PolicyPhaseResult, - atomic_save, - capture_rng_state, clone_weights, require_equal, weight_fingerprint, ) +from sequifier.runtime.random_state import capture_rng_state +from sequifier.trajectory.sources import file_checksum @dataclass @@ -38,14 +40,6 @@ class RunCheckpoint: def phase_index(self): return len(self.results) - @property - def status(self): - if self.phase_index == len(self.plan["phases"]): - return "complete" - if self.phase_state is not None: - return "running" - return "handoff" if self.results else "ready" - @classmethod def load(cls, path, config, plan, data_identity): path = Path(path) @@ -59,9 +53,9 @@ def load(cls, path, config, plan, data_identity): if ( not isinstance(payload, dict) or payload.get("artifact_type") != "sequifier_policy_run" - or payload.get("format_version") != 4 + or payload.get("format_version") != 5 ): - raise ValueError("policy resume requires a version-4 policy run checkpoint") + raise ValueError("policy resume requires a version-5 policy run checkpoint") try: require_equal(plan, payload["plan"], "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") @@ -101,7 +95,6 @@ def validate(self, config): validate_finite_state(self.incoming_actor, "incoming_actor") # Parsing the owned representation also validates the persisted policy schema. self.representation.policy - artifacts = PhaseArtifacts(config.phase_directory, self.representation) for index, result in enumerate(self.results): phase = config.phases[index] require_equal( @@ -109,7 +102,25 @@ def validate(self, config): (result.index, result.name, result.algorithm, result.handoff), "resume.completed_phase", ) - artifacts.validate(result, self.completed_artifacts[index]) + record = self.completed_artifacts[index] + path = phase_actor_path(config.phase_directory, index, phase.name) + require_equal(str(path), record["path"], "resume.phase_actor.path") + require_equal( + str(path), result.provenance["actor_path"], "resume.actor_path" + ) + require_equal( + str(path.with_name("report.json")), + result.provenance["report_path"], + "resume.report_path", + ) + require_equal( + record["sha256"], file_checksum(path), "resume.phase_actor.sha256" + ) + require_equal( + record["result_fingerprint"], + fingerprint(result.as_dict()), + "resume.phase_result", + ) validate_finite_state(result.as_dict(), "completed_phase") incoming_identity = weight_fingerprint(self.incoming_actor) require_equal( @@ -172,7 +183,7 @@ def validate_actor(self, actor, units): if self.phase_state is not None: state = self.phase_state weights += [ - state["actor"], + state["networks"]["actor"], state["reference"], state["selection"]["best_weights"], ] @@ -196,7 +207,7 @@ def save(self, path, device, state=None): path, { "artifact_type": "sequifier_policy_run", - "format_version": 4, + "format_version": 5, "plan": self.plan, "representation": self.representation.canonical, "representation_fingerprint": self.representation.identity, @@ -213,15 +224,31 @@ def save(self, path, device, state=None): }, ) - def finish_phase(self, actor, result, artifacts): + def finish_phase(self, actor, result): require_equal(self.phase_index, result.index, "handoff.phase_index") - record = artifacts.commit(actor, result) + require_equal( + result.selected_weight_hash, + weight_fingerprint(actor.network.state_dict()), + "handoff.selected_actor", + ) + path = result.provenance["actor_path"] + save_policy(actor, path, metadata=result.as_dict()) + record = { + "path": path, + "sha256": file_checksum(path), + "result_fingerprint": fingerprint(result.as_dict()), + } weights = clone_weights(actor.network) self.results.append(result) self.completed_artifacts.append(record) self.incoming_actor = weights self.phase_state = None + def write_reports(self): + """Reports are derived views; deleting one does not invalidate recovery.""" + for result in self.results: + write_json_atomic(result.provenance["report_path"], result.as_dict()) + def validate_finite_state(value, field): """Check algorithm-owned optimizer/network trees before restoring them.""" diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index c28a2025..8f4058c9 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -3,11 +3,8 @@ import hashlib import json import math -import os -import uuid from copy import deepcopy from dataclasses import dataclass -from pathlib import Path from typing import TYPE_CHECKING, Callable, Generic, TypeVar import torch @@ -16,7 +13,7 @@ from sequifier.config.policy_optimization import OptimizationPlan from sequifier.config.policy_plan import PhaseConfig from sequifier.rl.policy import CategoricalPolicy -from sequifier.runtime.random_state import RandomState, RandomStateManager +from sequifier.runtime.random_state import capture_rng_state, restore_rng_state if TYPE_CHECKING: from sequifier.config.train_config import PolicyRunConfig @@ -42,70 +39,6 @@ def weight_fingerprint(weights): return digest.hexdigest() -def atomic_save(path, payload): - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - torch.save(payload, temporary) - temporary.replace(path) - sync_file(path) - finally: - temporary.unlink(missing_ok=True) - - -def write_json_atomic(path, payload): - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - temporary.write_text( - json.dumps(payload, indent=2, allow_nan=False) + "\n", encoding="utf-8" - ) - temporary.replace(path) - sync_file(path) - finally: - temporary.unlink(missing_ok=True) - - -def sync_file(path): - """Flush an atomically published file before committing a reference to it.""" - path = Path(path) - with path.open("rb") as handle: - os.fsync(handle.fileno()) - if os.name != "nt": - descriptor = os.open(path.parent, os.O_RDONLY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - -def capture_rng_state(device): - return { - "local": RandomStateManager(device).capture_local().state_dict(), - "cuda_all": torch.cuda.get_rng_state_all() - if torch.cuda.is_available() - else None, - "mps": torch.mps.get_rng_state() if torch.backends.mps.is_available() else None, - } - - -def restore_rng_state(device, state): - RandomStateManager(device).restore(RandomState.from_state_dict(state["local"])) - if state["cuda_all"] is not None: - if ( - not torch.cuda.is_available() - or len(state["cuda_all"]) != torch.cuda.device_count() - ): - raise ValueError("policy checkpoint CUDA RNG topology differs") - torch.cuda.set_rng_state_all(state["cuda_all"]) - if state["mps"] is not None: - if not torch.backends.mps.is_available(): - raise ValueError("policy checkpoint requires the saved MPS RNG") - torch.mps.set_rng_state(state["mps"]) - - def require_equal(expected, actual, field): """Locate the first mismatching contract field rather than a generic failure.""" if isinstance(expected, dict) and isinstance(actual, dict): @@ -270,50 +203,132 @@ def finish(self, context, transaction, clocks, metrics, provenance): return actor, result -def apply_gradients(module, optimizer, scheduler, clip): - torch.nn.utils.clip_grad_norm_(module.parameters(), clip, error_if_nonfinite=True) - optimizer.step() - if any(not torch.isfinite(p).all() for p in module.parameters()): - raise ValueError( - "policy optimizer produced non-finite parameters; transaction was not committed" - ) - scheduler.step() - optimizer.zero_grad(set_to_none=True) +class PhaseSession: + """Own common progress and optimization; algorithms own their update loops.""" + def __init__(self, context, *, modules=None, frozen=None): + self.context = context + self.modules = {"actor": context.actor.network, **(modules or {})} + self.networks = {**self.modules, **(frozen or {})} + if self.modules.keys() != context.optimization.keys(): + raise ValueError("phase optimization roles disagree with trainable modules") + self.optimization = { + role: context.optimization[role].build(module) + for role, module in self.modules.items() + } + self.steps = dict.fromkeys(self.modules, 0) + self.sampler = torch.Generator().manual_seed(context.run.seed + context.index) + self.state = context.resume_state or {} + self.transaction = self.state.get("transaction", 0) + self.metrics = list(self.state.get("metrics", [])) + selection = getattr(context.phase, "evaluation", context.phase) + monitor = getattr(selection, "metric", getattr(selection, "monitor", None)) + self.selection = Selection( + context.phase, monitor, selection.direction, self.state.get("selection") + ) + if self.state: + require_equal( + sorted(self.networks), sorted(self.state["networks"]), "phase.networks" + ) + require_equal( + sorted(self.optimization), + sorted(self.state["optimization"]), + "phase.optimization", + ) + for name, module in self.networks.items(): + module.load_state_dict(self.state["networks"][name], strict=True) + for role, (optimizer, scheduler) in self.optimization.items(): + saved = self.state["optimization"][role] + optimizer.load_state_dict(saved["optimizer"]) + scheduler.load_state_dict(saved["scheduler"]) + steps = saved["steps"] + budget = context.optimization[role].step_budget + if ( + (not isinstance(steps, int)) + or steps < self.transaction + or (budget is not None and steps > budget) + ): + raise ValueError( + f"{role} optimizer clock exceeds committed progress/budget" + ) + self.steps[role] = steps + self.sampler.set_state(self.state["sampler"]) + self.restore_random() + + def restore_random(self): + if self.state: + restore_rng_state(self.context.actor.device, self.state["random"]) + + def zero_grad(self): + for optimizer, _ in self.optimization.values(): + optimizer.zero_grad(set_to_none=True) + + def step(self): + # Check every gradient before mutating any participating network. + norms = { + role: float( + torch.nn.utils.clip_grad_norm_( + module.parameters(), + self.context.optimization[role].gradient_clip, + error_if_nonfinite=True, + ) + ) + for role, module in self.modules.items() + } + for role, (optimizer, scheduler) in self.optimization.items(): + optimizer.step() + if any( + not torch.isfinite(p).all() for p in self.modules[role].parameters() + ): + raise ValueError( + f"{role} optimizer produced non-finite parameters; transaction was not committed" + ) + scheduler.step() + self.steps[role] += 1 + self.zero_grad() + return norms -def phase_events(phase, transaction, *, final): - """Evaluation and checkpointing use the committed transaction clock.""" - return ( - final or transaction % phase.evaluation_interval == 0, - final or transaction % phase.checkpoint_interval == 0, - ) + def should_evaluate(self, final): + return final or self.transaction % self.context.phase.evaluation_interval == 0 + def observe(self, metric): + self.selection.observe(self.context.actor, metric, self.transaction) -def actor_state( - context, optimizer, scheduler, selection, transaction, metrics, **extra -): - return { - "version": 1, - "actor": clone_weights(context.actor.network), - "optimizer": optimizer.state_dict(), - "scheduler": scheduler.state_dict(), - "selection": selection.state_dict(), - "transaction": transaction, - "metrics": metrics, - "reference": clone_weights(context.reference.network) - if context.reference is not None - else None, - "reference_identity": context.reference_identity, - "random": capture_rng_state(context.actor.device), - **extra, - } + def should_checkpoint(self, final): + return final or self.transaction % self.context.phase.checkpoint_interval == 0 + def checkpoint(self, final, **extra): + if not self.should_checkpoint(final): + return + self.context.commit( + { + **extra, + "version": 2, + "networks": { + name: clone_weights(module) + for name, module in self.networks.items() + }, + "optimization": { + role: { + "optimizer": optimizer.state_dict(), + "scheduler": scheduler.state_dict(), + "steps": self.steps[role], + } + for role, (optimizer, scheduler) in self.optimization.items() + }, + "sampler": self.sampler.get_state(), + "selection": self.selection.state_dict(), + "transaction": self.transaction, + "metrics": self.metrics, + "reference": clone_weights(self.context.reference.network) + if self.context.reference is not None + else None, + "reference_identity": self.context.reference_identity, + "random": capture_rng_state(self.context.actor.device), + } + ) -def restore_actor_state(context, optimizer, scheduler): - state = context.resume_state - if state is None: - return - context.actor.network.load_state_dict(state["actor"], strict=True) - optimizer.load_state_dict(state["optimizer"]) - scheduler.load_state_dict(state["scheduler"]) - restore_rng_state(context.actor.device, state["random"]) + def finish(self, clocks, provenance): + return self.selection.finish( + self.context, self.transaction, clocks, self.metrics, provenance + ) diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index c23ecaf1..4eace4ac 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -1,193 +1,174 @@ -"""Return-conditioned actors executed through the ordinary rollout coordinator.""" +"""Policy rollout through the same environment and causal history used in training.""" -import torch +import math +import warnings +from copy import deepcopy +from uuid import uuid4 -from sequifier.config.rollout_config import RolloutConfig -from sequifier.io.policy_window import PolicyWindowView -from sequifier.io.values import DomainValue, domain_fields -from sequifier.rl.distributions import CategoricalActionDistribution, action_generator +from sequifier.config.policy_infer_config import PolicyRolloutConfig +from sequifier.config.rollout_config import InputBinding +from sequifier.io.values import numeric_scalar +from sequifier.rl.collection import decide, validate_online_inputs +from sequifier.rl.distributions import action_generator +from sequifier.rl.environment import ( + OnlineEnvironmentAdapter, + OnlineResetResult, + require_seeded_reset, + validate_online_adapter, + validate_reset, + validate_step, +) from sequifier.rl.history import PolicyHistory from sequifier.rl.policy import CategoricalPolicy -from sequifier.rollout.contracts import EnvironmentAdapter, ResetRequest -from sequifier.rollout.coordinator import collect_rollout -from sequifier.rollout.history import numeric_scalar from sequifier.trajectory.returns import remaining_return -from sequifier.trajectory.schema import ( - SeedHistory, - Trajectory, - Transition, - validate_trajectory, -) - - -class PolicyExecutor: - """Callable predictor/observer; current actions enter history only after step().""" - - def __init__( - self, - policy: CategoricalPolicy, - desired_return: float | None, - deterministic: bool = True, - ): - self.policy = policy - self.initial_return = desired_return - self.deterministic = deterministic - self.active = False - - def reset(self, request: ResetRequest) -> None: - spec = self.policy.config - if not isinstance(self.deterministic, bool): - raise ValueError("policy deterministic selection must be a boolean") - if sum(request.attention_valid_mask) != 1: - raise ValueError( - "policy reset requires one current observation; prior actions are unknown" - ) - if not spec.return_conditioned and self.initial_return is not None: - raise ValueError("behavior cloning has no desired-return input") - self.previous_action = len(spec.action_values) - self.return_trace = ( - [numeric_scalar(self.initial_return, "initial desired_return")] - if spec.return_conditioned - else [] - ) - self.history = PolicyHistory(spec) - self.history.reset( - {column: values[-1] for column, values in request.seed_history.items()}, - request.static_inputs, - request.episode_id, - self.return_trace[0] if spec.return_conditioned else None, - ) - self.previous_action_trace = [] - self.log_probabilities = [] - self.generator = action_generator(self.policy.device, request.seed) - self.pending = None - self.ended = False - self.active = True - self.episode_id = request.episode_id - self.step_id = 0 - - def __call__(self, history: SeedHistory) -> dict[str, DomainValue]: - if not self.active or self.pending is not None or self.ended: - raise ValueError( - "policy predictions require an active episode and an executed previous prediction" - ) - spec = self.policy.config - observation = { - column: values[-1] for column, values in history.observations.items() - } - if observation != dict( - self.history.current.observation - ) or history.static_inputs != dict(self.history.current.static_inputs): - raise ValueError("policy causal history disagrees with rollout history") - inputs = self.history.inputs() - with torch.no_grad(): - distribution = CategoricalActionDistribution( - self.policy(inputs.to(self.policy.device))[:, -1, :] - ) - action = distribution.select(self.deterministic, self.generator) - log_probability = float(distribution.log_prob(action)[0]) - index = int(action[0]) - self.pending = (observation, index, log_probability) - return {spec.action_column: spec.action_values[index]} - - def observe(self, transition: Transition) -> None: - if not self.active or self.pending is None: - raise ValueError("policy observed a transition without a predicted action") - observation, index, log_probability = self.pending - spec = self.policy.config - if ( - transition.episode_id != self.episode_id - or transition.step_id != self.step_id - ): - raise ValueError("policy transition episode/step IDs are out of sync") - if transition.observation != observation or transition.predictions != { - spec.action_column: spec.action_values[index] - }: - raise ValueError("executed transition does not match the policy decision") - self.previous_action_trace.append(self.previous_action) - self.log_probabilities.append(log_probability) - if spec.return_conditioned: - self.return_trace.append( - remaining_return( - self.return_trace[-1], transition.reward, spec.returns.gamma - ) - ) - self.history.advance( - index, - transition.successor_observation, - self.return_trace[-1] if spec.return_conditioned else None, - ) - self.previous_action = index - self.pending = None - self.ended = transition.terminated or transition.truncated - self.step_id += 1 +from sequifier.trajectory.schema import SeedHistory, Trajectory, Transition def collect_policy_rollout( policy: CategoricalPolicy, - adapter: EnvironmentAdapter, + adapter: OnlineEnvironmentAdapter, seed: SeedHistory, - config: RolloutConfig, + config: PolicyRolloutConfig, max_steps: int, *, desired_return: float | None = None, deterministic: bool = True, random_seed: int = 1010, ) -> Trajectory: - """Collect one reset-initialized policy episode/fragment and always close adapter. - - Each call/reset starts the requested budget again. Time limits and collection - cutoffs retain the coordinator's distinct boundary semantics. - """ - executor = PolicyExecutor(policy, desired_return, deterministic) - spec = policy.config - - def validate_history(history): - if config.mode != "policy": - raise ValueError("categorical execution requires rollout mode: policy") - horizon = spec.returns.finite_horizon if spec.return_conditioned else None - if horizon is not None and config.max_episode_steps != horizon: + """Execute a supplied initial observation, preserving actual episode boundaries.""" + was_training, failed = policy.training, False + try: + validate_online_adapter(adapter, config.environment) + require_seeded_reset(adapter) + if (not isinstance(max_steps, int)) or max_steps < 1: + raise ValueError("policy collection budget must be a positive integer") + if not isinstance(deterministic, bool): + raise ValueError("policy deterministic selection must be a boolean") + spec, units = policy.config, config.environment.units + if units != getattr(policy, "units", {}): + raise ValueError("rollout environment units differ from the policy") + seed = SeedHistory.model_validate(seed.model_dump()) + if ( + sum(seed.attention_valid_mask) != 1 + or len(seed.attention_valid_mask) != spec.context_length + ): raise ValueError( - "finite-horizon policy execution requires max_episode_steps equal to its return horizon" + "policy seed requires one current observation in its context window" ) - observation = { - column: values[-1] for column, values in history.observations.items() - } - domain_fields(observation, set(spec.observation_columns), "policy observation") - # Validate the final successor too, even when no subsequent action is needed. - inputs = PolicyWindowView(spec).inputs( - [observation], - [len(spec.action_values)], + if not spec.return_conditioned and desired_return is not None: + raise ValueError("behavior cloning has no desired-return input") + returns = ( [numeric_scalar(desired_return, "desired_return")] if spec.return_conditioned - else None, - history.static_inputs, + else [] ) - policy.validate_inputs(inputs.to(policy.device)) - - was_training = policy.training - policy.eval() - try: - trajectory = collect_rollout( - executor, - adapter, - seed, - config, - [spec.action_column], - max_steps, - random_seed=random_seed, - history_validator=validate_history, - observer=executor, - metadata={"policy_alignment": spec.alignment}, + if ( + spec.return_conditioned + and spec.returns.finite_horizon is not None + and config.max_episode_steps != spec.returns.finite_horizon + ): + raise ValueError( + "policy max_episode_steps must equal its finite return horizon" + ) + initial = OnlineResetResult( + {column: values[-1] for column, values in seed.observations.items()}, + seed.static_inputs, + units, + ) + episode_id = str(uuid4()) + history = PolicyHistory(spec) + history.reset( + initial.observation, + initial.static_inputs, + episode_id, + returns[0] if returns else None, + ) + policy.eval() + validate_online_inputs(policy, history.inputs()) + reset = validate_reset( + adapter.reset_from_observation(deepcopy(initial), random_seed), units + ) + if dict(reset.observation) != dict(initial.observation) or dict( + reset.static_inputs + ) != dict(initial.static_inputs): + raise ValueError( + "seeded reset must preserve the supplied observation and static inputs" + ) + generator = action_generator(policy.device, random_seed) + transitions, previous_actions, log_probabilities = [], [], [] + for step_id in range(max_steps): + current = history.current + decision = decide( + policy, None, history.inputs(), generator, deterministic=deterministic + ) + result = validate_step(adapter.step(decision.domain_action), units) + truncated = result.truncated or ( + config.max_episode_steps is not None + and step_id + 1 >= config.max_episode_steps + ) + previous_actions.append(current.previous_action) + log_probabilities.append(decision.log_probability) + if spec.return_conditioned: + returns.append( + remaining_return(returns[-1], result.reward, spec.returns.gamma) + ) + successor = ( + dict(result.observation) if result.observation is not None else None + ) + if successor is not None: + history.advance( + decision.action, successor, returns[-1] if returns else None + ) + validate_online_inputs(policy, history.inputs()) + transitions.append( + Transition( + episode_id=episode_id, + step_id=step_id, + observation=dict(current.observation), + predictions=decision.domain_action, + external_inputs=successor or {}, + reward=result.reward, + successor_observation=successor, + terminated=result.terminated, + truncated=truncated, + bootstrap_allowed=not result.terminated, + ) + ) + if result.terminated or truncated: + break + last = transitions[-1] + return Trajectory( + format="sequifier.scalar-policy-trajectory", + episode_id=episode_id, + seed=seed, + bindings=[ + InputBinding(input=name, source="environment", key=name) + for name in spec.observation_columns + ], + prediction_columns=[spec.action_column], + transitions=transitions, + collection_cutoff=not (last.terminated or last.truncated), + total_return=math.fsum(step.reward for step in transitions), + metadata={ + "policy_alignment": spec.alignment, + "policy_execution": { + "config": spec.model_dump(mode="json"), + "initial_desired_return": desired_return, + "remaining_returns": returns, + "previous_action_ids": previous_actions, + "action_log_probabilities": log_probabilities, + "deterministic": deterministic, + }, + }, ) + except BaseException: + failed = True + raise finally: policy.train(was_training) - trajectory.metadata["policy_execution"] = { - "config": spec.model_dump(mode="json"), - "initial_desired_return": desired_return, - "remaining_returns": executor.return_trace, - "previous_action_ids": executor.previous_action_trace, - "action_log_probabilities": executor.log_probabilities, - "deterministic": deterministic, - } - return validate_trajectory(trajectory) + try: + adapter.close() + except Exception as exc: + if not failed: + raise + warnings.warn(f"Adapter close also failed: {exc}", RuntimeWarning) diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 7a8bd55c..16105fba 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -7,14 +7,7 @@ from sequifier.io.policy_window import action_global_ids, action_target from sequifier.rl.batch_order import BatchOrder from sequifier.rl.offline_data import OfflineData -from sequifier.rl.run_state import ( - PhaseContext, - Selection, - actor_state, - apply_gradients, - phase_events, - restore_actor_state, -) +from sequifier.rl.run_state import PhaseContext, PhaseSession def action_nll(actor, batch): @@ -39,12 +32,8 @@ def evaluate_imitation(actor, dataset, batch_size): def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, OfflineData]): actor, phase, data = context.actor.eval(), context.phase, context.data - optimizer, scheduler = context.optimization["actor"].build(actor) - restore_actor_state(context, optimizer, scheduler) - state = context.resume_state or {} - selection = Selection(phase, phase.monitor, phase.direction, state.get("selection")) - transaction = state.get("transaction", 0) - metrics = list(state.get("metrics", [])) + session = PhaseSession(context) + state = session.state training = data.training batches = BatchOrder( len(training), @@ -53,13 +42,17 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin sample=training.sampler, state=state["batch_order"] if state else None, ) - if batches.epoch > phase.epochs or transaction != batches.transactions: + if ( + batches.epoch > phase.epochs + or session.transaction != batches.transactions + or session.steps["actor"] != session.transaction + ): raise ValueError("imitation checkpoint epoch/transaction clocks disagree") if batches.epoch == phase.epochs and batches.order is not None: raise ValueError("completed imitation checkpoint has a live sampler order") while batches.epoch < phase.epochs: group = batches.next() - optimizer.zero_grad(set_to_none=True) + session.zero_grad() total = 0.0 for start in range(0, len(group), phase.batch_size): indices = group[start : start + phase.batch_size] @@ -69,40 +62,22 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin raise ValueError("non-finite imitation action likelihood") loss.backward() total += float(loss.detach()) - apply_gradients( - actor, optimizer, scheduler, context.optimization["actor"].gradient_clip - ) - transaction += 1 - evaluate, checkpoint = phase_events( - phase, transaction, final=batches.epoch == phase.epochs - ) - if evaluate: + session.step() + session.transaction += 1 + final = batches.epoch == phase.epochs + if session.should_evaluate(final): score = evaluate_imitation(actor, data.validation, phase.batch_size) - selection.observe(actor, score, transaction) - metrics.append( + session.observe(score) + session.metrics.append( { - "transaction": transaction, + "transaction": session.transaction, "epochs_completed": batches.epoch, "training_action_nll": total, phase.monitor: score, } ) - if checkpoint: - context.commit( - actor_state( - context, - optimizer, - scheduler, - selection, - transaction, - metrics, - batch_order=batches.state_dict(), - ) - ) - return selection.finish( - context, - transaction, - {"optimizer_steps": transaction, "epochs": batches.epoch}, - metrics, + session.checkpoint(final, batch_order=batches.state_dict()) + return session.finish( + {"optimizer_steps": session.steps["actor"], "epochs": batches.epoch}, {"data": data.contract}, ) diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index 13bc7857..cf4e6691 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -5,7 +5,11 @@ PolicySchemaConfig, ReturnDefinition, ) -from sequifier.config.policy_infer_config import PolicyInferenceConfig +from sequifier.config.policy_infer_config import ( + PolicyInferenceConfig, + PolicyRolloutConfig, +) +from sequifier.config.policy_optimization import OptimizationConfig from sequifier.config.policy_plan import ( DPOPhaseConfig, FrozenNormalizationConfig, @@ -39,7 +43,6 @@ OnlineEnvironmentAdapter, OnlineResetResult, OnlineStepResult, - SeededAdapterBridge, ) from sequifier.rl.learner import run_policy_training from sequifier.rl.policy import CategoricalPolicy @@ -50,7 +53,7 @@ PolicyPhaseResult, PolicyRunResult, ) -from sequifier.rl.runtime import PolicyExecutor, collect_policy_rollout +from sequifier.rl.runtime import collect_policy_rollout from sequifier.trajectory.export import import_sequences from sequifier.trajectory.importers import source_episodes from sequifier.trajectory.preferences import ( @@ -77,9 +80,10 @@ "OnlineEvaluationConfig", "OnlineResetResult", "OnlineStepResult", - "PolicyExecutor", "PolicyImitationPhaseConfig", "PolicyInferenceConfig", + "PolicyRolloutConfig", + "OptimizationConfig", "PolicyPhaseResult", "PolicyResumeConfig", "PolicyRunConfig", @@ -94,7 +98,6 @@ "PreferenceSpan", "ReferenceConfig", "ReturnDefinition", - "SeededAdapterBridge", "SequenceEpisode", "SequenceImportConfig", "SequenceSourceConfig", diff --git a/src/sequifier/rollout/contracts.py b/src/sequifier/rollout/contracts.py index a64af01b..14f7ae3a 100644 --- a/src/sequifier/rollout/contracts.py +++ b/src/sequifier/rollout/contracts.py @@ -3,13 +3,10 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Mapping, Protocol +from typing import Mapping, Protocol from sequifier.io.values import DomainValue -if TYPE_CHECKING: - from sequifier.trajectory.schema import Transition - @dataclass(frozen=True) class ResetRequest: @@ -47,11 +44,3 @@ def reset(self, request: ResetRequest) -> None: def step(self, request: StepRequest) -> StepResult: ... def close(self) -> None: ... - - -class RolloutObserver(Protocol): - """Per-episode state updated only after an executed, recorded transition.""" - - def reset(self, request: ResetRequest) -> None: ... - - def observe(self, transition: Transition) -> None: ... diff --git a/src/sequifier/rollout/coordinator.py b/src/sequifier/rollout/coordinator.py index db83173a..7a18632f 100644 --- a/src/sequifier/rollout/coordinator.py +++ b/src/sequifier/rollout/coordinator.py @@ -7,11 +7,10 @@ from uuid import uuid4 from sequifier.config.rollout_config import RolloutConfig -from sequifier.io.values import DomainValue, domain_fields +from sequifier.io.values import DomainValue from sequifier.rollout.contracts import ( EnvironmentAdapter, ResetRequest, - RolloutObserver, StepRequest, StepResult, ) @@ -25,16 +24,6 @@ from sequifier.trajectory.schema import SeedHistory, Trajectory, Transition -def coordinate_steps(predict: Callable, execute: Callable, max_steps: int) -> None: - """Execute every prediction, including the final budgeted one; stop at a boundary.""" - if not isinstance(max_steps, int) or isinstance(max_steps, bool) or max_steps < 1: - raise ValueError("rollout collection budget must be a positive integer") - for step_id in range(max_steps): - prediction = predict() - if execute(prediction, step_id): - break - - def collect_rollout( predictor: Callable, adapter: EnvironmentAdapter, @@ -46,7 +35,6 @@ def collect_rollout( random_seed: int = 1010, metadata: dict | None = None, history_validator: Callable | None = None, - observer: RolloutObserver | None = None, ) -> Trajectory: """Collect one episode fragment. This call owns and always closes the adapter. @@ -55,19 +43,19 @@ def collect_rollout( predictor wrapper, using fixed training metadata. """ failed = False - fields = domain_fields if config.mode == "policy" else numeric_fields try: + if (not isinstance(max_steps, int)) or max_steps < 1: + raise ValueError("rollout collection budget must be a positive integer") validate_adapter(adapter) history = validate_seed(seed) - if config.mode != "policy": - for column, values in history.observations.items(): - for value in values: - numeric_scalar(value, f"forecast seed.{column}") - numeric_fields( - history.static_inputs, - set(history.static_inputs), - "forecast static inputs", - ) + for column, values in history.observations.items(): + for value in values: + numeric_scalar(value, f"forecast seed.{column}") + numeric_fields( + history.static_inputs, + set(history.static_inputs), + "forecast static inputs", + ) initial_history = history.model_copy(deep=True) if history_validator is not None: history_validator(history.model_copy(deep=True)) @@ -85,8 +73,6 @@ def collect_rollout( random_seed, ) adapter.reset(deepcopy(reset_request)) - if observer is not None: - observer.reset(deepcopy(reset_request)) transitions = [] external_keys = { binding.key @@ -94,15 +80,12 @@ def collect_rollout( if binding.source == "environment" } - def predict(): - return fields( + for step_id in range(max_steps): + predictions = numeric_fields( predictor(history.model_copy(deep=True)), set(prediction_columns), "predictions", ) - - def execute(predictions, step_id): - nonlocal history observation = { key: values[-1] for key, values in history.observations.items() } @@ -125,7 +108,7 @@ def execute(predictions, step_id): ) external: dict[str, DomainValue] = { key: value - for key, value in fields( + for key, value in numeric_fields( result.external_inputs, external_keys, "adapter.external_inputs" ).items() } @@ -157,19 +140,12 @@ def execute(predictions, step_id): bootstrap_allowed=not result.terminated, ) ) - if observer is not None: - observer.observe(transitions[-1].model_copy(deep=True)) history = next_history - return result.terminated or truncated - - coordinate_steps(predict, execute, max_steps) + if result.terminated or truncated: + break last = transitions[-1] return Trajectory( - format=( - "sequifier.scalar-policy-trajectory" - if config.mode == "policy" - else "sequifier.numeric-forecast-trajectory" - ), + format="sequifier.numeric-forecast-trajectory", episode_id=episode_id, seed=initial_history, bindings=config.bindings, diff --git a/src/sequifier/rollout_api.py b/src/sequifier/rollout_api.py index 83ac4586..0c2ec045 100644 --- a/src/sequifier/rollout_api.py +++ b/src/sequifier/rollout_api.py @@ -4,7 +4,6 @@ from sequifier.rollout.contracts import ( EnvironmentAdapter, ResetRequest, - RolloutObserver, StepRequest, StepResult, ) @@ -23,7 +22,6 @@ "RolloutConfig", "EnvironmentAdapter", "ResetRequest", - "RolloutObserver", "StepRequest", "StepResult", "SeedHistory", diff --git a/src/sequifier/runtime/random_state.py b/src/sequifier/runtime/random_state.py index 7a97128c..5827d419 100644 --- a/src/sequifier/runtime/random_state.py +++ b/src/sequifier/runtime/random_state.py @@ -19,6 +19,7 @@ class RandomState: torch_cpu: torch.Tensor cuda: tuple[torch.Tensor, ...] | None mps: torch.Tensor | None = None + all_devices: bool = False def state_dict(self) -> dict[str, Any]: return { @@ -27,6 +28,7 @@ def state_dict(self) -> dict[str, Any]: "torch_cpu": self.torch_cpu, "cuda": self.cuda, "mps": self.mps, + "all_devices": self.all_devices, } @classmethod @@ -38,21 +40,25 @@ class RandomStateManager: def __init__(self, device: torch.device) -> None: self.device = device - def capture_local(self) -> RandomState: + def capture_local(self, *, all_devices: bool = False) -> RandomState: return RandomState( python=random.getstate(), numpy=np.random.get_state(), torch_cpu=torch.get_rng_state(), cuda=( - (torch.cuda.get_rng_state(self.device),) + tuple(torch.cuda.get_rng_state_all()) + if all_devices and torch.cuda.is_available() + else (torch.cuda.get_rng_state(self.device),) if self.device.type == "cuda" and torch.cuda.is_available() else None ), mps=( torch.mps.get_rng_state() - if self.device.type == "mps" and torch.backends.mps.is_available() + if (all_devices or self.device.type == "mps") + and torch.backends.mps.is_available() else None ), + all_devices=all_devices, ) def gather(self, distributed: Any) -> DistributedRandomState: @@ -69,10 +75,20 @@ def select_for_rank(self, state: DistributedRandomState, rank: int) -> RandomSta return RandomState.from_state_dict(state.states[rank]) def restore(self, state: RandomState) -> None: + if state.all_devices: + if state.cuda is not None and ( + not torch.cuda.is_available() + or len(state.cuda) != torch.cuda.device_count() + ): + raise ValueError("checkpoint CUDA RNG topology differs") + if state.mps is not None and not torch.backends.mps.is_available(): + raise ValueError("checkpoint requires the saved MPS RNG") random.setstate(state.python) np.random.set_state(state.numpy) torch.set_rng_state(state.torch_cpu) - if ( + if state.all_devices and state.cuda is not None: + torch.cuda.set_rng_state_all(list(state.cuda)) + elif ( state.cuda is not None and self.device.type == "cuda" and torch.cuda.is_available() @@ -86,7 +102,15 @@ def restore(self, state: RandomState) -> None: torch.cuda.set_rng_state(cuda_state, self.device) if ( state.mps is not None - and self.device.type == "mps" + and (state.all_devices or self.device.type == "mps") and torch.backends.mps.is_available() ): torch.mps.set_rng_state(state.mps) + + +def capture_rng_state(device): + return RandomStateManager(device).capture_local(all_devices=True).state_dict() + + +def restore_rng_state(device, state): + RandomStateManager(device).restore(RandomState.from_state_dict(state)) diff --git a/src/sequifier/templates/infer-policy.yaml b/src/sequifier/templates/infer-policy.yaml new file mode 100644 index 00000000..f3a84faf --- /dev/null +++ b/src/sequifier/templates/infer-policy.yaml @@ -0,0 +1,10 @@ +project_root: . +model_path: models/policy/actor.pt +data_path: PLEASE_FILL +model_type: generative +read_format: csv +write_format: csv +device: cpu +deterministic: true +# Omit desired_return for behavior cloning. +desired_return: 10.0 diff --git a/documentation/example_inputs/configs/train-dpo-control.yaml b/src/sequifier/templates/train-dpo.yaml similarity index 100% rename from documentation/example_inputs/configs/train-dpo-control.yaml rename to src/sequifier/templates/train-dpo.yaml diff --git a/documentation/example_inputs/configs/train-grpo-control.yaml b/src/sequifier/templates/train-grpo.yaml similarity index 100% rename from documentation/example_inputs/configs/train-grpo-control.yaml rename to src/sequifier/templates/train-grpo.yaml diff --git a/src/sequifier/templates/train-iql.yaml b/src/sequifier/templates/train-iql.yaml new file mode 100644 index 00000000..9fd42196 --- /dev/null +++ b/src/sequifier/templates/train-iql.yaml @@ -0,0 +1,45 @@ +project_root: . +model_name: offline-iql +device: cpu +seed: 1010 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 16 + attention: {n_heads: 4} + feed_forward: {dim: 128} + positional_encoding_scope: global + interfaces: + actor: + input_columns: [OBSERVATION_COLUMN] + target_columns: [action] + policy: + observation_columns: [OBSERVATION_COLUMN] + action_values: [0.0, 1.0] + return_conditioned: false + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {OBSERVATION_COLUMN: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} +policy_interface: actor +context_length: 16 +sources: + demonstrations.recorded: + paths: [PLEASE_FILL.csv] + columns: + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: {OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN} +validation_fraction: 0.2 +resume: {policy: if_exists} +phases: +- name: offline_improvement + algorithm: iql + sources: [demonstrations.recorded] + updates: 10000 + batch_size: 64 diff --git a/src/sequifier/templates/train-policy.yaml b/src/sequifier/templates/train-policy.yaml new file mode 100644 index 00000000..6f851cb8 --- /dev/null +++ b/src/sequifier/templates/train-policy.yaml @@ -0,0 +1,42 @@ +project_root: . +model_name: policy +device: cpu +seed: 1010 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 4 + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: [OBSERVATION_COLUMN] + target_columns: [action] + policy: + observation_columns: [OBSERVATION_COLUMN] + action_values: [0.0, 1.0] + return_conditioned: true + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} + decoder: {type: linear, prediction_length: 1, support: 1} +policy_interface: actor +context_length: 4 +sources: + demonstrations.recorded: + paths: [PLEASE_FILL.csv] +validation_fraction: 0.2 +episode_cache_size: 8 +resume: {policy: never} +phases: +- algorithm: policy_imitation + epochs: 80 + sources: [demonstrations.recorded] + name: policy_imitation + batch_size: 64 + optimization: {actor: {learning_rate: 0.005}} diff --git a/documentation/example_inputs/configs/train-ppo-toy.yaml b/src/sequifier/templates/train-ppo.yaml similarity index 96% rename from documentation/example_inputs/configs/train-ppo-toy.yaml rename to src/sequifier/templates/train-ppo.yaml index 451140c0..64860f35 100644 --- a/documentation/example_inputs/configs/train-ppo-toy.yaml +++ b/src/sequifier/templates/train-ppo.yaml @@ -44,8 +44,6 @@ phases: gamma: 0.99 gae_lambda: 0.95 clip_epsilon: 0.2 - actor_learning_rate: 0.0003 - value_learning_rate: 0.0003 evaluation_interval: 2 checkpoint_interval: 2 evaluation: diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index 46aaa31b..3742cb35 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -55,7 +55,7 @@ class Transition(Record): predictions: dict[str, DomainValue] external_inputs: dict[str, DomainValue] reward: FiniteFloat - successor_observation: dict[str, DomainValue] + successor_observation: dict[str, DomainValue] | None terminated: bool truncated: bool bootstrap_allowed: bool @@ -106,28 +106,38 @@ def validate_temporal_contract(self): raise ValueError( f"transition {index} prediction fields do not match the contract" ) - if set(transition.external_inputs) != external_keys: - raise ValueError( - f"transition {index} external input fields do not match bindings" - ) - if set(transition.successor_observation) != inputs: - raise ValueError( - f"transition {index} successor fields do not match inputs" - ) - for binding in self.bindings: - producer = ( - transition.predictions - if binding.source == "prediction" - else transition.external_inputs - ) + if transition.successor_observation is None: if ( - binding.key not in producer - or transition.successor_observation[binding.input] - != producer[binding.key] + self.format != "sequifier.scalar-policy-trajectory" + or not transition.terminated + or transition.external_inputs ): raise ValueError( - f"transition {index} successor violates binding for {binding.input!r}" + "only a terminal policy transition may omit its successor" + ) + else: + if set(transition.external_inputs) != external_keys: + raise ValueError( + f"transition {index} external input fields do not match bindings" + ) + if set(transition.successor_observation) != inputs: + raise ValueError( + f"transition {index} successor fields do not match inputs" + ) + for binding in self.bindings: + producer = ( + transition.predictions + if binding.source == "prediction" + else transition.external_inputs ) + if ( + binding.key not in producer + or transition.successor_observation[binding.input] + != producer[binding.key] + ): + raise ValueError( + f"transition {index} successor violates binding for {binding.input!r}" + ) if transition.bootstrap_allowed != (not transition.terminated): raise ValueError( "termination must disable bootstrap; truncation preserves the final observation" diff --git a/src/sequifier/trajectory/store.py b/src/sequifier/trajectory/store.py index 681d4af6..d13dc6e1 100644 --- a/src/sequifier/trajectory/store.py +++ b/src/sequifier/trajectory/store.py @@ -1,32 +1,14 @@ -"""Atomic JSON storage, with validation at both read and write boundaries.""" +"""Validated JSON trajectory storage with shared atomic publication.""" import json -import os -import tempfile from pathlib import Path +from sequifier.io.atomic import write_json_atomic from sequifier.trajectory.schema import Trajectory, validate_trajectory def save_trajectory(trajectory: Trajectory, path: str | Path) -> None: - validated = validate_trajectory(trajectory) - # Reject non-JSON/non-finite metadata as well as invalid numeric records. - payload = json.dumps(validated.model_dump(mode="json"), indent=2, allow_nan=False) - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - temporary = None - try: - with tempfile.NamedTemporaryFile( - mode="w", encoding="utf-8", dir=destination.parent, delete=False - ) as stream: - temporary = stream.name - stream.write(payload + "\n") - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, destination) - finally: - if temporary is not None and os.path.exists(temporary): - os.unlink(temporary) + write_json_atomic(path, validate_trajectory(trajectory).model_dump(mode="json")) def load_trajectory(path: str | Path) -> Trajectory: From 03866667315669eb1179ba9ee114efda71b36329 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 10 Nov 2026 09:07:08 +0100 Subject: [PATCH 22/71] cleanup --- documentation/policy-learning.md | 27 ++- release-notes/unreleased-policy-training.md | 4 +- src/sequifier/config/policy_optimization.py | 34 +-- src/sequifier/config/policy_plan.py | 8 +- src/sequifier/config/train_config.py | 85 +++++++- src/sequifier/config/training_plan.py | 89 -------- src/sequifier/consolidated-docs.md | 27 ++- src/sequifier/io/episode_features.py | 146 +++++++++++++ src/sequifier/io/iql_dataset.py | 32 ++- src/sequifier/io/policy_dataset.py | 193 ++++------------- src/sequifier/io/preference_dataset.py | 6 +- src/sequifier/io/prepared_policy_data.py | 4 +- src/sequifier/rl/collection.py | 111 +++++++--- src/sequifier/rl/grpo_collector.py | 46 ++-- src/sequifier/rl/grpo_update.py | 41 ++-- src/sequifier/rl/inference.py | 219 ++++++++++---------- src/sequifier/rl/learner.py | 100 +-------- src/sequifier/rl/offline_data.py | 6 +- src/sequifier/rl/ppo_collector.py | 102 ++++----- src/sequifier/rl/ppo_evaluation.py | 110 ++++------ src/sequifier/rl/preparation.py | 23 +- src/sequifier/rl/references.py | 110 ++++++++++ src/sequifier/rl/registry.py | 1 - src/sequifier/rl/run_checkpoint.py | 141 +++++++------ src/sequifier/rl/run_state.py | 19 +- src/sequifier/rl/runtime.py | 37 +--- src/sequifier/rl/training.py | 10 +- src/sequifier/trajectory/tables.py | 151 ++++++++------ 28 files changed, 957 insertions(+), 925 deletions(-) delete mode 100644 src/sequifier/config/training_plan.py create mode 100644 src/sequifier/io/episode_features.py create mode 100644 src/sequifier/rl/references.py diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 240622d1..3c8dae43 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -2,7 +2,7 @@ `sequifier train` and `sequifier.rl_api.run_policy_training` execute the same `PolicyRunConfig`. Its ordered `phases` may contain `policy_imitation`, `iql`, -`ppo`, `dpo`, and `grpo` in any compatible order. Each phase receives a deep copy +`ppo`, `dpo`, and `grpo` in any compatible order. Each phase takes ownership of its predecessor's selected actor. Optimizers, schedulers, critics, samplers, and environments start fresh at phase boundaries. @@ -163,6 +163,8 @@ and scale. Unknown return tails are excluded. The input return label at decision Episode/group/duplicate-content components remain wholly in training or validation. Explicit validation sources undergo the same overlap checks. Episodes load through bounded caches; windows are constructed on demand. +Episode fitting and decision indexing are shared independently of imitation +batches, IQL transitions, and preference pairs. ## Offline IQL @@ -240,7 +242,9 @@ memory without changing the summed sequence objective. DPO and GRPO each declare `reference: {source: phase_start}` (default) or `{source: artifact, path: models/reference/actor.pt}`. A phase-start reference is -a deep copy before any phase update. References stay in evaluation mode with +an independent weight snapshot before any phase update. Artifact references are +validated during preflight and reuse cached CPU weights; their files are checked +again before phase startup. References stay in evaluation mode with gradients disabled. Full representations and actual weight hashes must match. Embedded checkpoint reference weights survive removal of the original file; a still-present artifact with changed weights is rejected on resume. @@ -324,7 +328,9 @@ false requires divisibility. `max_buffer_bytes` defaults to 536870912. `evaluation_interval` and `checkpoint_interval` default to one complete collection/optimization iteration. Evaluation uses a separate adapter, -independent seeds and a whole-evaluation POSIX/main-thread deadline. It requires +independent seeds and a whole-evaluation POSIX/main-thread deadline. It evaluates +the live actor synchronously and restores every module's training mode and the +training RNG state afterward. It requires `environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; `selection: deterministic | stochastic` defaults to deterministic. Default `metric: mean_completed_return`, `direction: max` selects true completed returns. @@ -398,15 +404,17 @@ conflicts are checked separately before output mutation and at phase boundaries. Prepared data supplies its fitting view and frozen-representation validation; algorithm-specific transition checks remain with the IQL dataset. -The policy-training checkpoint is `sequifier_policy_run`, format version 5. +The policy-training checkpoint is `sequifier_policy_run`, format version 6. +Version-5 checkpoints are read and upgraded on the next save. Completed results determine the phase boundary; optional version-2 phase state records the latest committed transaction. It contains the resolved plan, frozen representation, source/partition and adapter identities, run RNG, completed -results and phase actor paths/checksums. Live phase state contains role-keyed -networks and optimizers/schedulers/step counts, sampler, selection, metrics and +phase records pairing each result with its actor checksum and result fingerprint. +Actor/report paths come from the result provenance. Live phase state contains +role-keyed networks and optimizers/schedulers/step counts, sampler, selection, metrics and algorithm-specific progress. DPO/GRPO references are embedded. Earlier selected -actors remain in their phase artifacts. Earlier unreleased checkpoints are not -supported. +actors remain in their phase artifacts. Checkpoints older than version 5 are +not supported. Writes use temporary files, flush them, and atomically replace their destinations. At a handoff, the selected `actor.pt` is published with its result metadata, then @@ -459,7 +467,8 @@ capabilities, `PolicyWindowView`, and model loading/execution helpers. Use `sequifier infer` with the final artifact for categorical sequence inference or rollout. `collect_policy_rollout` uses the same causal `PolicyHistory` and -`decide` path as online training and evaluation. Return-conditioned actors require +`PolicyCursor` decision/step path as online training and evaluation. +Return-conditioned actors require an explicit desired return. `PolicyRolloutConfig` declares `environment`, `trajectory_path`, `seed_index` and optional `max_episode_steps`; the environment must support seeded reset. Successors always come from the environment, so no diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index b07cde53..06cadb66 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -16,10 +16,10 @@ seeded reset preserves supplied initial observations during inference. Exact continuation requires validated snapshots or documented reproducible group starts; non-exact restarts report a discontinuity. Failed/partial groups are never reused. -- One atomic version-5 `sequifier_policy_run` checkpoint owns phase progress, +- One atomic version-6 (with version-5 resume support) `sequifier_policy_run` checkpoint owns phase progress, role-keyed optimizer/network state, references, source identities and RNG. Phase actors are durable exports referenced by checksum; reports are derived - and regenerated on resume. No phase manifests or old-checkpoint migration. + and regenerated on resume. No phase manifests are needed. - Shared phase sessions handle selection, checkpoints and optimization. Each phase configures optimizer roles under `optimization`; gradient validation and clipping precede all optimizer steps in a transaction. diff --git a/src/sequifier/config/policy_optimization.py b/src/sequifier/config/policy_optimization.py index 6fa6537f..184c6ff5 100644 --- a/src/sequifier/config/policy_optimization.py +++ b/src/sequifier/config/policy_optimization.py @@ -1,8 +1,6 @@ """Concrete optimizer ownership and scheduler budgets for policy phases.""" -from dataclasses import dataclass - -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from sequifier.config.components import ComponentSpec from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class @@ -25,25 +23,8 @@ class OptimizationConfig(BaseModel): learning_rate: float = Field(default=3e-4, gt=0) gradient_clip: float = Field(default=1.0, gt=0) - def resolve(self, step_budget): - return OptimizationPlan( - self.optimizer, - self.scheduler, - self.learning_rate, - self.gradient_clip, - step_budget, - ) - - -@dataclass(frozen=True) -class OptimizationPlan: - optimizer: ComponentSpec - scheduler: ComponentSpec - learning_rate: float - gradient_clip: float - step_budget: int | None - - def __post_init__(self): + @model_validator(mode="after") + def validate_components(self): if self.optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"} or { "lr", "params", @@ -59,13 +40,16 @@ def __post_init__(self): raise ValueError( "policy schedulers must advance independently on optimizer steps" ) + return self + + def validate_budget(self, step_budget): if ( - self.step_budget is not None + step_budget is not None and self.scheduler.name == "OneCycleLR" - and self.scheduler.arguments.get("total_steps") != self.step_budget + and self.scheduler.arguments.get("total_steps") != step_budget ): raise ValueError( - f"OneCycleLR.total_steps must equal the maximum authored optimizer-step budget ({self.step_budget})" + f"OneCycleLR.total_steps must equal the maximum authored optimizer-step budget ({step_budget})" ) def build(self, module): diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 7b6fb08a..24e099c7 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -23,19 +23,19 @@ class PhaseConfig(BaseModel): def optimizer_step_budget(self, training_size=None): raise NotImplementedError("built-in phase must declare its optimizer budget") - def optimization_plans(self, training_size=None): + def optimization_configs(self): unknown = self.optimization.keys() - set(self.optimization_roles) if unknown: raise ValueError(f"unsupported optimization roles: {sorted(unknown)}") - budget = self.optimizer_step_budget(training_size) return { - role: self.optimization.get(role, OptimizationConfig()).resolve(budget) + role: self.optimization.get(role, OptimizationConfig()) for role in self.optimization_roles } @model_validator(mode="after") def validate_optimization(self): - self.optimization_plans() + for config in self.optimization_configs().values(): + config.validate_budget(self.optimizer_step_budget()) return self @field_validator("name") diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index e73e591b..7b088e1e 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -75,10 +75,6 @@ SequenceSourceConfig, resolve_sequence_source, ) -from sequifier.config.training_plan import ( - TrainingPhaseSpecModel as TrainingPhaseSpecModel, -) -from sequifier.config.training_plan import TrainingPlanModel, TrainingSourceSpecModel from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -966,6 +962,87 @@ def validate_loss_weights(cls, value): return value +class TrainingSourceSpecModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + source: str + weight: Optional[float] = Field(default=None, gt=0) + batches_per_selection: Optional[int] = Field(default=None, gt=0) + + @field_validator("source") + @classmethod + @beartype + def validate_source(cls, value): + parts = value.split(".") + if len(parts) > 2: + raise ValueError("Source refs use only dataset or dataset.part") + for part in parts: + _identifier(part, "Source reference component") + return value + + +class TrainingPhaseSpecModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + algorithm: Literal["supervised"] = "supervised" + + name: str + epochs: int = Field(gt=0) + mode: Literal["sequential", "interleaved"] + selection: Optional[Literal["round_robin", "weighted_random"]] = None + sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) + + @field_validator("name") + @classmethod + @beartype + def validate_name(cls, value): + return _identifier(value, "Training phase name") + + @model_validator(mode="after") + @beartype + def validate_mode_fields(self): + if self.mode == "sequential": + if self.selection is not None: + raise ValueError("Sequential phases cannot configure selection") + if any(source.batches_per_selection is not None for source in self.sources): + raise ValueError( + "batches_per_selection is invalid in sequential phases" + ) + if any(source.weight is not None for source in self.sources): + raise ValueError("weight is invalid in sequential phases") + else: + self.selection = self.selection or "round_robin" + if self.selection == "round_robin" and any( + source.weight is not None for source in self.sources + ): + raise ValueError("weight is only valid for weighted_random selection") + return self + + +class TrainingPlanModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + curriculum_training: bool = False + curriculum_column: Optional[str] = Field(default=None, min_length=1) + phases: list[TrainingPhaseSpecModel] = Field(..., min_length=1) + + @model_validator(mode="after") + @beartype + def validate_unique_names(self): + if self.curriculum_column is not None and not self.curriculum_training: + raise ValueError( + "curriculum_column may only be set when curriculum_training is true" + ) + if self.curriculum_training and self.curriculum_column is None: + raise ValueError( + "curriculum_column must be set when curriculum_training is true" + ) + names = [phase.name for phase in self.phases] + if len(names) != len(set(names)): + raise ValueError("Training phase names must be unique") + return self + + class EvaluationMonitorSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/src/sequifier/config/training_plan.py b/src/sequifier/config/training_plan.py deleted file mode 100644 index 858a2f83..00000000 --- a/src/sequifier/config/training_plan.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Authored supervised training phases and curriculum schedules.""" - -from typing import Literal, Optional - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator - -from sequifier.config.identifiers import identifier as _identifier -from sequifier.typechecking import beartype - - -class TrainingSourceSpecModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - source: str - weight: Optional[float] = Field(default=None, gt=0) - batches_per_selection: Optional[int] = Field(default=None, gt=0) - - @field_validator("source") - @classmethod - @beartype - def validate_source(cls, value): - parts = value.split(".") - if len(parts) > 2: - raise ValueError("Source refs use only dataset or dataset.part") - for part in parts: - _identifier(part, "Source reference component") - return value - - -class TrainingPhaseSpecModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - algorithm: Literal["supervised"] = "supervised" - - name: str - epochs: int = Field(gt=0) - mode: Literal["sequential", "interleaved"] - selection: Optional[Literal["round_robin", "weighted_random"]] = None - sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) - - @field_validator("name") - @classmethod - @beartype - def validate_name(cls, value): - return _identifier(value, "Training phase name") - - @model_validator(mode="after") - @beartype - def validate_mode_fields(self): - if self.mode == "sequential": - if self.selection is not None: - raise ValueError("Sequential phases cannot configure selection") - if any(source.batches_per_selection is not None for source in self.sources): - raise ValueError( - "batches_per_selection is invalid in sequential phases" - ) - if any(source.weight is not None for source in self.sources): - raise ValueError("weight is invalid in sequential phases") - else: - self.selection = self.selection or "round_robin" - if self.selection == "round_robin" and any( - source.weight is not None for source in self.sources - ): - raise ValueError("weight is only valid for weighted_random selection") - return self - - -class TrainingPlanModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - curriculum_training: bool = False - curriculum_column: Optional[str] = Field(default=None, min_length=1) - phases: list[TrainingPhaseSpecModel] = Field(min_length=1) - - @model_validator(mode="after") - @beartype - def validate_unique_names(self): - if self.curriculum_column is not None and not self.curriculum_training: - raise ValueError( - "curriculum_column may only be set when curriculum_training is true" - ) - if self.curriculum_training and self.curriculum_column is None: - raise ValueError( - "curriculum_column must be set when curriculum_training is true" - ) - names = [phase.name for phase in self.phases] - if len(names) != len(set(names)): - raise ValueError("Training phase names must be unique") - return self diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 65994511..4e28b426 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -2352,7 +2352,7 @@ srun torchrun \ `sequifier train` and `sequifier.rl_api.run_policy_training` execute the same `PolicyRunConfig`. Its ordered `phases` may contain `policy_imitation`, `iql`, -`ppo`, `dpo`, and `grpo` in any compatible order. Each phase receives a deep copy +`ppo`, `dpo`, and `grpo` in any compatible order. Each phase takes ownership of its predecessor's selected actor. Optimizers, schedulers, critics, samplers, and environments start fresh at phase boundaries. @@ -2513,6 +2513,8 @@ and scale. Unknown return tails are excluded. The input return label at decision Episode/group/duplicate-content components remain wholly in training or validation. Explicit validation sources undergo the same overlap checks. Episodes load through bounded caches; windows are constructed on demand. +Episode fitting and decision indexing are shared independently of imitation +batches, IQL transitions, and preference pairs. ## Offline IQL @@ -2590,7 +2592,9 @@ memory without changing the summed sequence objective. DPO and GRPO each declare `reference: {source: phase_start}` (default) or `{source: artifact, path: models/reference/actor.pt}`. A phase-start reference is -a deep copy before any phase update. References stay in evaluation mode with +an independent weight snapshot before any phase update. Artifact references are +validated during preflight and reuse cached CPU weights; their files are checked +again before phase startup. References stay in evaluation mode with gradients disabled. Full representations and actual weight hashes must match. Embedded checkpoint reference weights survive removal of the original file; a still-present artifact with changed weights is rejected on resume. @@ -2674,7 +2678,9 @@ false requires divisibility. `max_buffer_bytes` defaults to 536870912. `evaluation_interval` and `checkpoint_interval` default to one complete collection/optimization iteration. Evaluation uses a separate adapter, -independent seeds and a whole-evaluation POSIX/main-thread deadline. It requires +independent seeds and a whole-evaluation POSIX/main-thread deadline. It evaluates +the live actor synchronously and restores every module's training mode and the +training RNG state afterward. It requires `environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; `selection: deterministic | stochastic` defaults to deterministic. Default `metric: mean_completed_return`, `direction: max` selects true completed returns. @@ -2748,15 +2754,17 @@ conflicts are checked separately before output mutation and at phase boundaries. Prepared data supplies its fitting view and frozen-representation validation; algorithm-specific transition checks remain with the IQL dataset. -The policy-training checkpoint is `sequifier_policy_run`, format version 5. +The policy-training checkpoint is `sequifier_policy_run`, format version 6. +Version-5 checkpoints are read and upgraded on the next save. Completed results determine the phase boundary; optional version-2 phase state records the latest committed transaction. It contains the resolved plan, frozen representation, source/partition and adapter identities, run RNG, completed -results and phase actor paths/checksums. Live phase state contains role-keyed -networks and optimizers/schedulers/step counts, sampler, selection, metrics and +phase records pairing each result with its actor checksum and result fingerprint. +Actor/report paths come from the result provenance. Live phase state contains +role-keyed networks and optimizers/schedulers/step counts, sampler, selection, metrics and algorithm-specific progress. DPO/GRPO references are embedded. Earlier selected -actors remain in their phase artifacts. Earlier unreleased checkpoints are not -supported. +actors remain in their phase artifacts. Checkpoints older than version 5 are +not supported. Writes use temporary files, flush them, and atomically replace their destinations. At a handoff, the selected `actor.pt` is published with its result metadata, then @@ -2809,7 +2817,8 @@ capabilities, `PolicyWindowView`, and model loading/execution helpers. Use `sequifier infer` with the final artifact for categorical sequence inference or rollout. `collect_policy_rollout` uses the same causal `PolicyHistory` and -`decide` path as online training and evaluation. Return-conditioned actors require +`PolicyCursor` decision/step path as online training and evaluation. +Return-conditioned actors require an explicit desired return. `PolicyRolloutConfig` declares `environment`, `trajectory_path`, `seed_index` and optional `max_episode_steps`; the environment must support seeded reset. Successors always come from the environment, so no diff --git a/src/sequifier/io/episode_features.py b/src/sequifier/io/episode_features.py new file mode 100644 index 00000000..f76e25a2 --- /dev/null +++ b/src/sequifier/io/episode_features.py @@ -0,0 +1,146 @@ +"""Fit and validate frozen policy features over an episode view.""" + +import polars as pl +import torch + +from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.io.policy_window import encode_feature +from sequifier.io.preprocessing import ( + _finite_mean_and_std, + create_id_map, + get_combined_statistics, + normalize_values, +) +from sequifier.trajectory.returns import return_to_go + + +def return_labels(episode, length, gamma): + if episode.transitions is None or episode.transitions.rewards is None: + raise ValueError("return-conditioned examples require recorded rewards") + return return_to_go(episode.transitions.rewards[:length], gamma) + + +class EpisodeFeatures: + """Selected episode lengths define fitting coverage independently of batching.""" + + def __init__(self, episodes, config, lengths=None): + self.episodes, self.config = episodes, config + self.lengths = ( + lengths + if lengths is not None + else [episodes.metadata(i).length for i in range(len(episodes))] + ) + self._validated_config = self._validated_statistics = None + ids = [episodes.metadata(i).episode_id for i in range(len(episodes))] + if len(ids) != len(set(ids)): + raise ValueError("duplicate episode IDs in fitting view") + + def values(self, episode, length, column): + return ( + episode.observations[column][:length] + if column in episode.observations + else [episode.static_inputs[column]] + ) + + def fit_categorical_config(self) -> CategoricalPolicyConfig: + """Fit only declared raw categorical features, using the training episodes.""" + values = self.config.model_dump() + for column in self.config.categorical_columns: + if ( + column in self.config.id_maps + or column in self.config.categorical_n_classes + ): + continue + vocabulary = set() + for episode, length in zip(self.episodes, self.lengths): + vocabulary.update(self.values(episode, length, column)) + if not vocabulary: + raise ValueError(f"cannot fit empty categorical column {column!r}") + if len({isinstance(value, str) for value in vocabulary}) > 1: + raise ValueError( + f"{column}: mixed string/numeric category labels are ambiguous" + ) + if any( + not isinstance(value, str) and value != int(value) + for value in vocabulary + ): + raise ValueError( + f"{column}: categorical numeric labels must be integers" + ) + mapping = create_id_map(pl.DataFrame({column: sorted(vocabulary)}), column) + values["id_maps"][column] = mapping + key = ( + "observation_n_classes" + if column in self.config.observation_columns + else "static_n_classes" + ) + values[key][column] = max(mapping.values()) + 1 + return CategoricalPolicyConfig.model_validate(values) + + def use_config(self, config: CategoricalPolicyConfig) -> None: + if self._validated_config == config: + return + self.config = config + self._validated_statistics = None + # Validate held-out values against the fixed training codec before updates. + for episode, length in zip(self.episodes, self.lengths): + for column, values in episode.observations.items(): + for value in values[:length]: + encode_feature(value, column, config) + for column, value in episode.static_inputs.items(): + encode_feature(value, column, config) + + self._validated_config = config.model_copy(deep=True) + + def validate_normalization(self, statistics): + """Check FP32 transforms episode-by-episode without constructing windows.""" + if self._validated_statistics == statistics: + return + for episode, length in zip(self.episodes, self.lengths): + for column in self.config.observation_columns + self.config.static_columns: + if column in self.config.categorical_n_classes: + continue + values = self.values(episode, length, column) + tensor = torch.tensor(values, dtype=torch.float32) + transformed = normalize_values(tensor, statistics[column]) + if ( + not torch.isfinite(tensor).all() + or not torch.isfinite(transformed).all() + ): + raise ValueError( + f"episode {episode.episode_id}, {column}: non-finite FP32 input/normalization" + ) + if self.config.return_conditioned: + returns = return_labels(episode, length, self.config.returns.gamma) + if not torch.isfinite( + torch.tensor(returns, dtype=torch.float32) + / self.config.returns.scale + ).all(): + raise ValueError( + f"episode {episode.episode_id}: non-finite conditioned return" + ) + + self._validated_statistics = {k: dict(v) for k, v in statistics.items()} + + def normalization_statistics(self): + """Use the same stable sample moments as supervised preprocessing.""" + moments = {} + columns = self.config.observation_columns + self.config.static_columns + for episode, length in zip(self.episodes, self.lengths): + for column in columns: + if column in self.config.categorical_n_classes: + continue + values = self.values(episode, length, column) + mean, std = _finite_mean_and_std(pl.Series(column, values), column) + size = len(values) + if column in moments: + count, old_mean, old_std = moments[column] + mean, std = get_combined_statistics( + size, mean, std, count, old_mean, old_std + ) + size += count + moments[column] = size, mean, std + return { + column: {"mean": mean, "std": std} + for column, (_, mean, std) in moments.items() + } diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py index 954f5bd4..662595dd 100644 --- a/src/sequifier/io/iql_dataset.py +++ b/src/sequifier/io/iql_dataset.py @@ -1,6 +1,5 @@ """Fixed offline transitions with causal current and successor contexts.""" -from bisect import bisect_right from dataclasses import dataclass import torch @@ -23,14 +22,14 @@ class IQLBatch(TensorRecord): class IQLTransitionDataset: """Index eligible decisions while retaining the source's bounded episode cache.""" - def __init__(self, policy_dataset): - if policy_dataset.config.return_conditioned: + def __init__(self, decisions): + if decisions.config.return_conditioned: raise ValueError("IQL requires an unconditioned actor") - self.policy_dataset = policy_dataset - self.view = PolicyWindowView(policy_dataset.config) - self.cumulative_lengths = policy_dataset.cumulative_lengths - self.episode_ids = policy_dataset.episode_ids - for episode_index, episode in enumerate(policy_dataset.episodes): + self.decisions = decisions + self.view = PolicyWindowView(decisions.config) + self.cumulative_lengths = decisions.cumulative_lengths + self.episode_ids = decisions.episode_ids + for episode_index, episode in enumerate(decisions.episodes): source = episode_index transitions = episode.transitions if transitions is None or any( @@ -65,7 +64,7 @@ def use_config(self, config): def validate_normalization(self, actor): # Interior successors are ordinary observations, already checked by the # policy dataset. Only the final successor can contain additional values. - for episode in self.policy_dataset.episodes: + for episode in self.decisions.episodes: step = len(episode.actions) - 1 successor = self._successor(episode, step) if successor is None: @@ -92,18 +91,11 @@ def _successor(episode, step): return None def __len__(self): - return len(self.policy_dataset) + return len(self.decisions) def __getitem__(self, index): - if index < 0: - index += len(self) - if not 0 <= index < len(self): - raise IndexError(index) - episode_index = bisect_right(self.cumulative_lengths, index) - step = index - ( - self.cumulative_lengths[episode_index - 1] if episode_index else 0 - ) - episode = self.policy_dataset.episodes[episode_index] + episode_index, step = self.decisions.locate(index) + episode = self.decisions.episodes[episode_index] transitions = episode.transitions successor = self._successor(episode, step) current = self.view.episode_inputs(episode, step, None) @@ -116,7 +108,7 @@ def __getitem__(self, index): current=current, successor=next_inputs, action=torch.tensor( - [action_index(episode.actions[step], self.policy_dataset.config)], + [action_index(episode.actions[step], self.decisions.config)], dtype=torch.long, ), reward=torch.tensor([transitions.rewards[step]], dtype=torch.float32), diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index 829a202e..68ddcb81 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -4,39 +4,24 @@ from collections import OrderedDict from collections.abc import Sequence -import polars as pl import torch from torch.utils.data import Dataset, Sampler from sequifier.config.policy_config import CategoricalPolicyConfig from sequifier.io.batch import SequifierBatch from sequifier.io.episode_corpus import EpisodeCorpus, EpisodeView, episode_content_key +from sequifier.io.episode_features import EpisodeFeatures, return_labels from sequifier.io.partitioning import split_components from sequifier.io.policy_window import ( PolicyWindowView, action_global_ids, action_index, action_target, - encode_feature, ) -from sequifier.io.preprocessing import ( - _finite_mean_and_std, - create_id_map, - get_combined_statistics, - normalize_values, -) -from sequifier.io.sequence import SequenceEpisode -from sequifier.trajectory.returns import return_to_go - -def _return_labels(episode: SequenceEpisode, length: int, gamma: float): - if episode.transitions is None or episode.transitions.rewards is None: - raise ValueError("return-conditioned examples require recorded rewards") - return return_to_go(episode.transitions.rewards[:length], gamma) - -class PolicySequenceDataset(Dataset): - """One decision per standard SequifierBatch; the storage format is incidental.""" +class EpisodeDecisions(EpisodeFeatures): + """Episode-safe decision indexing shared by imitation and transition views.""" def __init__( self, @@ -97,8 +82,7 @@ def __init__( self.episode_fingerprints = [ self.episodes.metadata(i).fingerprint for i in range(len(self.episodes)) ] - self._validated_config = None - self._validated_statistics = None + super().__init__(self.episodes, config, self.lengths) def __len__(self): return self.cumulative_lengths[-1] if self.cumulative_lengths else 0 @@ -106,94 +90,6 @@ def __len__(self): def sampler(self, generator): return EpisodeWindowSampler(self, generator) - def fit_categorical_config(self) -> CategoricalPolicyConfig: - """Fit only declared raw categorical features, using the training episodes.""" - values = self.config.model_dump() - for column in self.config.categorical_columns: - if ( - column in self.config.id_maps - or column in self.config.categorical_n_classes - ): - continue - vocabulary = set() - for episode, length in zip(self.episodes, self.lengths): - vocabulary.update( - episode.observations[column][:length] - if column in episode.observations - else [episode.static_inputs[column]] - ) - if not vocabulary: - raise ValueError(f"cannot fit empty categorical column {column!r}") - if len({isinstance(value, str) for value in vocabulary}) > 1: - raise ValueError( - f"{column}: mixed string/numeric category labels are ambiguous" - ) - if any( - not isinstance(value, str) and value != int(value) - for value in vocabulary - ): - raise ValueError( - f"{column}: categorical numeric labels must be integers" - ) - mapping = create_id_map(pl.DataFrame({column: sorted(vocabulary)}), column) - values["id_maps"][column] = mapping - key = ( - "observation_n_classes" - if column in self.config.observation_columns - else "static_n_classes" - ) - values[key][column] = max(mapping.values()) + 1 - return CategoricalPolicyConfig.model_validate(values) - - def use_config(self, config: CategoricalPolicyConfig) -> None: - if self._validated_config == config: - return - self.config = config - self._validated_statistics = None - # Validate held-out values against the fixed training codec before updates. - for episode, length in zip(self.episodes, self.lengths): - for column, values in episode.observations.items(): - for value in values[:length]: - encode_feature(value, column, config) - for column, value in episode.static_inputs.items(): - encode_feature(value, column, config) - - self._validated_config = config.model_copy(deep=True) - - def validate_normalization(self, statistics): - """Check FP32 transforms episode-by-episode without constructing windows.""" - if self._validated_statistics == statistics: - return - for index, (episode, length) in enumerate(zip(self.episodes, self.lengths)): - for column in self.config.observation_columns + self.config.static_columns: - if column in self.config.categorical_n_classes: - continue - values = ( - episode.observations[column][:length] - if column in episode.observations - else [episode.static_inputs[column]] - ) - tensor = torch.tensor(values, dtype=torch.float32) - transformed = normalize_values(tensor, statistics[column]) - if ( - not torch.isfinite(tensor).all() - or not torch.isfinite(transformed).all() - ): - raise ValueError( - f"episode {episode.episode_id}, {column}: non-finite FP32 input/normalization" - ) - if self.config.return_conditioned: - _, returns = self._episode_labels(index) - if not torch.isfinite( - torch.tensor(returns, dtype=torch.float32) - / self.config.returns.scale - ).all(): - raise ValueError( - f"episode {episode.episode_id}: non-finite conditioned return" - ) - - self._validated_statistics = {k: dict(v) for k, v in statistics.items()} - def _episode_labels(self, index: int): if index not in self._labels: episode = self.episodes[index] @@ -202,7 +98,7 @@ def _episode_labels(self, index: int): action_index(value, self.config) for value in episode.actions[:length] ] returns = ( - _return_labels(episode, length, self.config.returns.gamma) + return_labels(episode, length, self.config.returns.gamma) if self.config.return_conditioned else None ) @@ -212,7 +108,7 @@ def _episode_labels(self, index: int): self._labels.move_to_end(index) return self._labels[index] - def __getitem__(self, index: int) -> SequifierBatch: + def locate(self, index): if index < 0: index += len(self) if not 0 <= index < len(self): @@ -221,50 +117,7 @@ def __getitem__(self, index: int) -> SequifierBatch: step = index - ( self.cumulative_lengths[episode_index - 1] if episode_index else 0 ) - episode = self.episodes[episode_index] - action_ids, returns = self._episode_labels(episode_index) - inputs = PolicyWindowView(self.config).episode_inputs( - episode, step, returns, action_ids - ) - targets = torch.zeros((1, self.config.context_length), dtype=torch.int64) - targets[0, -1] = action_global_ids(self.config)[action_ids[step]] - metadata = dict(inputs.metadata) - metadata["target_valid_mask"] = torch.zeros_like(inputs.attention_valid_mask) - metadata["target_valid_mask"][0, -1] = True - return SequifierBatch( - inputs.inputs, - {action_target(self.config): targets}, - metadata, - torch.tensor([episode_index], dtype=torch.int64), - torch.tensor([episode.item_positions[step]], dtype=torch.int64), - ) - - def normalization_statistics(self): - """Use the same stable sample moments as supervised preprocessing.""" - moments = {} - columns = self.config.observation_columns + self.config.static_columns - for episode, length in zip(self.episodes, self.lengths): - for column in columns: - if column in self.config.categorical_n_classes: - continue - values = ( - episode.observations[column][:length] - if column in episode.observations - else [episode.static_inputs[column]] - ) - mean, std = _finite_mean_and_std(pl.Series(column, values), column) - size = len(values) - if column in moments: - count, old_mean, old_std = moments[column] - mean, std = get_combined_statistics( - size, mean, std, count, old_mean, old_std - ) - size += count - moments[column] = size, mean, std - return { - column: {"mean": mean, "std": std} - for column, (_, mean, std) in moments.items() - } + return episode_index, step class EpisodeWindowSampler(Sampler): @@ -304,7 +157,7 @@ def keys(index): range(len(dataset.episodes)), keys, validation_fraction, seed ) return tuple( - PolicySequenceDataset( + type(dataset)( dataset.episodes.subset(indices), dataset.config, episode_cache_size=dataset.episode_cache_size, @@ -329,9 +182,37 @@ def dataset(self, names): self.corpora[name] = EpisodeCorpus( [self.sources[name]], self.config, self.cache_size ) - self.datasets[key] = PolicySequenceDataset( + self.datasets[key] = EpisodeDecisions( EpisodeView.combine([self.corpora[name].view() for name in key]), self.config, episode_cache_size=self.cache_size, ) return self.datasets[key] + + +class PolicySequenceDataset(EpisodeDecisions, Dataset): + """A supervised batch view over the common episode decision index.""" + + def __getitem__(self, index): + return imitation_example(self, index) + + +def imitation_example(dataset, index) -> SequifierBatch: + episode_index, step = dataset.locate(index) + episode = dataset.episodes[episode_index] + action_ids, returns = dataset._episode_labels(episode_index) + inputs = PolicyWindowView(dataset.config).episode_inputs( + episode, step, returns, action_ids + ) + targets = torch.zeros((1, dataset.config.context_length), dtype=torch.int64) + targets[0, -1] = action_global_ids(dataset.config)[action_ids[step]] + metadata = dict(inputs.metadata) + metadata["target_valid_mask"] = torch.zeros_like(inputs.attention_valid_mask) + metadata["target_valid_mask"][0, -1] = True + return SequifierBatch( + inputs.inputs, + {action_target(dataset.config): targets}, + metadata, + torch.tensor([episode_index], dtype=torch.int64), + torch.tensor([episode.item_positions[step]], dtype=torch.int64), + ) diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index a338f2dd..cf29aff4 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -7,8 +7,8 @@ from sequifier.fingerprints import fingerprint from sequifier.io.episode_corpus import EpisodeView, episode_content_key +from sequifier.io.episode_features import EpisodeFeatures from sequifier.io.partitioning import split_components -from sequifier.io.policy_dataset import PolicySequenceDataset from sequifier.io.policy_window import PolicyWindowView, action_index from sequifier.trajectory.preferences import load_preferences @@ -119,11 +119,9 @@ def fitting_dataset(self): entries[(span.source, span.episode_id)] = self.corpora[ span.source ].episodes.entries[index] - cache_size = next(iter(self.corpora.values())).episode_cache_size - self._fitting = PolicySequenceDataset( + self._fitting = EpisodeFeatures( EpisodeView(entries.values()), self.config, - episode_cache_size=cache_size, ) return self._fitting diff --git a/src/sequifier/io/prepared_policy_data.py b/src/sequifier/io/prepared_policy_data.py index 4f0ac9e0..2e58abb1 100644 --- a/src/sequifier/io/prepared_policy_data.py +++ b/src/sequifier/io/prepared_policy_data.py @@ -3,7 +3,7 @@ from collections.abc import Sized from typing import TYPE_CHECKING, Protocol -from sequifier.io.policy_dataset import PolicySequenceDataset +from sequifier.io.episode_features import EpisodeFeatures if TYPE_CHECKING: from sequifier.rl.policy import CategoricalPolicy @@ -16,6 +16,6 @@ def training(self) -> Sized: ... @property def contract(self) -> list: ... - def fitting_dataset(self) -> PolicySequenceDataset: ... + def fitting_dataset(self) -> EpisodeFeatures: ... def validate(self, actor: "CategoricalPolicy") -> None: ... diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index 19c64451..f7a98c28 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -1,11 +1,15 @@ """Independent seed streams and categorical online decision scoring.""" import hashlib +import math from dataclasses import dataclass import torch -from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.rl.distributions import CategoricalActionDistribution, action_generator +from sequifier.rl.environment import validate_reset, validate_step +from sequifier.rl.history import PolicyHistory +from sequifier.trajectory.returns import remaining_return def stream_seed(seed, stream, reset_count): @@ -24,36 +28,79 @@ class PolicyDecision: value: float -def validate_online_inputs(actor, inputs): - inputs = inputs.to(actor.device) - actor.model_inputs(inputs) - return inputs +class PolicyCursor: + """Own causal history and action RNG for one live environment. + Decision and execution are separate so collectors can score a reference and + account for issued actions before handing control to the environment. + """ -@torch.no_grad() -def decide(actor, value, inputs, generator, *, deterministic=False): - """Score action and history-only V from exactly the same causal input.""" - actor.eval() - if value is not None: - value.eval() - inputs = inputs.to(actor.device) - features, metadata = actor.model_inputs(inputs) - distribution = CategoricalActionDistribution( - actor.prepared_logits(features, metadata)[:, -1] - ) - action = distribution.select(deterministic, generator) - estimate = ( - value.prepared_value(features, metadata) - if value is not None - else torch.zeros(1, device=actor.device) - ) - if not torch.isfinite(estimate).all(): - raise ValueError("non-finite collection value") - index = int(action[0]) - return PolicyDecision( - {actor.config.action_column: actor.config.action_values[index]}, - index, - float(distribution.log_prob(action)[0]), - float(distribution.entropy()[0]), - float(estimate[0]), - ) + def __init__(self, actor, adapter, units, seed, *, strict_vocabulary=True): + self.actor, self.adapter, self.units = actor, adapter, units + self.history = PolicyHistory(actor.config, strict_vocabulary=strict_vocabulary) + self.generator = action_generator(actor.device, seed) + self.desired_return = None + self.episode_return, self.episode_length, self.ended = 0.0, 0, True + self.last_result = None + + def refresh(self): + self.inputs = self.history.inputs().to(self.actor.device) + self.actor.model_inputs(self.inputs) + + def reset(self, result, episode_id, desired_return=None): + result = validate_reset(result, self.units) + self.history.reset( + result.observation, result.static_inputs, episode_id, desired_return + ) + self.desired_return = desired_return + self.episode_return, self.episode_length, self.ended = 0.0, 0, False + self.last_result = None + self.refresh() + + @torch.no_grad() + def decide(self, value=None, *, deterministic=False): + """Score action and history-only V from exactly the same causal input.""" + actor = self.actor + actor.eval() + if value is not None: + value.eval() + features, metadata = actor.model_inputs(self.inputs) + distribution = CategoricalActionDistribution( + actor.prepared_logits(features, metadata)[:, -1] + ) + action = distribution.select(deterministic, self.generator) + estimate = ( + value.prepared_value(features, metadata) + if value is not None + else torch.zeros(1, device=actor.device) + ) + if not torch.isfinite(estimate).all(): + raise ValueError("non-finite collection value") + index = int(action[0]) + return PolicyDecision( + {actor.config.action_column: actor.config.action_values[index]}, + index, + float(distribution.log_prob(action)[0]), + float(distribution.entropy()[0]), + float(estimate[0]), + ) + + def step(self, decision): + result = validate_step(self.adapter.step(decision.domain_action), self.units) + self.last_result = result + self.episode_return += float(result.reward) + self.episode_length += 1 + self.ended = result.terminated or result.truncated + if not math.isfinite(self.episode_return): + raise ValueError("non-finite collected episode return") + if self.actor.config.return_conditioned: + assert self.desired_return is not None + self.desired_return = remaining_return( + self.desired_return, result.reward, self.actor.config.returns.gamma + ) + if result.observation is not None: + self.history.advance( + decision.action, result.observation, self.desired_return + ) + self.refresh() + return result diff --git a/src/sequifier/rl/grpo_collector.py b/src/sequifier/rl/grpo_collector.py index d67e2086..860fd1d9 100644 --- a/src/sequifier/rl/grpo_collector.py +++ b/src/sequifier/rl/grpo_collector.py @@ -6,16 +6,10 @@ import torch from sequifier.fingerprints import fingerprint -from sequifier.rl.collection import decide, stream_seed, validate_online_inputs -from sequifier.rl.distributions import action_generator -from sequifier.rl.environment import ( - require_group_start, - validate_reset, - validate_start_token, - validate_step, -) +from sequifier.rl.collection import PolicyCursor, stream_seed +from sequifier.rl.environment import require_group_start, validate_start_token from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer -from sequifier.rl.history import PolicyHistory, frame_storage_bytes +from sequifier.rl.history import frame_storage_bytes class GroupCollectionError(RuntimeError): @@ -57,22 +51,26 @@ def collect_groups( trajectories, initial_key = [], None for sample in range(phase.group_size): # All samples restore the same opaque condition, then independently seed future randomness. - result = validate_reset( + cursor = PolicyCursor( + actor, + adapter, + context.run.units, + stream_seed(seed, f"group-{group_index}-actions", sample), + ) + episode_id = f"group-{group_index}-sample-{sample}" + cursor.reset( adapter.reset_group( validate_start_token(token), stream_seed(seed, f"group-{group_index}-environment", sample), ), - context.run.units, + episode_id, ) - history = PolicyHistory(actor.config, strict_vocabulary=True) - episode_id = f"group-{group_index}-sample-{sample}" - history.reset(result.observation, result.static_inputs, episode_id) - inputs = validate_online_inputs(actor, history.inputs()) + inputs = cursor.inputs key = fingerprint( { "features": {k: v.tolist() for k, v in inputs.inputs.items()}, "metadata": {k: v.tolist() for k, v in inputs.metadata.items()}, - "units": dict(result.units), + "units": dict(cursor.units), } ) if initial_key is not None and key != initial_key: @@ -80,19 +78,15 @@ def collect_groups( "GRPO samples have different canonical initial histories or units" ) initial_key = key - generator = action_generator( - actor.device, - stream_seed(seed, f"group-{group_index}-actions", sample), - ) frames, actions, rewards, old_scores, ref_scores = [], [], [], [], [] reason = "max_steps" for step in range(phase.max_steps): - frame = history.current + frame = cursor.history.current used += frame_storage_bytes(frame) + 1024 if used > phase.max_buffer_bytes: raise ValueError("GRPO group exceeded max_buffer_bytes") - inputs = history.inputs().to(actor.device) - decision = decide(actor, None, inputs, generator) + inputs = cursor.inputs + decision = cursor.decide() reference_score = float( reference.evaluate_actions( inputs, @@ -102,8 +96,7 @@ def collect_groups( )[0] ) executed += 1 - raw = adapter.step(decision.domain_action) - step_result = validate_step(raw, context.run.units) + step_result = cursor.step(decision) reward = float(step_result.reward) * phase.reward_scale if not torch.isfinite( torch.tensor( @@ -117,9 +110,6 @@ def collect_groups( rewards.append(reward) old_scores.append(decision.log_probability) ref_scores.append(reference_score) - if step_result.observation is not None: - history.advance(decision.action, step_result.observation) - validate_online_inputs(actor, history.inputs()) if step_result.terminated or step_result.truncated: reason = "terminated" if step_result.terminated else "truncated" break diff --git a/src/sequifier/rl/grpo_update.py b/src/sequifier/rl/grpo_update.py index da84a30a..90bbdedd 100644 --- a/src/sequifier/rl/grpo_update.py +++ b/src/sequifier/rl/grpo_update.py @@ -85,29 +85,26 @@ def update_grpo(session, buffer): stopped = True break session.zero_grad() - for offset in range(0, len(indices), phase.batch_size): - # Whole trajectories belong to a minibatch; causal windows are - # chunked within each trajectory without changing its weight. - for i in indices[offset : offset + phase.batch_size]: - trajectory = trajectories[i] - for inputs, actions, old, ref in trajectory.batches( - actor, phase.scoring_chunk_size - ): - values = grpo_objective( - actor.evaluate_actions(inputs, actions), - old, - ref, - trajectory.advantage, - phase, + for i in indices: + trajectory = trajectories[i] + for inputs, actions, old, ref in trajectory.batches( + actor, phase.scoring_chunk_size + ): + values = grpo_objective( + actor.evaluate_actions(inputs, actions), + old, + ref, + trajectory.advantage, + phase, + ) + ( + values[0].sum() / len(trajectory.actions) / len(indices) + ).backward() + for key, value in zip(totals, values): + totals[key] += float(value.detach().sum()) / len( + trajectory.actions ) - ( - values[0].sum() / len(trajectory.actions) / len(indices) - ).backward() - for key, value in zip(totals, values): - totals[key] += float(value.detach().sum()) / len( - trajectory.actions - ) - scored += 1 + scored += 1 session.step() if stopped: break diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 151aa50d..5c546a99 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -1,6 +1,8 @@ """CLI policy prediction and environment execution using the shared model lifecycle.""" +from itertools import islice from pathlib import Path +from typing import cast import polars as pl import torch @@ -9,7 +11,7 @@ from sequifier.config.policy_infer_config import PolicyInferenceConfig from sequifier.config.policy_train_config import PolicyTableColumns from sequifier.helpers import configure_determinism, normalize_path, write_data -from sequifier.io.batch import collate_model_inputs +from sequifier.io.batch import SequifierInputs, collate_model_inputs from sequifier.io.policy_window import PolicyWindowView from sequifier.io.sequence import ordered_sequence_rows from sequifier.io.values import domain_scalar @@ -48,61 +50,84 @@ def infer_policy(config: PolicyInferenceConfig) -> None: f"policy inference is missing columns: {sorted(required - set(data.columns))}" ) - def budget(row): - return ( - numeric_scalar( - config.desired_return - if config.desired_return is not None - else row[config.desired_return_column], - "desired_return", - ) - if spec.return_conditioned - else None - ) - if config.rollout is not None: - assert config.generation_steps is not None - rollout = config.rollout - if rollout.seed_index >= len(data): - raise ValueError("policy rollout seed_index is outside the input rows") - row = data.row(rollout.seed_index, named=True) - seed = SeedHistory( - observations={ - column: [0.0] * (spec.context_length - 1) - + [domain_scalar(row[column], column)] - for column in spec.observation_columns - }, - attention_valid_mask=[False] * (spec.context_length - 1) + [True], - static_inputs={ - column: domain_scalar(row[column], column) - for column in spec.static_columns - }, - ) - desired_return = budget(row) - trajectory = collect_policy_rollout( - policy, - load_online_adapter(rollout.environment), - seed, - rollout, - config.generation_steps, - desired_return=desired_return, - deterministic=config.deterministic, - random_seed=config.seed, - ) - trajectory.metadata.update( - { - "model_path": config.model_path, - "model_interface": config.model_interface, - "seed_index": rollout.seed_index, - } + infer_rollout(policy, data, config) + return + predictions = predict_policy(policy, data, config) + if config.output_path is None: + from sequifier.infer import inference_output_path + + destination = inference_output_path( + config.project_root, + config.write_format, + "predictions", + Path(config.model_path).stem, + 0, ) - destination = normalize_path(rollout.trajectory_path, config.project_root) - save_trajectory(trajectory, destination) - logger.info( - f"Wrote {len(trajectory.transitions)} policy transitions to '{destination}'" + else: + destination = normalize_path(config.output_path, config.project_root) + Path(destination).parent.mkdir(parents=True, exist_ok=True) + write_data(predictions, destination, config.write_format) + logger.info(f"Wrote {len(predictions)} policy predictions to '{destination}'") + + +def desired_return(config, spec, row): + return ( + numeric_scalar( + config.desired_return + if config.desired_return is not None + else row[config.desired_return_column], + "desired_return", ) - return + if spec.return_conditioned + else None + ) + + +def infer_rollout(policy, data, config): + spec = policy.config + assert config.generation_steps is not None + rollout = config.rollout + if rollout.seed_index >= len(data): + raise ValueError("policy rollout seed_index is outside the input rows") + row = data.row(rollout.seed_index, named=True) + seed = SeedHistory( + observations={ + column: [0.0] * (spec.context_length - 1) + + [domain_scalar(row[column], column)] + for column in spec.observation_columns + }, + attention_valid_mask=[False] * (spec.context_length - 1) + [True], + static_inputs={ + column: domain_scalar(row[column], column) for column in spec.static_columns + }, + ) + budget = desired_return(config, spec, row) + trajectory = collect_policy_rollout( + policy, + load_online_adapter(rollout.environment), + seed, + rollout, + config.generation_steps, + desired_return=budget, + deterministic=config.deterministic, + random_seed=config.seed, + ) + trajectory.metadata.update( + { + "model_path": config.model_path, + "model_interface": config.model_interface, + "seed_index": rollout.seed_index, + } + ) + destination = normalize_path(rollout.trajectory_path, config.project_root) + save_trajectory(trajectory, destination) + logger.info( + f"Wrote {len(trajectory.transitions)} policy transitions to '{destination}'" + ) + +def policy_windows(data, spec, config): # Rows without a sequence ID are independent single-observation decisions. # Within sequences, recorded actions determine the causal previous-action input. indexed = data.with_row_index("__policy_row") @@ -111,39 +136,6 @@ def budget(row): if config.sequence_id_column in data.columns else [indexed.slice(i, 1) for i in range(len(indexed))] ) - outputs = [] - generator = action_generator(policy.device, config.seed) - pending_inputs = [] - pending_rows = [] - - def flush(): - if not pending_inputs: - return - with torch.no_grad(): - inputs = collate_model_inputs(pending_inputs).to(policy.device) - decision = CategoricalActionDistribution(policy(inputs)[:, -1, :]) - actions = decision.select(config.deterministic, generator).cpu().tolist() - probabilities = ( - decision.distribution.probs.cpu().tolist() - if config.output_probabilities - else None - ) - for index, (row_index, action) in enumerate(zip(pending_rows, actions)): - output = { - "__policy_row": row_index, - spec.action_column: spec.action_values[action], - } - if probabilities is not None: - output.update( - { - f"{spec.action_column}__probability_{action_id}": value - for action_id, value in enumerate(probabilities[index]) - } - ) - outputs.append(output) - pending_inputs.clear() - pending_rows.clear() - view = PolicyWindowView(spec) for group in groups: if config.item_position_column not in group.columns: @@ -190,16 +182,45 @@ def flush(): sequence_id=0, ) returns = ( - [budget(row) for row in group.iter_rows(named=True)] + [desired_return(config, spec, row) for row in group.iter_rows(named=True)] if spec.return_conditioned else None ) for step, row_index in enumerate(group["__policy_row"]): - pending_inputs.append(view.episode_inputs(episode, step, returns)) - pending_rows.append(row_index) - if len(pending_inputs) >= config.inference_batch_size: - flush() - flush() + yield row_index, view.episode_inputs(episode, step, returns) + + +@torch.no_grad() +def predict_policy(policy, data, config): + spec = policy.config + windows = policy_windows(data, spec, config) + generator = action_generator(policy.device, config.seed) + outputs = [] + while batch := list(islice(windows, config.inference_batch_size)): + rows, histories = zip(*batch) + inputs = cast(SequifierInputs, collate_model_inputs(histories)).to( + policy.device + ) + distribution = CategoricalActionDistribution(policy(inputs)[:, -1, :]) + actions = distribution.select(config.deterministic, generator).cpu().tolist() + probabilities = ( + distribution.distribution.probs.cpu().tolist() + if config.output_probabilities + else None + ) + for index, (row_index, action) in enumerate(zip(rows, actions)): + output = { + "__policy_row": row_index, + spec.action_column: spec.action_values[action], + } + if probabilities is not None: + output.update( + { + f"{spec.action_column}__probability_{action_id}": value + for action_id, value in enumerate(probabilities[index]) + } + ) + outputs.append(output) predictions = pl.DataFrame(outputs).sort("__policy_row").drop("__policy_row") identity = [ column @@ -211,18 +232,4 @@ def flush(): if identity else predictions ) - if config.output_path is None: - from sequifier.infer import inference_output_path - - destination = inference_output_path( - config.project_root, - config.write_format, - "predictions", - Path(config.model_path).stem, - 0, - ) - else: - destination = normalize_path(config.output_path, config.project_root) - Path(destination).parent.mkdir(parents=True, exist_ok=True) - write_data(predictions, destination, config.write_format) - logger.info(f"Wrote {len(predictions)} policy predictions to '{destination}'") + return predictions diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index ef2d4694..39a41347 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -1,6 +1,5 @@ """Sequential policy coordination over prepared data and validated recovery state.""" -from copy import deepcopy from dataclasses import replace from pathlib import Path @@ -20,6 +19,7 @@ validate_declared_policy, validate_output_paths, ) +from sequifier.rl.references import PolicyReferences from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_checkpoint import RunCheckpoint from sequifier.rl.run_state import ( @@ -73,89 +73,6 @@ def _initialize_actor(config, data): return actor -def _artifact_identity(config, path, representation, field): - actor = load_policy( - path, device=config.device, interface_name=config.policy_interface - ) - representation.validate_actor(actor, config.units, field) - return weight_fingerprint(actor.network.state_dict()) - - -def _preflight_references(config, recovery, resuming): - if resuming and config.initialize_from and Path(config.initialize_from).is_file(): - require_equal( - recovery.initialization["weight_hash"], - _artifact_identity( - config, - config.initialize_from, - recovery.representation, - "resume.initialize_from", - ), - "resume.initialize_from.weights", - ) - for index, phase in enumerate(config.phases): - reference = getattr(phase, "reference", None) - if reference is None or reference.source != "artifact": - continue - path = Path(reference.path) - if path.is_file(): - identity = _artifact_identity( - config, path, recovery.representation, f"phases.{phase.name}.reference" - ) - if str(index) in recovery.reference_contract: - require_equal( - recovery.reference_contract[str(index)], - identity, - f"phases.{phase.name}.reference.weights", - ) - recovery.reference_contract[str(index)] = identity - elif index > recovery.phase_index or ( - index == recovery.phase_index and recovery.phase_state is None - ): - raise FileNotFoundError( - f"reference artifact required before phase start: {path}" - ) - - -def _phase_reference(config, recovery, actor): - phase = config.phases[recovery.phase_index] - reference, reference_identity = None, None - declaration = getattr(phase, "reference", None) - if declaration is not None: - if recovery.phase_state is not None: - reference = deepcopy(actor) - reference.network.load_state_dict( - recovery.phase_state["reference"], strict=True - ) - elif declaration.source == "artifact": - reference = load_policy( - declaration.path, - device=config.device, - interface_name=config.policy_interface, - ) - else: - reference = deepcopy(actor) - reference.eval().requires_grad_(False) - recovery.representation.validate_actor( - reference, config.units, f"phases.{phase.name}.reference" - ) - reference_identity = weight_fingerprint(reference.network.state_dict()) - if recovery.phase_state is not None: - require_equal( - reference_identity, - recovery.phase_state["reference_identity"], - "resume.reference.weights", - ) - if declaration.source == "artifact": - require_equal( - recovery.reference_contract[str(recovery.phase_index)], - reference_identity, - "reference.artifact.weights", - ) - - return reference, reference_identity - - def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: """Validate the whole run, execute phase-local lifecycles, and export its final actor.""" if not isinstance(config, PolicyRunConfig): @@ -222,9 +139,9 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: clone_weights(actor.network), capture_rng_state(actor.device), ) - _preflight_references(config, recovery, resuming) + references = PolicyReferences(config, recovery, resuming=resuming) saved_random = capture_rng_state(actor.device) - optimization = preflight_optimization(config, actor, data) + step_budgets = preflight_optimization(config, actor, data) restore_rng_state(actor.device, recovery.random if resuming else saved_random) if not resuming: @@ -241,12 +158,12 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: ) validate_output_paths(config, inputs) incoming_weight_hash = weight_fingerprint(actor.network.state_dict()) - phase_actor = deepcopy(actor).eval() - reference, reference_identity = _phase_reference(config, recovery, phase_actor) + actor.eval() + reference, reference_identity = references.resolve(actor) def commit(state): representation.validate_actor( - phase_actor, config.units, f"phases.{phase.name}.actor" + actor, config.units, f"phases.{phase.name}.actor" ) recovery.save(checkpoint, actor.device, state) @@ -254,14 +171,13 @@ def commit(state): config, phase_index, phase, - phase_actor, - representation, + actor, data[phase_index], reference, reference_identity, current_state, commit, - optimization[phase_index], + step_budgets[phase_index], ) actor, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) representation.validate_actor( diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index 81f433a9..7a6e5c64 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -3,13 +3,13 @@ from dataclasses import dataclass, field from sequifier.io.iql_dataset import IQLTransitionDataset -from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode +from sequifier.io.policy_dataset import EpisodeDecisions, split_by_episode @dataclass class OfflineData: - training: PolicySequenceDataset - validation: PolicySequenceDataset + training: EpisodeDecisions + validation: EpisodeDecisions transitions: tuple[IQLTransitionDataset, IQLTransitionDataset] | None = field( default=None, init=False ) diff --git a/src/sequifier/rl/ppo_collector.py b/src/sequifier/rl/ppo_collector.py index 42b29f17..520d4958 100644 --- a/src/sequifier/rl/ppo_collector.py +++ b/src/sequifier/rl/ppo_collector.py @@ -1,13 +1,8 @@ """One persistent environment slot; horizons cut history without resetting it.""" -import math - import torch -from sequifier.rl.collection import decide, stream_seed, validate_online_inputs -from sequifier.rl.distributions import action_generator -from sequifier.rl.environment import validate_reset, validate_step -from sequifier.rl.history import PolicyHistory +from sequifier.rl.collection import PolicyCursor, stream_seed from sequifier.rl.ppo_buffer import PPOBufferBuilder, PPODecision @@ -18,32 +13,24 @@ def __init__(self, actor, value, adapter, phase, seed): self.adapter = adapter self.phase = phase self.seed = seed - self.history = PolicyHistory(actor.config, strict_vocabulary=True) - self.generator = action_generator( - actor.device, stream_seed(seed, "training-actions", 0) + self.cursor = PolicyCursor( + actor, + adapter, + phase.environment.units, + stream_seed(seed, "training-actions", 0), ) - self.ended = True - self.episode_return = 0.0 - self.episode_length = 0 @property def reset_count(self): - return self.history.reset_count + return self.cursor.history.reset_count def reset(self): - result = validate_reset( + self.cursor.reset( self.adapter.reset( stream_seed(self.seed, "training-reset", self.reset_count) ), - self.phase.environment.units, - ) - self.history.reset( - result.observation, result.static_inputs, f"train-{self.reset_count}" + f"train-{self.reset_count}", ) - self.ended = False - self.episode_return = 0.0 - self.episode_length = 0 - validate_online_inputs(self.actor, self.history.inputs()) def collect(self, horizon, policy_version): if self.actor.ppo_policy_version != policy_version: @@ -59,40 +46,35 @@ def collect(self, horizon, policy_version): raise ValueError( f"PPO buffer requires approximately {minimum} bytes before collection" ) - if self.ended: + if self.cursor.ended: self.reset() builder = PPOBufferBuilder( self.actor.config, policy_version, horizon, - self.history.frames, + self.cursor.history.frames, self.phase.max_buffer_bytes, ) current_reference = len(builder.frames) - 1 completed = [] with torch.no_grad(): for _ in range(horizon): - if self.ended: + if self.cursor.ended: self.reset() - current_reference = builder.append_frame(self.history.current) + current_reference = builder.append_frame( + self.cursor.history.current + ) if current_reference is None: raise ValueError( "live PPO decision lacks its observation reference" ) - current = self.history.current - decision = decide( - self.actor, self.value, self.history.inputs(), self.generator - ) - result = validate_step( - self.adapter.step(decision.domain_action), - self.phase.environment.units, - ) + current = self.cursor.history.current + decision = self.cursor.decide(self.value) + result = self.cursor.step(decision) successor_reference = None successor_value = 0.0 if result.observation is not None: - self.history.advance(decision.action, result.observation) - successor_inputs = self.history.inputs().to(self.actor.device) - features, metadata = self.actor.model_inputs(successor_inputs) + features, metadata = self.actor.model_inputs(self.cursor.inputs) if not result.terminated: successor_value = float( self.value.prepared_value(features, metadata)[0] @@ -101,7 +83,9 @@ def collect(self, horizon, policy_version): torch.tensor(successor_value, dtype=torch.float32) ): raise ValueError("non-finite PPO successor value") - successor_reference = builder.append_frame(self.history.current) + successor_reference = builder.append_frame( + self.cursor.history.current + ) reward = float(result.reward) * self.phase.reward_scale if not torch.isfinite( torch.tensor([reward, successor_value], dtype=torch.float32) @@ -128,17 +112,12 @@ def collect(self, horizon, policy_version): ) ) current_reference = successor_reference - self.episode_return += float(result.reward) - if not math.isfinite(self.episode_return): - raise ValueError("non-finite training episode return") - self.episode_length += 1 - self.ended = result.terminated or result.truncated - if self.ended: + if self.cursor.ended: completed.append( { "episode_id": current.episode_id, - "return": self.episode_return, - "length": self.episode_length, + "return": self.cursor.episode_return, + "length": self.cursor.episode_length, "terminated": result.terminated, "truncated": result.truncated, "success": result.success, @@ -150,33 +129,34 @@ def collect(self, horizon, policy_version): def state_dict(self): return { - "history": self.history.state_dict(), + "history": self.cursor.history.state_dict(), "reset_count": self.reset_count, - "generator": self.generator.get_state(), - "ended": self.ended, - "episode_return": self.episode_return, - "episode_length": self.episode_length, + "generator": self.cursor.generator.get_state(), + "ended": self.cursor.ended, + "episode_return": self.cursor.episode_return, + "episode_length": self.cursor.episode_length, } def load_state_dict(self, state, *, exact): reset_count = int(state["reset_count"]) if reset_count < 1: raise ValueError("PPO checkpoint has an invalid reset count") - self.generator.set_state(state["generator"]) + self.cursor.generator.set_state(state["generator"]) if exact: - self.history.load_state_dict(state["history"]) + self.cursor.history.load_state_dict(state["history"]) + self.cursor.refresh() if ( self.reset_count != reset_count - or self.history.current.episode_id != f"train-{reset_count - 1}" + or self.cursor.history.current.episode_id != f"train-{reset_count - 1}" ): raise ValueError( "PPO checkpoint episode identity disagrees with reset count" ) - self.ended = bool(state["ended"]) - self.episode_return = float(state["episode_return"]) - self.episode_length = int(state["episode_length"]) + self.cursor.ended = bool(state["ended"]) + self.cursor.episode_return = float(state["episode_return"]) + self.cursor.episode_length = int(state["episode_length"]) else: - self.history.reset_count = reset_count - self.ended = True - self.episode_return = 0.0 - self.episode_length = 0 + self.cursor.history.reset_count = reset_count + self.cursor.ended = True + self.cursor.episode_return = 0.0 + self.cursor.episode_length = 0 diff --git a/src/sequifier/rl/ppo_evaluation.py b/src/sequifier/rl/ppo_evaluation.py index 2e32889a..7276461e 100644 --- a/src/sequifier/rl/ppo_evaluation.py +++ b/src/sequifier/rl/ppo_evaluation.py @@ -1,17 +1,13 @@ """Bounded local policy evaluation with independent seeds, adapters and RNG.""" -import math import signal import statistics import threading import time from contextlib import contextmanager -from copy import deepcopy -from sequifier.rl.collection import decide, stream_seed, validate_online_inputs -from sequifier.rl.distributions import action_generator -from sequifier.rl.environment import load_online_adapter, validate_reset, validate_step -from sequifier.rl.history import PolicyHistory +from sequifier.rl.collection import PolicyCursor, stream_seed +from sequifier.rl.environment import load_online_adapter from sequifier.runtime.random_state import capture_rng_state, restore_rng_state @@ -54,7 +50,7 @@ def expired(signum, frame): def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): - """Evaluate a frozen snapshot; completed returns exclude censored episodes. + """Evaluate synchronously with stochastic layers disabled; completed returns exclude censored episodes. The timeout covers adapter creation/reset/steps for the entire evaluation. Adapters must release resources promptly from close(). @@ -65,16 +61,13 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): "evaluation environment units differ from the portable actor representation" ) saved_random = capture_rng_state(actor.device) + training_modes = {module: module.training for module in actor.modules()} started = time.monotonic() adapter = None episodes = [] timed_out = False - attempted = 0 - episode_return = 0.0 - length = 0 - active = False try: - snapshot = deepcopy(actor).eval() + actor.eval() with _deadline(evaluation.timeout_seconds): adapter = load_online_adapter(evaluation.environment) if adapter is forbidden_adapter: @@ -83,74 +76,49 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): "evaluation factory returned the live training adapter" ) for index, seed in enumerate(evaluation.seeds): - attempted += 1 - episode_return, length = 0.0, 0 - active = True - result = validate_reset( - adapter.reset(stream_seed(seed, "evaluation-reset", 0)), + cursor = PolicyCursor( + actor, + adapter, evaluation.environment.units, + stream_seed(seed, "evaluation-actions", 0), ) - history = PolicyHistory(snapshot.config, strict_vocabulary=True) - history.reset( - result.observation, result.static_inputs, f"evaluation-{index}" - ) - generator = action_generator( - snapshot.device, stream_seed(seed, "evaluation-actions", 0) - ) - terminated = False - success = None - for _ in range(evaluation.max_episode_steps): - decision = decide( - snapshot, - None, - history.inputs(), - generator, - deterministic=evaluation.selection == "deterministic", + completed_episode = False + try: + cursor.reset( + adapter.reset(stream_seed(seed, "evaluation-reset", 0)), + f"evaluation-{index}", ) - step = validate_step( - adapter.step(decision.domain_action), - evaluation.environment.units, + for _ in range(evaluation.max_episode_steps): + step = cursor.step( + cursor.decide( + deterministic=evaluation.selection == "deterministic" + ) + ) + if cursor.ended: + completed_episode = step.terminated + break + finally: + episodes.append( + { + "seed": seed, + "return": cursor.episode_return, + "length": cursor.episode_length, + "completed": completed_episode, + "censored": not completed_episode, + "success": cursor.last_result.success + if completed_episode and cursor.last_result is not None + else None, + } ) - length += 1 - episode_return += float(step.reward) - if not math.isfinite(episode_return): - raise ValueError("non-finite evaluation episode return") - if step.observation is not None: - history.advance(decision.action, step.observation) - validate_online_inputs(snapshot, history.inputs()) - terminated = step.terminated - if step.terminated or step.truncated: - success = step.success - break - episodes.append( - { - "seed": seed, - "return": episode_return, - "length": length, - "completed": terminated, - "censored": not terminated, - "success": success if terminated else None, - } - ) - active = False except EvaluationTimeout: timed_out = True - if active: - episodes.append( - { - "seed": evaluation.seeds[attempted - 1], - "return": episode_return, - "length": length, - "completed": False, - "censored": True, - "success": None, - } - ) finally: try: if adapter is not None: adapter.close() finally: + for module, training in training_modes.items(): + module.training = training restore_rng_state(actor.device, saved_random) completed = [episode for episode in episodes if episode["completed"]] returns = [episode["return"] for episode in completed] @@ -167,7 +135,7 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): else None, "completed_count": len(completed), "censored_count": len(episodes) - len(completed), - "unattempted_count": len(evaluation.seeds) - attempted, + "unattempted_count": len(evaluation.seeds) - len(episodes), "timed_out": timed_out, "episodes": episodes, "seconds": time.monotonic() - started, @@ -176,7 +144,7 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): if successes and all(success is not None for success in successes): result["success_rate"] = statistics.fmean(successes) if evaluation.metric == "score_mean": - if timed_out or attempted != len(evaluation.seeds): + if timed_out or len(episodes) != len(evaluation.seeds): raise ValueError("GRPO evaluation requires every finite-horizon score") result["score_mean"] = statistics.fmean( episode["return"] for episode in episodes diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index e2134d29..11c0500c 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -8,7 +8,7 @@ from sequifier.io.policy_dataset import PolicyDataCatalog from sequifier.io.prepared_policy_data import PreparedPolicyData from sequifier.model.factory import build_transformer_network -from sequifier.rl.checkpoint import phase_actor_path +from sequifier.rl.checkpoint import phase_actor_path, validate_policy_weights from sequifier.rl.environment import ( adapter_identity, load_online_adapter, @@ -161,13 +161,7 @@ def build_actor(config, fitted, statistics): config.policy_interface, ).eval() actor.units = dict(config.units) - actor.set_normalization_statistics(statistics) - if any( - (tensor.is_floating_point() and tensor.dtype != torch.float32) - or not torch.isfinite(tensor).all() - for tensor in actor.network.state_dict().values() - ): - raise ValueError("policy initialization requires finite FP32 network weights") + validate_policy_weights(actor) return actor @@ -226,11 +220,12 @@ def preflight_environments(config): def preflight_optimization(config, actor, data): """Resolve constructors and authored scheduler budgets before creating outputs.""" - plans = [] + budgets = [] for phase, part in zip(config.phases, data): size = len(part.training) if part is not None else None - roles = phase.optimization_plans(size) - for plan in roles.values(): - plan.build(actor) - plans.append(roles) - return plans + budget = phase.optimizer_step_budget(size) + for optimization in phase.optimization_configs().values(): + optimization.validate_budget(budget) + optimization.build(actor) + budgets.append(budget) + return budgets diff --git a/src/sequifier/rl/references.py b/src/sequifier/rl/references.py new file mode 100644 index 00000000..20695a0b --- /dev/null +++ b/src/sequifier/rl/references.py @@ -0,0 +1,110 @@ +"""Own reference resolution, validation and frozen weight reuse.""" + +from copy import deepcopy +from pathlib import Path + +from sequifier.rl.checkpoint import load_policy +from sequifier.rl.run_state import require_equal, weight_fingerprint +from sequifier.trajectory.sources import file_checksum + + +class PolicyReferences: + def __init__(self, config, recovery, *, resuming): + self.config, self.recovery = config, recovery + self.artifacts, self.checksums = {}, {} + if ( + resuming + and config.initialize_from + and Path(config.initialize_from).is_file() + ): + _, identity = self._artifact(config.initialize_from) + require_equal( + recovery.initialization["weight_hash"], + identity, + "resume.initialize_from.weights", + ) + for index, phase in enumerate(config.phases): + reference = getattr(phase, "reference", None) + if reference is None or reference.source != "artifact": + continue + if Path(reference.path).is_file(): + _, identity = self._artifact(reference.path) + expected = recovery.reference_contract.setdefault(str(index), identity) + require_equal( + expected, identity, f"phases.{phase.name}.reference.weights" + ) + elif index > recovery.phase_index or ( + index == recovery.phase_index and recovery.phase_state is None + ): + raise FileNotFoundError( + f"reference artifact required before phase start: {reference.path}" + ) + # Completed references and initialization are no longer needed in memory. + pending = { + phase.reference.path + for phase in config.phases[recovery.phase_index :] + if getattr(phase, "reference", None) is not None + and phase.reference.source == "artifact" + } + self.artifacts = { + path: value for path, value in self.artifacts.items() if path in pending + } + + def _artifact(self, path): + checksum = file_checksum(path) + if path not in self.artifacts or self.checksums[path] != checksum: + source = load_policy( + path, device="cpu", interface_name=self.config.policy_interface + ) + self.recovery.representation.validate_actor( + source, self.config.units, f"reference.{path}" + ) + weights = source.network.state_dict() + self.artifacts[path] = weights, weight_fingerprint(weights) + self.checksums[path] = checksum + return self.artifacts[path] + + def resolve(self, actor): + recovery = self.recovery + declaration = getattr( + self.config.phases[recovery.phase_index], "reference", None + ) + if declaration is None: + return None, None + if recovery.phase_state is not None: + weights, identity = ( + recovery.phase_state["reference"], + recovery.phase_state["reference_identity"], + ) + elif declaration.source == "artifact": + weights, identity = self._artifact(declaration.path) + require_equal( + recovery.reference_contract[str(recovery.phase_index)], + identity, + "reference.artifact.weights", + ) + else: + weights = recovery.incoming_actor + identity = weight_fingerprint(weights) + reference = deepcopy(actor).eval().requires_grad_(False) + reference.network.load_state_dict(weights, strict=True) + return reference, identity + + +def validate_reference_state(phase, recovery): + state = recovery.phase_state + if state is not None: + reference = getattr(phase, "reference", None) + if (reference is None) != (state["reference"] is None): + raise ValueError("checkpoint reference disagrees with the phase") + if reference is not None: + identity = weight_fingerprint(state["reference"]) + require_equal( + identity, state["reference_identity"], "resume.embedded_reference" + ) + expected = ( + weight_fingerprint(recovery.incoming_actor) + if reference.source == "phase_start" + else recovery.reference_contract[str(recovery.phase_index)] + ) + require_equal(expected, identity, "resume.reference.weights") diff --git a/src/sequifier/rl/registry.py b/src/sequifier/rl/registry.py index 56f8ab15..8ccb0e47 100644 --- a/src/sequifier/rl/registry.py +++ b/src/sequifier/rl/registry.py @@ -17,7 +17,6 @@ class BuiltinAlgorithm: runtime: Callable prepare_data: Callable[..., PreparedPolicyData] | None = None - state_version: int = 2 BUILTIN_ALGORITHMS = { diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 444ef864..2d5a1c67 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -1,7 +1,7 @@ """Policy recovery with artifact-backed, authoritative committed phase boundaries.""" import math -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field, fields from pathlib import Path import torch @@ -9,8 +9,9 @@ from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save, write_json_atomic from sequifier.rl.checkpoint import phase_actor_path, save_policy -from sequifier.rl.registry import BUILTIN_ALGORITHMS +from sequifier.rl.references import validate_reference_state from sequifier.rl.run_state import ( + PHASE_STATE_VERSION, ActorRepresentation, PolicyPhaseResult, clone_weights, @@ -21,6 +22,25 @@ from sequifier.trajectory.sources import file_checksum +@dataclass(frozen=True) +class CompletedPhase: + result: PolicyPhaseResult + sha256: str + result_fingerprint: str + + @classmethod + def from_dict(cls, values): + result = values["result"] + return cls( + PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}), + values["sha256"], + values["result_fingerprint"], + ) + + def as_dict(self): + return asdict(self) + + @dataclass class RunCheckpoint: plan: dict @@ -32,13 +52,16 @@ class RunCheckpoint: incoming_actor: dict random: dict reference_contract: dict = field(default_factory=dict) - results: list[PolicyPhaseResult] = field(default_factory=list) - completed_artifacts: list[dict] = field(default_factory=list) + completed: list[CompletedPhase] = field(default_factory=list) phase_state: dict | None = None + @property + def results(self): + return tuple(record.result for record in self.completed) + @property def phase_index(self): - return len(self.results) + return len(self.completed) @classmethod def load(cls, path, config, plan, data_identity): @@ -53,29 +76,43 @@ def load(cls, path, config, plan, data_identity): if ( not isinstance(payload, dict) or payload.get("artifact_type") != "sequifier_policy_run" - or payload.get("format_version") != 5 + or payload.get("format_version") not in {5, 6} ): - raise ValueError("policy resume requires a version-5 policy run checkpoint") + raise ValueError( + "policy resume requires a version-5 or version-6 policy run checkpoint" + ) try: require_equal(plan, payload["plan"], "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") - results = [ - PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}) - for result in payload["completed_results"] + if payload["format_version"] == 5: + records = payload.pop("completed_artifacts") + results = payload.pop("completed_results") + require_equal(len(results), len(records), "resume.completed_phases") + for result, record in zip(results, records): + require_equal( + result["provenance"]["actor_path"], + record["path"], + "resume.actor_path", + ) + payload["completed"] = [ + { + "result": result, + "sha256": record["sha256"], + "result_fingerprint": record["result_fingerprint"], + } + for result, record in zip(results, records) + ] + completed = [ + CompletedPhase.from_dict(record) for record in payload["completed"] ] checkpoint = cls( - payload["plan"], - ActorRepresentation(payload["representation"]), - payload["data_identity"], - payload["data_contract"], - payload["adapter_contract"], - payload["initialization"], - payload["incoming_actor"], - payload["random"], - payload["reference_contract"], - results, - payload["completed_artifacts"], - payload["phase_state"], + **{ + f.name: payload[f.name] + for f in fields(cls) + if f.name not in {"representation", "completed"} + }, + representation=ActorRepresentation(payload["representation"]), + completed=completed, ) require_equal( checkpoint.representation.identity, @@ -88,23 +125,20 @@ def load(cls, path, config, plan, data_identity): raise ValueError(f"malformed policy run checkpoint: {exc}") from exc def validate(self, config): - if len(self.results) > len(config.phases) or len( - self.completed_artifacts - ) != len(self.results): + if self.phase_index > len(config.phases): raise ValueError("checkpoint completed handoffs disagree with the plan") validate_finite_state(self.incoming_actor, "incoming_actor") # Parsing the owned representation also validates the persisted policy schema. self.representation.policy - for index, result in enumerate(self.results): + for index, record in enumerate(self.completed): + result = record.result phase = config.phases[index] require_equal( (index, phase.name, phase.algorithm, phase.handoff), (result.index, result.name, result.algorithm, result.handoff), "resume.completed_phase", ) - record = self.completed_artifacts[index] path = phase_actor_path(config.phase_directory, index, phase.name) - require_equal(str(path), record["path"], "resume.phase_actor.path") require_equal( str(path), result.provenance["actor_path"], "resume.actor_path" ) @@ -114,10 +148,10 @@ def validate(self, config): "resume.report_path", ) require_equal( - record["sha256"], file_checksum(path), "resume.phase_actor.sha256" + record.sha256, file_checksum(path), "resume.phase_actor.sha256" ) require_equal( - record["result_fingerprint"], + record.result_fingerprint, fingerprint(result.as_dict()), "resume.phase_result", ) @@ -135,8 +169,7 @@ def validate(self, config): return if self.phase_index == len(config.phases): raise ValueError("completed run cannot have a live phase state") - phase = config.phases[self.phase_index] - if state["version"] != BUILTIN_ALGORITHMS[phase.algorithm].state_version: + if state["version"] != PHASE_STATE_VERSION: raise ValueError("unsupported phase checkpoint state version") transaction = state["transaction"] if ( @@ -160,20 +193,7 @@ def validate(self, config): raise ValueError( "checkpoint best actor, metric and transaction must be recorded together" ) - reference = getattr(phase, "reference", None) - if (reference is None) != (state["reference"] is None): - raise ValueError("checkpoint reference disagrees with the phase") - if reference is not None: - identity = weight_fingerprint(state["reference"]) - require_equal( - identity, state["reference_identity"], "resume.embedded_reference" - ) - expected = ( - incoming_identity - if reference.source == "phase_start" - else self.reference_contract[str(self.phase_index)] - ) - require_equal(expected, identity, "resume.reference.weights") + validate_reference_state(config.phases[self.phase_index], self) validate_finite_state(state, "phase") def validate_actor(self, actor, units): @@ -207,20 +227,15 @@ def save(self, path, device, state=None): path, { "artifact_type": "sequifier_policy_run", - "format_version": 5, - "plan": self.plan, + "format_version": 6, + **{ + f.name: getattr(self, f.name) + for f in fields(self) + if f.name not in {"representation", "completed"} + }, "representation": self.representation.canonical, "representation_fingerprint": self.representation.identity, - "data_identity": self.data_identity, - "data_contract": self.data_contract, - "adapter_contract": self.adapter_contract, - "reference_contract": self.reference_contract, - "initialization": self.initialization, - "random": self.random, - "incoming_actor": self.incoming_actor, - "completed_artifacts": self.completed_artifacts, - "completed_results": [result.as_dict() for result in self.results], - "phase_state": self.phase_state, + "completed": [record.as_dict() for record in self.completed], }, ) @@ -233,14 +248,10 @@ def finish_phase(self, actor, result): ) path = result.provenance["actor_path"] save_policy(actor, path, metadata=result.as_dict()) - record = { - "path": path, - "sha256": file_checksum(path), - "result_fingerprint": fingerprint(result.as_dict()), - } weights = clone_weights(actor.network) - self.results.append(result) - self.completed_artifacts.append(record) + self.completed.append( + CompletedPhase(result, file_checksum(path), fingerprint(result.as_dict())) + ) self.incoming_actor = weights self.phase_state = None diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 8f4058c9..17d9615b 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -3,14 +3,12 @@ import hashlib import json import math -from copy import deepcopy from dataclasses import dataclass from typing import TYPE_CHECKING, Callable, Generic, TypeVar import torch from sequifier.config.policy_config import CategoricalPolicyConfig -from sequifier.config.policy_optimization import OptimizationPlan from sequifier.config.policy_plan import PhaseConfig from sequifier.rl.policy import CategoricalPolicy from sequifier.runtime.random_state import capture_rng_state, restore_rng_state @@ -20,6 +18,7 @@ PhaseType = TypeVar("PhaseType", bound=PhaseConfig) DataType = TypeVar("DataType") +PHASE_STATE_VERSION = 2 def clone_weights(module): @@ -125,13 +124,12 @@ class PhaseContext(Generic[PhaseType, DataType]): index: int phase: PhaseType actor: CategoricalPolicy - representation: ActorRepresentation data: DataType reference: CategoricalPolicy | None reference_identity: str | None resume_state: dict | None commit: Callable[[dict], None] - optimization: dict[str, OptimizationPlan] + step_budget: int | None class Selection: @@ -174,7 +172,7 @@ def state_dict(self): } def finish(self, context, transaction, clocks, metrics, provenance): - actor = deepcopy(context.actor).eval() + actor = context.actor.eval() if self.phase.handoff == "best": if self.best_weights is None: raise ValueError( @@ -210,10 +208,11 @@ def __init__(self, context, *, modules=None, frozen=None): self.context = context self.modules = {"actor": context.actor.network, **(modules or {})} self.networks = {**self.modules, **(frozen or {})} - if self.modules.keys() != context.optimization.keys(): + self.configs = context.phase.optimization_configs() + if self.modules.keys() != self.configs.keys(): raise ValueError("phase optimization roles disagree with trainable modules") self.optimization = { - role: context.optimization[role].build(module) + role: self.configs[role].build(module) for role, module in self.modules.items() } self.steps = dict.fromkeys(self.modules, 0) @@ -242,7 +241,7 @@ def __init__(self, context, *, modules=None, frozen=None): optimizer.load_state_dict(saved["optimizer"]) scheduler.load_state_dict(saved["scheduler"]) steps = saved["steps"] - budget = context.optimization[role].step_budget + budget = context.step_budget if ( (not isinstance(steps, int)) or steps < self.transaction @@ -269,7 +268,7 @@ def step(self): role: float( torch.nn.utils.clip_grad_norm_( module.parameters(), - self.context.optimization[role].gradient_clip, + self.configs[role].gradient_clip, error_if_nonfinite=True, ) ) @@ -303,7 +302,7 @@ def checkpoint(self, final, **extra): self.context.commit( { **extra, - "version": 2, + "version": PHASE_STATE_VERSION, "networks": { name: clone_weights(module) for name, module in self.networks.items() diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index 4eace4ac..b53b93d2 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -8,19 +8,15 @@ from sequifier.config.policy_infer_config import PolicyRolloutConfig from sequifier.config.rollout_config import InputBinding from sequifier.io.values import numeric_scalar -from sequifier.rl.collection import decide, validate_online_inputs -from sequifier.rl.distributions import action_generator +from sequifier.rl.collection import PolicyCursor from sequifier.rl.environment import ( OnlineEnvironmentAdapter, OnlineResetResult, require_seeded_reset, validate_online_adapter, validate_reset, - validate_step, ) -from sequifier.rl.history import PolicyHistory from sequifier.rl.policy import CategoricalPolicy -from sequifier.trajectory.returns import remaining_return from sequifier.trajectory.schema import SeedHistory, Trajectory, Transition @@ -76,15 +72,11 @@ def collect_policy_rollout( units, ) episode_id = str(uuid4()) - history = PolicyHistory(spec) - history.reset( - initial.observation, - initial.static_inputs, - episode_id, - returns[0] if returns else None, - ) policy.eval() - validate_online_inputs(policy, history.inputs()) + cursor = PolicyCursor( + policy, adapter, units, random_seed, strict_vocabulary=False + ) + cursor.reset(initial, episode_id, returns[0] if returns else None) reset = validate_reset( adapter.reset_from_observation(deepcopy(initial), random_seed), units ) @@ -94,14 +86,11 @@ def collect_policy_rollout( raise ValueError( "seeded reset must preserve the supplied observation and static inputs" ) - generator = action_generator(policy.device, random_seed) transitions, previous_actions, log_probabilities = [], [], [] for step_id in range(max_steps): - current = history.current - decision = decide( - policy, None, history.inputs(), generator, deterministic=deterministic - ) - result = validate_step(adapter.step(decision.domain_action), units) + current = cursor.history.current + decision = cursor.decide(deterministic=deterministic) + result = cursor.step(decision) truncated = result.truncated or ( config.max_episode_steps is not None and step_id + 1 >= config.max_episode_steps @@ -109,17 +98,11 @@ def collect_policy_rollout( previous_actions.append(current.previous_action) log_probabilities.append(decision.log_probability) if spec.return_conditioned: - returns.append( - remaining_return(returns[-1], result.reward, spec.returns.gamma) - ) + assert cursor.desired_return is not None + returns.append(cursor.desired_return) successor = ( dict(result.observation) if result.observation is not None else None ) - if successor is not None: - history.advance( - decision.action, successor, returns[-1] if returns else None - ) - validate_online_inputs(policy, history.inputs()) transitions.append( Transition( episode_id=episode_id, diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 16105fba..4de386d1 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -4,6 +4,7 @@ from sequifier.config.policy_plan import PolicyImitationPhaseConfig from sequifier.io.batch import collate_sequence_batches +from sequifier.io.policy_dataset import imitation_example from sequifier.io.policy_window import action_global_ids, action_target from sequifier.rl.batch_order import BatchOrder from sequifier.rl.offline_data import OfflineData @@ -24,7 +25,10 @@ def evaluate_imitation(actor, dataset, batch_size): total = 0.0 for start in range(0, len(dataset), batch_size): batch = collate_sequence_batches( - [dataset[i] for i in range(start, min(start + batch_size, len(dataset)))] + [ + imitation_example(dataset, i) + for i in range(start, min(start + batch_size, len(dataset))) + ] ) total += float(action_nll(actor, batch).sum()) return total / len(dataset) @@ -56,7 +60,9 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin total = 0.0 for start in range(0, len(group), phase.batch_size): indices = group[start : start + phase.batch_size] - batch = collate_sequence_batches([training[i] for i in indices]) + batch = collate_sequence_batches( + [imitation_example(training, i) for i in indices] + ) loss = action_nll(actor, batch).sum() / len(group) if not torch.isfinite(loss): raise ValueError("non-finite imitation action likelihood") diff --git a/src/sequifier/trajectory/tables.py b/src/sequifier/trajectory/tables.py index ad0c152f..f893e2ae 100644 --- a/src/sequifier/trajectory/tables.py +++ b/src/sequifier/trajectory/tables.py @@ -77,6 +77,55 @@ def load(self, identity): return pl.from_arrow(rows.slice(start - self.offsets[first], end - start)) +def model_decoder(metadata, action_column, action_values): + """Decode model-encoded scalars using one fixed set of inverse codecs.""" + inverse_maps = ( + { + column: {encoded: label for label, encoded in mapping.items()} + for column, mapping in metadata.id_maps.items() + } + if metadata is not None + else {} + ) + + def decode(value, column): + value = domain_scalar(value, column) + if metadata is not None: + if column in inverse_maps: + inverse = inverse_maps[column] + if ( + isinstance(value, str) + or value != int(value) + or int(value) not in inverse + ): + raise ValueError(f"{column}: encoded ID has no domain label") + value = inverse[int(value)] + if column == action_column: + if isinstance(value, str) and not isinstance(action_values[0], str): + try: + value = numeric_scalar(float(value), column) + except ValueError: + pass + matches = [ + action + for action in action_values + if action == value or str(action) == str(value) + ] + if len(matches) == 1: + value = matches[0] + elif ( + metadata.normalize_real_columns + and column in metadata.selected_columns_statistics + ): + value = denormalize_values( + numeric_scalar(value, column), + metadata.selected_columns_statistics[column], + ) + return value + + return decode + + @dataclass(frozen=True) class TableEpisodeReference: source: SequenceSourceConfig @@ -92,48 +141,11 @@ def load(self) -> SequenceEpisode: data = ordered_sequence_rows( self.table.load(self.sequence_id), columns.item_position ) - metadata = self.metadata - model_encoded = self.prepared or self.source.encoding == "model" - - def decode(value, column): - value = domain_scalar(value, column) - if model_encoded and metadata is not None: - if column in metadata.id_maps: - inverse = { - encoded: label - for label, encoded in metadata.id_maps[column].items() - } - if ( - isinstance(value, str) - or value != int(value) - or int(value) not in inverse - ): - raise ValueError(f"{column}: encoded ID has no domain label") - value = inverse[int(value)] - if column == (columns.action or self.policy.action_column): - if isinstance(value, str) and not isinstance( - self.policy.action_values[0], str - ): - try: - value = numeric_scalar(float(value), column) - except ValueError: - pass - matches = [ - action - for action in self.policy.action_values - if action == value or str(action) == str(value) - ] - if len(matches) == 1: - value = matches[0] - elif ( - metadata.normalize_real_columns - and column in metadata.selected_columns_statistics - ): - value = denormalize_values( - numeric_scalar(value, column), - metadata.selected_columns_statistics[column], - ) - return value + decode = model_decoder( + self.metadata if self.prepared or self.source.encoding == "model" else None, + columns.action or self.policy.action_column, + self.policy.action_values, + ) preceding = None if self.prepared: @@ -164,24 +176,7 @@ def decode(value, column): ) -def table_episode( - data, - policy, - columns, - *, - sequence_id, - preceding=None, - static_data=None, - episode_end="unknown", - episode_namespace="", - decode=domain_scalar, -) -> SequenceEpisode: - """Canonical table-to-episode validation shared by training and inference.""" - data = ordered_sequence_rows(data, columns.item_position) - rows = list(data.iter_rows(named=True)) - if not rows: - raise ValueError("sequence source changed while loading") - action_column = columns.action or policy.action_column +def table_features(data, policy, columns, static_data): observations, static_inputs = ( {name: mapping.get(name, name) for name in names} for names, mapping in ( @@ -189,6 +184,7 @@ def table_episode( (policy.static_columns, columns.static_inputs), ) ) + action_column = columns.action or policy.action_column inputs = set(observations.values()) if static_data is None: inputs.update(static_inputs.values()) @@ -222,14 +218,36 @@ def table_episode( f"policy table is missing columns: {sorted(required - set(data.columns))}" ) + return observations, static_inputs + + +def table_episode( + data, + policy, + columns, + *, + sequence_id, + preceding=None, + static_data=None, + episode_end="unknown", + episode_namespace="", + decode=domain_scalar, +) -> SequenceEpisode: + """Canonical table-to-episode validation shared by training and inference.""" + data = ordered_sequence_rows(data, columns.item_position) + if data.is_empty(): + raise ValueError("sequence source changed while loading") + action_column = columns.action or policy.action_column + observations, static_inputs = table_features(data, policy, columns, static_data) + def values(column, decoder=decode): - return tuple(decoder(row[column], column) for row in rows) + return tuple(decoder(value, column) for value in data[column]) def optional_values(column, decoder=decode): if ( column is None or column not in data.columns - or data[column].null_count() == len(rows) + or data[column].null_count() == len(data) ): return None return values(column, decoder) @@ -255,12 +273,11 @@ def constant(entries, label): if static_data is not None: if len(static_data) != 1: raise ValueError("static data requires exactly one row per sequenceId") - static_rows = list(static_data.iter_rows(named=True)) else: - static_rows = rows + static_data = data static = { name: constant( - (decode(row[column], column) for row in static_rows), + (decode(value, column) for value in static_data[column]), f"static input {name}", ) for name, column in static_inputs.items() @@ -276,13 +293,13 @@ def constant(entries, label): current = transitions.get(declared) if current is not None and not current[-1]: raise ValueError("episode_end conflicts with the recorded final boundary") - transitions[declared] = (False,) * (len(rows) - 1) + (True,) + transitions[declared] = (False,) * (len(data) - 1) + (True,) successors = None if columns.successors: if columns.successors.keys() != observations.keys(): raise ValueError("successor mapping must cover every observation column") successors = [] - for row in rows: + for row in data.iter_rows(named=True): present = [ row[column] is not None for column in columns.successors.values() ] From 082225833e1d744e662d094008bcc44268ab868a Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 10 Nov 2026 11:45:20 +0100 Subject: [PATCH 23/71] unify supervised and policy --- documentation/configs/train.md | 8 +- documentation/policy-learning.md | 13 +- documentation/supervised-policy-learning.md | 220 ++++++++++ .../supervised-policy-unification-plan.md | 236 +++++++++++ ...nreleased-supervised-policy-unification.md | 23 ++ src/sequifier/artifacts/model_artifact.py | 4 +- src/sequifier/artifacts/model_config.py | 57 +++ src/sequifier/artifacts/model_export.py | 2 + src/sequifier/config/infer_config.py | 4 +- src/sequifier/config/learning_run.py | 97 +++++ src/sequifier/config/policy_config.py | 109 ++++- src/sequifier/config/policy_infer_config.py | 4 + src/sequifier/config/policy_optimization.py | 45 +- src/sequifier/config/policy_plan.py | 2 + src/sequifier/config/policy_train_config.py | 4 + src/sequifier/config/train_config.py | 183 +++++++- src/sequifier/consolidated-docs.md | 21 +- src/sequifier/io/actions.py | 129 ++++++ src/sequifier/io/episode_corpus.py | 4 +- src/sequifier/io/episode_features.py | 9 + src/sequifier/io/iql_dataset.py | 12 +- src/sequifier/io/policy_dataset.py | 12 + src/sequifier/io/policy_materialization.py | 92 +++++ src/sequifier/io/policy_window.py | 75 +++- src/sequifier/io/sequence.py | 11 +- src/sequifier/io/tensors.py | 12 +- src/sequifier/io/values.py | 21 +- src/sequifier/model/factory.py | 21 +- src/sequifier/model/parameter_catalog.py | 4 +- .../model/prediction_distribution.py | 197 +++++++++ src/sequifier/rl/attachment.py | 150 +++++++ src/sequifier/rl/checkpoint.py | 25 +- src/sequifier/rl/collection.py | 25 +- src/sequifier/rl/distributions.py | 4 +- src/sequifier/rl/environment.py | 25 +- src/sequifier/rl/grpo_buffer.py | 7 +- src/sequifier/rl/grpo_collector.py | 10 +- src/sequifier/rl/history.py | 26 +- src/sequifier/rl/inference.py | 71 +++- src/sequifier/rl/iql.py | 6 +- src/sequifier/rl/joint_distribution.py | 265 ++++++++++++ src/sequifier/rl/learner.py | 28 ++ src/sequifier/rl/models.py | 70 +++- src/sequifier/rl/policy.py | 180 +++++++- src/sequifier/rl/ppo_buffer.py | 37 +- src/sequifier/rl/ppo_collector.py | 4 +- src/sequifier/rl/ppo_update.py | 4 +- src/sequifier/rl/preparation.py | 5 +- src/sequifier/rl/references.py | 5 +- src/sequifier/rl/run_checkpoint.py | 10 +- src/sequifier/rl/run_state.py | 21 +- src/sequifier/rl/runtime.py | 27 +- src/sequifier/rl/scoring.py | 10 +- src/sequifier/rl/training.py | 3 + src/sequifier/rl_api.py | 7 +- src/sequifier/runtime/builder.py | 15 +- .../train-policy-from-supervised.yaml | 19 + src/sequifier/train.py | 13 +- src/sequifier/training/checkpoint_service.py | 5 + src/sequifier/training/coordinator.py | 390 ++++++++++++++++++ src/sequifier/training/engine.py | 1 + src/sequifier/training/loss.py | 28 +- src/sequifier/training/optimization.py | 34 +- src/sequifier/training_api.py | 5 + src/sequifier/trajectory/export.py | 34 +- src/sequifier/trajectory/importers.py | 23 +- src/sequifier/trajectory/prepared.py | 18 +- src/sequifier/trajectory/schema.py | 98 ++++- src/sequifier/trajectory/tables.py | 175 +++++++- 69 files changed, 3295 insertions(+), 219 deletions(-) create mode 100644 documentation/supervised-policy-learning.md create mode 100644 documentation/supervised-policy-unification-plan.md create mode 100644 release-notes/unreleased-supervised-policy-unification.md create mode 100644 src/sequifier/config/learning_run.py create mode 100644 src/sequifier/io/actions.py create mode 100644 src/sequifier/io/policy_materialization.py create mode 100644 src/sequifier/model/prediction_distribution.py create mode 100644 src/sequifier/rl/attachment.py create mode 100644 src/sequifier/rl/joint_distribution.py create mode 100644 src/sequifier/templates/train-policy-from-supervised.yaml create mode 100644 src/sequifier/training/coordinator.py diff --git a/documentation/configs/train.md b/documentation/configs/train.md index adba68db..d6d9b4f9 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -779,9 +779,11 @@ PPO environment steps and GRPO groups are explicit per-phase budgets. Each phase selects `handoff: best | last` (default `best`). One frozen actor representation and causal history contract spans the run; all optimizer/critic state starts fresh at each handoff. Return conditioning supports imitation only. -Policy runs are local FP32 runs and cannot contain supervised forecasting phases. -The supervised `training_plan`/`dataset_training`/`global_training` schema above -remains independent. +Standalone legacy policy runs use their existing phase schema. Use +`run_type: unified` for supervised and policy phases in one plan, or +`initialize_from` plus `policy_binding.actions` to post-train an ordinary +supervised artifact. See the [unified learning guide](../supervised-policy-learning.md) +for typed actions, source bindings, prediction distributions, and recovery. See the [policy learning guide](../policy-learning.md) for full fields, objective formulas, environment capabilities, preference manifests, checkpoint boundaries, diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 3c8dae43..9143c359 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -1,4 +1,9 @@ -# Categorical policy learning +# Policy learning + +For supervised checkpoint attachment, mixed real/categorical heads, full model +input schemas, and mixed supervised/policy plans, see the +[unified learning guide](supervised-policy-learning.md). The configuration below +documents the compatible single-categorical authoring surface. `sequifier train` and `sequifier.rl_api.run_policy_training` execute the same `PolicyRunConfig`. Its ordered `phases` may contain `policy_imitation`, `iql`, @@ -6,12 +11,12 @@ of its predecessor's selected actor. Optimizers, schedulers, critics, samplers, and environments start fresh at phase boundaries. -This release supports one fixed categorical action, scalar or categorical +This legacy surface supports one fixed categorical action, scalar or categorical observations/static fields, causal fixed-length histories, FP32, and local single-process CPU/CUDA/MPS execution. Return conditioning is available only for imitation. A run containing another algorithm requires an unconditioned -actor throughout. Forecasting and policy phases cannot share a training plan. -Text tokenizers, continuous/joint actions, images/depth, distributed training, +actor throughout. Mixed phases use the unified run configuration linked above. +For this legacy surface, text tokenizers, continuous/joint actions, images/depth, distributed training, vectorized environments, and reward models are outside this contract. ## Authoring a run diff --git a/documentation/supervised-policy-learning.md b/documentation/supervised-policy-learning.md new file mode 100644 index 00000000..769f0dc1 --- /dev/null +++ b/documentation/supervised-policy-learning.md @@ -0,0 +1,220 @@ +# Supervised models and policy learning + +Policy learning uses the same model interfaces, input tensors, decoder branches, +categorical codecs, and portable artifacts as supervised learning. A policy +binding identifies the output heads that form an environment action. The +binding does not automatically add inputs or replace a pretrained decoder. + +## Post-train a supervised artifact + +An ordinary supervised PT model, or a supported supervised run checkpoint, can +initialize a policy run. Its interface does not need existing policy metadata. + +```yaml +project_root: . +model_name: posttrained +initialize_from: models/supervised/model.pt +policy_interface: main +policy_binding: + actions: + mode: {type: categorical, values: ["off", "on"]} + price: {type: real, scale: 0.1} + quantity: {type: real, scale: 0.5} +sources: + demonstrations: + paths: [data/demonstrations.parquet] + format: table +phases: + - name: imitation + algorithm: policy_imitation + sources: [demonstrations] + epochs: 5 +``` + +Use the actual interface and target names from the source artifact. Quote YAML +labels such as `"off"` and `"on"` if the YAML parser interprets them as booleans. +Omitting categorical `values` selects the artifact's entire decoder vocabulary, +including any executable special tokens. Explicit values select a subset and +are reported as an adaptation when that changes the original support. + +The artifact provides the architecture, context length, target positions, +normalization, and class maps. A separately supplied model must agree with it. +Original weights load strictly. Only explicitly introduced exploration parameters +are initialized. The run report records the transfer and its new/reused parameters. + +## Real and categorical heads + +Every real action head requires an explicit positive `scale`, measured in the +head's physical units. Its pretrained scalar output is the Gaussian location; +categorical heads retain their logits and persisted decoder-ID semantics. + +| Setting | Meaning | +| --- | --- | +| `scale` | Fixed Gaussian standard deviation by default. | +| `learn_scale: true` | Learn a global log scale, bounded by `min_scale` and `max_scale`. | +| `lower` / `upper` | Transform the Gaussian onto a positive, negative, or finite open interval using a softplus or logistic bijection. | +| `atoms: {0.0: 0.05}` | Assign fixed probability mass to an exact real value, alongside the continuous component. Probabilities must sum to less than one. | +| `active_when: {mode: [on]}` | Activate a field only for these values of an earlier action field. | +| `inactive_value: 0.0` | Deterministic value used when a field is inactive. | + +Bounds change the deterministic prediction function and are reported as an +adaptation. Samples are never clipped and then scored as unclipped Gaussians. +Collection retains transform latents for stable replay near numerical boundaries. +Externally supplied exact endpoints require a declared point mass. Inactive +fields are point masses with zero log-probability contribution, rather than +Gaussians with zero variance. + +The existing decoder determines factorization. Ordinary branches predict +independent fields; an autoregressive branch samples in its existing target +order, embedding each sampled prefix value. Recorded actions are scored with +their recorded prefix. Selecting a later autoregressive target requires selecting +all its predecessors. Composite branches and tied weights are retained. Hashed +categorical outputs are normalized over canonical executable candidates. + +Likelihoods sum over active action fields once per decision. Supervised loss +weights do not alter PPO ratios or DPO likelihoods. Independent categorical and +unbounded Gaussian entropy is exact; dependent/transformed entropy uses a +score-function estimator that includes the sampled prefix distribution. These +estimates can have higher variance. The GRPO sampled reference penalty retains +its existing interpretation. + +All five algorithms consume this joint action contract: imitation, IQL, PPO, +DPO, and GRPO. IQL projects real action fields and embeds categorical fields +before evaluating its scalar twin Q-functions. + +## Inputs, history, and output positions + +The full persisted execution schema describes scalar, static, depth, image, and +mask tensors. Online adapters can return nested JSON values, or array/tensor +payloads that can be converted to portable values. Supply images with the +model's existing crop, channel, frame-capacity, and byte conventions. Depth +arrays and their named masks must match the source execution schema. + +Prepared sources reuse depth masks and the existing image store. Table sources +can provide nested Parquet payloads. Use `columns.actions` to map logical action +names to table columns; `columns.previous_actions` maps preceding recorded actions. +Exported sequence shards preserve named action records and optional real latents. + +No previous-action input is inserted by default. To feed an existing model input +from a lagged action, declare, for example: + +```yaml +policy_binding: + previous_action_inputs: {last_quantity: quantity} + initial_inputs: {last_quantity: 0.0} + actions: + quantity: {type: real, scale: 0.5} +``` + +`initial_inputs` supplies the first decision's lagged value. The existing input's +codec and normalization are retained. Temporal padding, action activity, and +missing training labels are distinct. Complete relevant action tuples are +required for likelihood training; missing predecessors are not silently masked. + +`decision_position` defaults to `-1`. Other negative positions select existing +decoder positions while preserving support and prediction geometry. A model +with several prediction positions does not automatically execute an action block. +Only the selected current decision is executed by these runners. + +Forward attention is persisted separately from the learning objective. A +bidirectional encoder can consume an entirely observed history. Each historical +decision is reconstructed at its actual availability boundary, so later +observations cannot leak into its likelihood. An input unavailable at action time +must be supplied through an explicit task adaptation. + +Return conditioning is opt-in for attached models with a `__policy_return` +input. Offline conditioning needs a known reward tail. Online conditioning needs +`initial_desired_return`; subsequent budgets follow the configured return update. + +## Mixed training plans + +`run_type: unified` accepts ordinary supervised settings and policy sources in +one ordered plan: + +```yaml +run_type: unified +project_root: . +model_name: mixed_run +initialize_from: models/supervised/model.pt +policy_interface: main +policy_bindings: + main: + actions: + quantity: {type: real, scale: 0.5} +global_training: + training_objective: causal + context_length: 32 + batch_size: 32 + inference_batch_size: 32 + learning_rate: 0.0003 +dataset_training: + supervised: + model_interface: main + parts: + train: {metadata_config_path: configs/metadata.json} + criterion: {quantity: MSELoss} +sources: + demonstrations: {paths: [data/demonstrations.parquet], format: table} +phases: + - name: supervised_before + algorithm: supervised + mode: sequential + sources: [{source: supervised.train}] + epochs: 2 + - name: policy + algorithm: policy_imitation + sources: [demonstrations] + epochs: 2 + - name: supervised_after + algorithm: supervised + mode: sequential + sources: [{source: supervised.train}] + epochs: 1 +resume: {policy: if_exists, mode: reset_environments} +``` + +Replace the middle phase with IQL, PPO, DPO, or GRPO using that algorithm's +ordinary configuration. A policy phase may select `model_interface`; otherwise +it uses `policy_interface`. All interfaces remain in the model artifact. +Updates to a shared backbone can change other interfaces' predictions. + +The coordinator preflights source and environment contracts, commits incoming +artifact identities, and hands each phase to its native runner. Optimizers, +critics, references, and rollout state are phase-local. `handoff: best` or `last` +selects the complete actor, including exploration parameters. Native checkpoints +recover in-phase progress; atomic phase receipts recover completed handoffs. +Mixed plans currently execute locally. Existing standalone supervised distributed +execution remains available. Exact online resume still requires the adapter's +snapshot capabilities. + +The Python API exposes `LearningRunConfig` and `run_learning_plan` from +`sequifier.training_api`, and `Policy`, `PolicyBinding`, `PolicyConfig`, +`ActionFieldConfig`, and `load_policy` +from `sequifier.rl_api`. `load_policy(..., binding=...)` attaches a task directly +to an ordinary supervised artifact. + +## Distribution-aware supervised losses and inference + +Existing MSE, MAE, cross-entropy, and hashed-component losses retain their +semantics. An interface may declare `prediction_distributions` for real targets +and select `GaussianNLLLoss` in `dataset_training.criterion`. This uses the same +real distribution implementation as policy likelihoods and stores learned scales +in the model artifact. + +Artifacts retain ordinary point-prediction execution. For an artifact containing +policy metadata, set `execution_mode: prediction` to select ordinary forecast +inference, or `execution_mode: policy` for action sampling. Joint policy +inference writes every executed field; `output_probabilities: true` also writes +the joint action log probability. Deterministic generation uses sequential +locations/argmax choices; it does not claim a global mode of an autoregressive +joint distribution. + +New model artifacts use format version 3; versions 1 and 2 remain readable. +Joint trajectories and PPO collection records use version 2. Existing +single-categorical configuration and inference APIs remain available as +compatibility surfaces. ONNX remains a prediction export, not a general training +checkpoint or an automatically exported stochastic sampler. + +This implementation has not been exercised with tests or training experiments +yet. Probability, architecture-parity, learning, and interruption acceptance +checks remain to be performed before release. diff --git a/documentation/supervised-policy-unification-plan.md b/documentation/supervised-policy-unification-plan.md new file mode 100644 index 00000000..953cc063 --- /dev/null +++ b/documentation/supervised-policy-unification-plan.md @@ -0,0 +1,236 @@ +# Supervised and policy learning unification plan + +Status: proposed implementation plan, based on the current source and the two supplied analyses. No training implementation or experimental validation is included in this change. + +## 1. Intended outcome + +Unify around one model execution contract, one portable model artifact, and one run plan with algorithm-specific phase runners. Supervised objectives and policy objectives should consume the same model and prediction semantics. Episodes, preferences, critics, and online collection remain explicit algorithm-specific concepts. + +The two completion requirements are: + +1. **Architecture parity:** every architecture currently admitted by supervised learning can participate as an actor in policy learning, including arbitrary combinations of the currently supported scalar real and categorical targets, ordinary and autoregressive decoder branches, multiple interfaces, structured/depth/image ingestion, static conditioning, hashing, and nontrivial decoder support and prediction positions. +2. **Supervised-to-policy transfer:** every reconstructable supervised Sequifier model artifact can initialize a policy phase without first being trained or exported as a policy. Existing compatible prediction weights and preprocessing are retained. Necessary stochastic semantics are supplied explicitly for deterministic real heads. + +“Every architecture” concerns the model's supported execution surface. It does not mean that a dataset automatically contains rewards, preferences, successor states, or observations that are available at action time. Every policy still needs an explicit task binding and the data or environment required by its algorithm. A model consuming information unavailable at decision time needs a declared input adaptation; there is no exact behavior-preserving transfer in that case. + +This plan covers trainable Sequifier artifacts and supported run-checkpoint extraction. ONNX inference artifacts alone do not contain the complete training contract and are not a general source for further training. Retain existing supervised export behavior; specify policy deployment export capabilities separately rather than claiming arbitrary stochastic sampling exports automatically. + +## 2. What the current code establishes + +The supplied analyses correctly identify shared model construction and tensors, with separate policy configuration and execution. The following details affect the proposed work: + +| Boundary | Current implementation | Consequence | +| --- | --- | --- | +| Actor | `rl/policy.py` wraps the ordinary network but synthesizes policy inputs, metadata, and one categorical target. | Replace categorical assumptions without introducing a second architecture resolver. | +| Configuration | `PolicyRunConfig.resolve` permits only the selected interface, one action target, scalar fields, and decoder `support=prediction_length=1`. | Typed actions alone will not deliver architecture parity. | +| Transfer | `rl/learner.py::_initialize_actor` calls `load_policy`, then validates identical policy representation before loading weights. | A shared artifact file format currently does not imply supervised-to-policy compatibility. | +| Real targets | `TargetDecoderBranch` and the autoregressive branch already emit scalar real predictions. | Reuse those predictions as location parameters; do not discard and relearn the heads. | +| Autoregression | `AutoregressiveTransformerDecoderBranch` already supports teacher forcing and greedy generation across ordered targets at one temporal position. | Add stochastic generation to this execution path and score actual prefixes consistently. | +| IQL | `HistoryCritic` already evaluates a supplied categorical action through `nn.Embedding` and emits a scalar Q-value. | Analysis 2 overstates this particular redesign: generalize the action encoder, not an all-actions Q-vector architecture. | +| Environment | `OnlineEnvironmentAdapter.step` already accepts a mapping of action names to domain scalars. | Its action container can carry multiple scalar fields; collection, validation, history, and storage still assume one field. Observation transport must expand for non-scalar model inputs. | +| Trajectories | Records already contain named predictions, but policy validation restricts them to one action head and nonpositive log probabilities. | Version the policy record contract; continuous log densities can be positive. | +| Execution metadata | `artifacts/model_config.py` reconstructs execution using training-objective fields; the factory derives attention behavior from an objective. | Separate persisted forward semantics from the objective used to update parameters. | + +Primary implementation references: [configuration](../src/sequifier/config/train_config.py), [actor](../src/sequifier/rl/policy.py), [distributions](../src/sequifier/rl/distributions.py), [decoders](../src/sequifier/model/decoders.py), [network](../src/sequifier/model/network.py), [critics](../src/sequifier/rl/models.py), [artifact restoration](../src/sequifier/artifacts/model_config.py), and [trajectory validation](../src/sequifier/trajectory/schema.py). + +## 3. The shared contract + +Use this separation: + +```text +Authored model + persisted execution metadata + | + Shared model resolver/factory + | + Named prediction parameters + | + Shared prediction/distribution layer + / \ + Supervised objective Policy task binding + | + Imitation / IQL / PPO / DPO / GRPO + +One run coordinator owns phase transitions, artifacts, and recovery. +Each runner owns its algorithm's batches, collection, and update rules. +``` + +### 3.1 Model execution stays independent of the learner + +Persist the forward contract: named inputs, execution schema, normalization and codecs, attention policy, context geometry, decoder branches, target order, output positions, target transforms, parameter dtypes, and parameter sharing. A phase chooses an objective; it does not implicitly replace those forward semantics. + +Derive legacy forward behavior from existing objective metadata during artifact migration. In particular, do not silently turn every imported model into the current `training_objective="policy"` execution configuration. A bidirectional encoder over an entirely observed history is usable as an actor. Reconstructing old actions with later observations in that encoder would leak information; evaluate each historical decision using its actual observation boundary. BERT-specific masking and next-occurrence losses remain supervised data/objective behavior, not mandatory policy behavior. + +Keep `ComposableTransformerNetwork`, `ModelOutput`, and the existing named tensor boundary. Preserve the public `logits` field for compatibility even though its real-valued entries are prediction parameters. Add optional distribution metadata and parameter access without an immediate naming migration. + +### 3.2 Prediction distributions are reusable model capabilities + +Resolve a prediction specification for each target from existing interface metadata plus optional distribution settings. Types and categorical maps remain authoritative in the existing model metadata; policy configuration references targets and adds execution semantics rather than redefining them. + +For ordinary categorical heads, use categorical logits over persisted decoder IDs. For real heads, the existing scalar output is the location of a Gaussian by default. Supply an explicit positive exploration scale when attaching a distribution to a legacy point predictor; there is no universally appropriate scale across physical units. Initially support fixed per-target scales. Add learned global or state-dependent scales as optional model modules, with positive floors and stable parameterization. They must be owned and serialized by the network, not hidden inside an optimizer or inference wrapper. + +Preserve MSE, MAE, and other existing supervised losses. Adding a policy distribution must not reinterpret all historical supervised training as likelihood training. Gaussian NLL and other distribution-aware supervised losses become additional objectives; imitation is a weighted joint NLL objective using the same prediction contract. Class weights and supervised loss weights are loss reductions, not action-probability definitions. PPO ratios and DPO sequence scores use the true joint likelihood rather than a weighted average of head scores. + +Real domains are explicit: + +- Unbounded real: Gaussian with identity physical transform. +- Positive or finitely bounded real: a declared invertible transform or a support-aware family, with stable inverse and log-Jacobian calculations. Never clip a Gaussian sample and score it as an unclipped Gaussian. +- Exact endpoint actions or deterministic conditional fields: a specified atom/mixture or conditional point mass. An open-interval squashing transform cannot represent exact endpoints. Do not silently move recorded actions off a boundary. + +Freeze model-to-physical transforms across collection, replay, actor, and reference. Persist units and whether a scale is specified in model or physical coordinates. Newly imposed bounds may be incompatible with the pretrained point function; disclose this as adaptation rather than claiming exact preservation. + +### 3.3 Policy binding describes a task, not another model + +Introduce a `PolicyBinding` associated with a named model interface. It specifies: + +- Which targets and output positions form one environment decision. +- Input producers: observed fields, static values, named structured/media payloads, explicitly lagged action fields, and optional desired returns. +- Categorical executable support, real domains, and physical transforms. +- Joint factorization, target order, conditional activity, and legality rules. +- How predictions or sampled actions are fed back into future observations. +- How a decision or explicitly executed action block maps to environment time and rewards. + +Default transfer behavior preserves the source input schema. Previous actions and desired returns are optional bindings; do not append them automatically. If a source already contains the prior target as an input, bind that existing input with the correct lag and codec. For newly authored policies, the current observation/previous-action convention remains a convenient explicit preset. + +Retain all model interfaces. Select a policy interface per phase and retain inactive interfaces and tied parameters in the artifact. Optimizers follow explicit trainable/frozen parameter selection. Updating a shared backbone can change predictions through other interfaces; freezing only their decoders does not preserve their outputs. Offer supervised replay or regularization when retaining those behaviors matters. + +### 3.4 Typed actions and joint probability + +Represent actions internally as named tensors with dtypes, event shape, and explicit validity/activity masks. Preserve field order in the resolved schema; never depend on unordered dictionary iteration for likelihood factorization. Support arbitrary numbers of the real and categorical scalar heads currently available to supervised learning. Preserve event axes in the contract so adding true vector heads later does not require another buffer redesign. + +Provide a distribution execution interface along these lines: + +```python +sample(inputs, generator) -> ActionSample +deterministic(inputs) -> ActionSample +log_prob(inputs, recorded_action, context) -> DecisionScores +entropy(inputs, estimator_config) -> EntropyEstimate +kl(inputs, reference, estimator_config) -> KLEstimate +``` + +`ActionSample` carries model-space values, executed values, activity/support context, joint log probability, and optional latent values for numerically stable transformed-density replay. `DecisionScores` exposes one log probability per decision and optional per-field diagnostics. Keep the batch/decision axes; reduce action event axes exactly once. Use explicit capabilities for exact versus estimated entropy and KL. + +For independent branches, joint log probability is the sum over active fields. For an autoregressive branch, it is the sum of conditional log probabilities given the actual earlier sampled or recorded fields. Composite decoders retain their existing branch factorization. Do not flatten their architecture into one independent output layer. + +### 3.5 Autoregressive and hashed targets are required for parity + +Extend the existing autoregressive decoder with distribution-driven stochastic generation. Feed each sampled real or categorical value through the same value embedding used for teacher forcing. Scoring a recorded tuple must teacher-force its true prefix. Using logits conditioned on greedy prefixes produces the wrong joint likelihood. Share transformer, projection, embeddings, tied weights, and target ordering with supervised execution. + +When a selected action field depends on a non-action predecessor in an autoregressive branch, require a valid observed predecessor binding, or include the predecessor in the sampled probabilistic decision. Do not omit its likelihood while pretending a marginal probability has been computed. A general marginal over unobserved predecessors is additional inference work. + +For hashed categorical targets, reuse the existing collision-checked codebook and candidate scores. Normalize over the legal canonical action candidates with `logsumexp`. The product of hash-component probabilities is generally not already normalized over that candidate subset. Sample and score the same canonical candidates; do not sample arbitrary hash tuples and resolve them to a different action. Exact chunked normalization can control memory without changing semantics; approximate retrieval is a separate, explicitly limited mode. Keep supervised component losses intact. + +For dependent distributions, joint entropy requires an expectation over prefixes. Summing conditional entropies at an arbitrary recorded prefix is not generally joint entropy. Specify and validate any sampling-based estimator and its gradient, including real sampled prefixes, before enabling an entropy bonus. Preserve the distinction between a sampled reference penalty and exact KL; do not relabel the existing GRPO penalty as universally exact. + +## 4. Supervised artifact ingestion + +Add a policy attachment path to the common artifact loader: + +1. Restore the complete source network and resolved execution contract using the ordinary artifact machinery, without requiring `interface.policy`. +2. Select the source interface and resolve explicit action/input bindings against it. The artifact is authoritative for architecture, codecs, target ordering, preprocessing, and existing weights. If a run also supplies these declarations, validate agreement instead of building an unrelated new model. +3. Attach distributions to existing target outputs. Reuse categorical heads and real location heads. Initialize only declared additions, such as scale parameters, with reproducible seeds. +4. Load all original model parameters and buffers strictly. Permit only an enumerated set of newly introduced keys; do not use broad `strict=False` loading to hide mismatches. Verify tied/shared parameter identity as well as tensor values. +5. Emit a transfer report listing reused and new parameters, bindings, transforms, support restrictions, precision conversions, and any changed execution semantics. Compare source and attached-model prediction parameters on fixed inputs before training. +6. Export a normal model artifact containing the optional binding and distribution contract. Ordinary forecast inference and policy inference select their respective views of the same network. + +Distinguish two transfer modes: + +| Mode | Guarantee | Examples | +| --- | --- | --- | +| Attach | Original prediction function is retained, within declared numerical tolerance. Stochastic sampling is newly defined where necessary. | Existing real output becomes a Gaussian location with fixed scale; existing logits parameterize a categorical policy. | +| Adapt | Compatible weights are retained, but function equivalence is not promised. Every changed module or semantic choice is reported. | Adding action-history inputs, replacing an unusable input, changing attention visibility, restricting executable categories, imposing incompatible bounds, or changing context geometry. | + +Exact categorical transfer requires the same executable decoder support. Filtering special tokens changes the distribution even if logits are unchanged. For unbounded Gaussian heads with the original physical transform, the deterministic location reproduces the old point prediction. For an autoregressive policy, deterministic sequential predictions reproduce the old greedy path when support and transforms are unchanged; this does not imply that the greedy tuple is the global joint mode. + +Preserve source normalization by default. Never refit it merely because new policy episodes were supplied. Real action normalization must be reversible and shared by likelihood evaluation, critic action encoding, execution, and history. Sampling and scoring must remain consistent when dtypes are converted; replace the present unconditional FP32-weights rejection with an explicit precision policy while retaining stable probability arithmetic and deterministic compatibility checks. + +Initialization is distinct from exact resume. Fresh policy phases create new optimizer, critic, reference, sampler, and collector state according to the phase plan. Exact resume requires the complete run checkpoint and matching task/data/environment/distribution identities. Existing extraction of model state from supported run checkpoints should feed the same initialization path. + +## 5. Input, data, and time semantics + +Build a shared input materializer from the supervised preprocessing, execution schema, and window resolver. It accepts named raw values or explicitly model-encoded tensors and applies each transform exactly once. Policy and supervised source adapters call it instead of maintaining separate normalization or scalar-only encoding rules. + +Generalize history and episode representations to carry named previous-action fields and the full input payload required by the selected interface. Reuse existing depth/image/static tensor and mask contracts. Extend environment observations, snapshots, datasets, collation, memory accounting, and archival records accordingly. A symbolic media reference is sufficient only if replay can recover the identical content and transforms; persist payload identities or immutable materializations. Do not push images or depth arrays through `DomainValue` scalar validators. + +Keep source adapters distinct for supervised windows, episodes/transitions, preferences, and online collection. Share codecs, feature materialization, masks, and batching helpers. Rewards and terminal semantics cannot be inferred from arbitrary supervised windows. Require an explicit source mapping for them and preserve episode-level train/validation separation. + +Separate three masks: temporal padding, observed training labels, and semantically active action fields. Missing labels are not inactive actions. A missing predecessor in an autoregressive tuple cannot simply be masked out while treating subsequent terms as a complete joint likelihood. For likelihood-based phases, require complete relevant actions unless a supported marginalization method is explicitly selected. + +Remove the assumption that every policy decision is the last position of a `support=1` decoder. Preserve the source decoder's receptive field, padding requirements, and output-position semantics. The default binding selects the declared current decision position and leaves other outputs available for supervised losses. Existing `prediction_length` identifies output positions; it must not automatically become a future action chunk. + +Action blocks are an explicit extension: define each slot's execution time, whether intermediate observations are consumed, joint likelihood, interruption semantics, and reward/bootstrap aggregation before supporting them. For IQL, a multi-step action block needs matching offline transitions and duration-aware discounting. Selecting one current action from a model that emits several positions does not require block execution. + +The same availability check applies to bidirectional models and overlapping offline windows: each scored decision sees only what the policy would actually receive then, including any supplied masked-query token, static payload, or return-conditioning value. Return-conditioned actors remain loadable, but non-imitation phases need an explicit rule supplying and updating that input; do not silently remove the trained return feature. + +## 6. Algorithm changes + +| Algorithm | Required work | Acceptance condition | +| --- | --- | --- | +| Imitation | Use shared joint NLL over typed actions, with explicit decision reductions and activity masks. Reuse the objective in supervised training when likelihood training is requested. | Independent mixed-head NLL matches a manual sum; autoregressive NLL matches recorded-prefix scoring. Existing MSE/CE training remains valid. | +| PPO | Replace direct `CategoricalActionDistribution` construction in collection/update with distribution execution. Store exact action values, old joint log probability, support/activity context, and required transform latents. Keep V state-only. | Re-scoring a frozen collected actor gives ratio 1 within tolerance for every supported factorization and domain. No sampling/scoring mismatch at bounds. | +| GRPO | Apply the same action and score contract throughout group collection, buffers, reference scoring, and replay. Preserve group-start capabilities and trajectory-versus-decision weighting. | Old and reference scores replay consistently; equal actor/reference produces zero reference penalty; mixed actions execute in the environment. | +| DPO | Sum decision joint log likelihoods on preferred/rejected spans for actor and frozen reference. Persist identical data transforms, binding, support, and conditional masks. | Actor/reference sequence log ratios match manual mixed categorical/density calculations; no category-only or nonpositive-score assumptions remain. | +| IQL | Replace the single categorical action embedding with a schema-driven action encoder: categorical embeddings, transformed real projections, and explicit activity indicators, fused with the state representation. Retain scalar twin Q(h,a), V(h), target networks, and weighted actor log likelihood. Generalize state inputs and pooling. | Every action field reaches Q with finite gradients; Bellman/expectile/actor updates work on mixed actions; actor loss uses the same likelihood as imitation. | + +Critics can continue to own independent copies of the supported input/backbone architecture initially. Choose pooling through a declared state-representation adapter rather than hard-coding the last unmasked slot for every layout. Reusing the actor encoder is a later efficiency choice with explicit gradient and optimizer ownership; it is not necessary for unification. + +Begin continuous imitation, DPO, and weighted IQL actor fitting with fixed scales. Unrestricted learned scale can distort preference or weighted-likelihood fitting, and variance can collapse or expand without improving the intended behavior. Introduce learned-scale modes with explicit bounds/regularization and diagnostic acceptance criteria. PPO/GRPO can support controlled learned exploration once probability replay is established. Reward improvement is a benchmark goal, not a universal mathematical guarantee. + +## 7. Unified configuration and lifecycle + +Introduce one canonical run model with model/artifact initialization, named typed sources, policy bindings, execution settings, recovery settings, and ordered discriminated phases. Preserve algorithm-specific settings and budgets: supervised/imitation epochs, offline optimizer updates, PPO environment steps, and GRPO groups. References, critics, and rollout state remain phase-local. + +Migrate existing supervised and policy YAML/API forms into that canonical representation with compatibility adapters. A legacy single-categorical policy expands into a one-field binding with its existing previous-action and optional return inputs. Preserve its categorical IDs, start token, transforms, phase defaults, and scoring behavior. Avoid simultaneous wholesale renaming of public configuration keys. + +The coordinator performs whole-plan validation, resolves the model once, preflights task/data/environment capabilities, executes runners, selects best/last handoff state, and owns the common checkpoint envelope. Shared services cover parameter catalogs/freezing, optimizer transactions, scheduling, precision, metrics, RNG, atomic artifacts, and callbacks. Adapt the existing supervised and policy services incrementally; do not force RL collection and multiple optimizer roles into an epoch loop. + +Use an explicit capability matrix for execution: model architecture parity must not be confused with immediate parity across DDP/FSDP, compilation, integrations, or every device/dtype combination. Preserve existing supervised execution modes while adding policy capabilities with targeted validation. Distribute offline learners before attempting distributed environment collection if distributed policy execution is in scope. + +Each phase declares trainable modules and optimizer roles, sources, evaluation and monitor direction, budget units, and handoff behavior. A supervised phase can interleave its current datasets as before. Sequential supervised → RL → supervised phases are part of this plan; arbitrary simultaneous PPO plus supervised replay requires an explicit auxiliary-objective/update policy and is a subsequent extension, not an implicit meaning of interleaving phases. + +Default to fresh optimizer state at algorithm boundaries. Any continuation must name a compatible parameter/state mapping. Fresh policy references are captured at the specified phase boundary or loaded from a compatible artifact. Best/last selection restores associated actor distribution parameters and transforms, not only backbone weights. + +The checkpoint envelope contains model and binding identity, phase index, committed optimizer clocks, per-role optimizer/scheduler/scaler state, references/targets, sampler/RNG state, source fingerprints, rollout state, environment snapshot identity, and collection generation. Exact resume is guaranteed only where the environment exposes the necessary snapshot capabilities; reset-environment recovery remains a distinct mode. Rejected/stale buffers must not be consumed after a handoff or distribution change. + +## 8. Delivery sequence and completion gates + +These are implementation slices, not duration estimates. Several are substantial changes and should be split further when the first prototype exposes integration costs. + +| Slice | Deliverable and principal locations | Completion gate | +| --- | --- | --- | +| 1. Contracts and baselines | Inventory the currently admitted model combinations; specify forward semantics, prediction distributions, action bindings, and transfer reports. Start in `config`, `model`, and `artifacts`. | Representative supervised and legacy-policy fixtures establish current behavior; every supervised feature has a parity acceptance case. | +| 2. Minimal end-to-end transfer | Shared artifact attachment and a thin policy runtime for an unchanged ordinary supervised interface, one categorical target or one unbounded real location with fixed scale. Touch `rl/checkpoint.py`, `learner.py`, `policy.py`, and distribution resolution. | A real-only supervised artifact with no policy metadata completes a policy-imitation and PPO update without replacing its input or location head. A categorical artifact does the same. | +| 3. Mixed actions throughout the pipeline | Named action tensors, independent multiple heads, reversible transforms, datasets, history, trajectory schema, collectors, buffers, inference, and scalar environment execution. Add joint NLL. | A categorical-plus-two-real actor completes imitation and PPO collection/update, persists, reloads, and reproduces collection scores. Legacy single-categorical behavior still matches. | +| 4. Existing algorithm migration | Generalize GRPO and DPO sequence scoring and IQL action encoding against the established action contract. | All five algorithms train mixed-action fixtures; no algorithm directly instantiates a categorical-only distribution. | +| 5. Decoder architecture parity | Add stochastic autoregressive generation, recorded-prefix scoring, branch composition, tied heads, hashed candidate normalization, and conditional activity. | Sampled and teacher-forced tuple scores agree, normalization checks pass, and original decoder sharing survives reload. | +| 6. Input and geometry parity | Reuse the full supervised materializer and execution schema across policy data, environment observations, history, critics, and recovery. Remove scalar-only, one-interface, and unit-support restrictions after their acceptance cases pass. | Structured/depth/image/static/composite input fixtures, hashed inputs, multiple interfaces, nontrivial support, prediction-position selection, and observed-history bidirectional models train as actors. | +| 7. Common run plan | Compatibility loaders and coordinator around existing runners; then consolidate optimizer/metrics/checkpoint services. Touch `train.py`, `training/engine.py`, `rl/learner.py`, runtime and recovery modules. | A single supervised → PPO → supervised plan works, and interrupted phase transitions resume without duplicate updates or reinitialization. | +| 8. Compatibility and release | Versioned artifacts, trajectories and environment payload capabilities; public API/templates/docs; targeted execution-mode validation. | Legacy artifacts/configurations/records load through tested adapters; unsupported recovery versions fail explicitly; all mandatory parity cases pass. | + +Dependencies: slices 2–4 establish a usable supervised-to-RL path before broad lifecycle work. Decoder and input parity both build on the shared contracts and mixed-action representation. Slice 7 can begin with the compatibility schema earlier, but it is not a prerequisite for exporting a supervised artifact and post-training it in a separate run. The release is not architecture-complete until slices 5 and 6 are finished; “multiple independent scalar heads” is an intermediate milestone. + +## 9. Validation required for completion + +Use small deterministic fixtures for probability and recovery invariants, then a few controlled learning tasks. Do not rely solely on whether training losses decrease. + +1. **Transfer invariants:** strict reuse of source parameters/buffers and sharing; fixed-input predictions match before updates; explicit scale additions; unchanged normalization and categorical maps; artifact save/reload preserves both inference views. Include a model with no policy metadata and a model extracted from a supported supervised run checkpoint. +2. **Probability correctness:** manual independent sums; tiny enumerated autoregressive categorical joint normalization; numerical continuous density/transform checks; positive log-density acceptance; Gaussian location/scale gradients; hashed candidate normalization; boundary and conditional-atom behavior; padding/activity masks and missing-label rejection. +3. **Collection/replay equivalence:** seeded sample reproducibility on supported devices; sampled-prefix versus recorded-prefix scores; PPO/GRPO initial ratio 1; reference equality; immutable support and preprocessing; action-history lag; full-precision archived action/latent round trips. +4. **Architecture matrix:** every currently supported ingestion and decoder family plus feature combinations that change execution, including static prefixes, depth/media masks, parameter tying, multiple interfaces, support > 1, and multiple prediction positions. Test full likelihood and a gradient update, not just network construction. Cover ordinary/tabular and non-scalar inputs in critic execution as well. +5. **Algorithm behavior:** mixed imitation improves held-out likelihood; a deterministic toy mixed-action control task demonstrates PPO/GRPO updates and executable actions; DPO improves preferred relative log ratios; IQL has finite meaningful Q/V and actor updates without an action-count vector assumption. Keep reward and likelihood metrics separate. +6. **Lifecycle and compatibility:** supervised → RL → supervised handoff; stop/resume at committed updates and phase boundaries; best/last selection including scales; stale-buffer rejection; source/environment identity changes; legacy categorical config/artifact/trajectory adapters; explicit limits on exact recovery of old online checkpoints. + +Baseline performance checks should measure autoregressive sampling cost, hashed candidate scoring cost, and buffer bytes for structured inputs. Correctness is mandatory; use those results to prioritize caching or packing without changing the probability contract. + +## 10. Recommended scope decisions + +Proceed with these defaults unless a task explicitly requires something else: + +- Preserve pretrained interfaces and preprocessing on attachment; do not inject new history features by default. +- Use existing real heads as locations with explicitly configured fixed scales first. Add learnable scales without changing existing point-prediction APIs. +- Deliver independent mixed heads as the first broad action-space milestone, followed by the existing autoregressive and composite architecture surface before claiming parity. +- Support current-decision output selection first while preserving larger decoder support/prediction geometry. Add true action-block execution under a separate temporal contract. +- Represent conditional activity and deterministic values in the action contract now; implement their explicit probability semantics before advertising dependent constrained actions. General user-defined constraint solvers and arbitrary rejection/projection sampling are deferred. +- Keep algorithms and source adapters distinct. Consolidate their shared execution and lifecycle services around the contracts they actually share. +- Require a usable task binding and rewards/preferences/interaction source. Artifact ingestion makes a model trainable; it cannot manufacture the learning task. + +The first implementation should prove an unchanged supervised real-valued actor can be attached, sampled, scored, and updated by PPO. That exercises both overarching requirements immediately and prevents the work from stopping at a more generic policy schema that still cannot ingest an ordinary supervised checkpoint. diff --git a/release-notes/unreleased-supervised-policy-unification.md b/release-notes/unreleased-supervised-policy-unification.md new file mode 100644 index 00000000..4b46a7f7 --- /dev/null +++ b/release-notes/unreleased-supervised-policy-unification.md @@ -0,0 +1,23 @@ +# Supervised and policy model unification + +- Attach policy tasks directly to ordinary supervised model artifacts and + supported supervised run checkpoints, preserving prediction heads and inputs. +- Train named combinations of categorical and real action heads with imitation, + IQL, PPO, DPO, and GRPO. Retain autoregressive/composite decoders and hashed + categorical candidate semantics. +- Add bounded real distributions, optional point masses, conditional activity, + bounded global exploration scales, and transform-latent replay. +- Preserve full execution-schema payloads, including static, depth, image, and + mask inputs, throughout policy history and storage. +- Add mixed supervised/policy run plans with native phase recovery, atomic + handoffs, model identity checks, and source/environment preflight. +- Separate persisted forward attention from optimization objectives and allow + explicit prediction or policy inference views. +- Add shared real prediction distributions and supervised Gaussian NLL support. +- Write model artifact version 3 and typed policy trajectory/collection version 2; + retain readers for earlier model artifacts and single-categorical interfaces. + +Tests and training experiments are intentionally deferred. See +[the usage guide](../documentation/supervised-policy-learning.md) and +[the implementation plan](../documentation/supervised-policy-unification-plan.md) +for the acceptance criteria that remain to be verified. diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index e0d01349..fbbf70b7 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -18,7 +18,7 @@ from sequifier.io.atomic import atomic_save from sequifier.model.factory import build_transformer_network -MODEL_ARTIFACT_FORMAT_VERSION = 2 +MODEL_ARTIFACT_FORMAT_VERSION = 3 @dataclass(frozen=True) @@ -46,7 +46,7 @@ class ModelArtifact: metadata: ModelArtifactMetadata def validate(self) -> None: - if self.format_version not in (1, MODEL_ARTIFACT_FORMAT_VERSION): + if self.format_version not in (1, 2, MODEL_ARTIFACT_FORMAT_VERSION): raise ValueError( f"Unsupported model artifact format {self.format_version}; " f"expected {MODEL_ARTIFACT_FORMAT_VERSION}." diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index dac8ddf8..4d26c639 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -18,6 +18,62 @@ from sequifier.typechecking import beartype +def validate_execution_attachment(source: dict, current: dict) -> None: + """Require existing prediction semantics when reusing complete model weights.""" + from pydantic_core import to_jsonable_python + + from sequifier.artifacts.model_export import model_execution_config + + source_config, _ = resolved_config_from_model_config(source, device="cpu") + current_config, _ = resolved_config_from_model_config(current, device="cpu") + source = model_execution_config(source_config) + current = model_execution_config(current_config) + if source["backbone"]["architecture"] != current["backbone"]["architecture"]: + raise ValueError("model attachment cannot change backbone architecture") + if source.get("layer_type_dtypes") != current.get("layer_type_dtypes"): + raise ValueError("model attachment cannot silently change parameter precision") + if source["context_length"] != current["context_length"]: + raise ValueError("model attachment cannot change context length") + + def attention(values): + return values.get("forward_attention") or ( + "bidirectional" if values["training_objective"] == "bert" else "causal" + ) + + if attention(source) != attention(current): + raise ValueError("model attachment cannot change forward attention") + for name, interface in source["interfaces"].items(): + if name not in current["interfaces"]: + raise ValueError(f"initial model interface {name!r} was dropped") + for key in ( + "input_columns", + "target_columns", + "ingestion", + "decoder", + "id_maps", + "target_decoder_ids", + "target_global_to_decoder", + "selected_columns_statistics", + "normalize_real_columns", + "depth_layouts", + "static_conditioning", + "feature_layout", + ): + default = ( + {} + if key == "depth_layouts" + else True + if key == "normalize_real_columns" + else None + ) + if to_jsonable_python(interface.get(key, default)) != to_jsonable_python( + current["interfaces"][name].get(key, default) + ): + raise ValueError( + f"initial model execution differs at interfaces.{name}.{key}" + ) + + @beartype def resolved_config_from_model_config( values: dict, @@ -47,6 +103,7 @@ def resolved_config_from_model_config( else "parquet" ), training_objective=objective, + forward_attention=values.get("forward_attention"), context_length=context_length, target_offset=target_offset, inference_batch_size=1, diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index 0a854f56..e9f45577 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -38,6 +38,8 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: spec = training_config.global_training return { "training_objective": spec.training_objective, + "forward_attention": spec.forward_attention + or ("bidirectional" if spec.training_objective == "bert" else "causal"), "context_length": spec.context_length, "target_offset": spec.target_offset, "next_occurrence_config": ( diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index d5c0c219..6eeb4086 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -1,7 +1,7 @@ import json import os import re -from typing import Any, Generic, Optional, TypeVar, Union +from typing import Any, Generic, Literal, Optional, TypeVar, Union import numpy as np from pydantic import ( @@ -900,6 +900,8 @@ class _InferenceConfigBase( ): """Shared fields and validation for authored and resolved inference config.""" + execution_mode: Literal["prediction", "policy"] | None = None + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") project_root: str diff --git a/src/sequifier/config/learning_run.py b/src/sequifier/config/learning_run.py new file mode 100644 index 00000000..b0c529b3 --- /dev/null +++ b/src/sequifier/config/learning_run.py @@ -0,0 +1,97 @@ +"""A common authored run plan with supervised and policy phases.""" + +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from sequifier.config.policy_config import PolicySchemaConfig +from sequifier.config.policy_plan import ( + DPOPhaseConfig, + GRPOPhaseConfig, + IQLPhaseConfig, + PolicyImitationPhaseConfig, + PolicyResumeConfig, + PPOPhaseConfig, +) +from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.config.train_config import ( + DatasetTrainingSpecModel, + EvaluationSpecModel, + GlobalTrainingSpecModel, + ModelSpecModel, + TrainingPhaseSpecModel, +) + + +class SupervisedLearningPhase(TrainingPhaseSpecModel): + handoff: Literal["best", "last"] = "best" + + +LearningPhase = Annotated[ + SupervisedLearningPhase + | PolicyImitationPhaseConfig + | IQLPhaseConfig + | PPOPhaseConfig + | DPOPhaseConfig + | GRPOPhaseConfig, + Field(discriminator="algorithm"), +] + + +class LearningRunConfig(BaseModel): + """One model with typed sources and independently budgeted phase runners.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + run_type: Literal["unified"] = "unified" + project_root: str = "." + model_name: str = Field(min_length=1, pattern=r"^[^/\\]+$") + device: str = "cpu" + seed: int = Field(default=1010, ge=0, lt=2**32) + model: ModelSpecModel | None = None + initialize_from: str | None = None + global_training: GlobalTrainingSpecModel | None = None + dataset_training: dict[str, DatasetTrainingSpecModel] = Field(default_factory=dict) + evaluation: EvaluationSpecModel | None = None + policy_interface: str | None = None + policy_bindings: dict[str, PolicySchemaConfig] = Field(default_factory=dict) + sources: dict[str, SequenceSourceConfig] = Field(default_factory=dict) + preferences: dict[str, str] = Field(default_factory=dict) + units: dict[str, str] = Field(default_factory=dict) + phases: list[LearningPhase] = Field(min_length=1) + resume: PolicyResumeConfig = Field(default_factory=PolicyResumeConfig) + validation_fraction: float = Field(default=0.2, gt=0, lt=1) + episode_cache_size: int = Field(default=8, gt=0) + checkpoint_path: str = "" + model_path: str = "" + report_path: str = "" + phase_directory: str = "" + export_onnx: bool = False + + @model_validator(mode="after") + def validate_plan(self): + root = Path(self.project_root).resolve() + self.project_root = str(root) + if self.model_name in {".", ".."}: + raise ValueError("model_name must name one directory") + for key, default in ( + ("checkpoint_path", f"checkpoints/{self.model_name}/learning-run.pt"), + ("model_path", f"models/{self.model_name}/model.pt"), + ("report_path", f"outputs/training/{self.model_name}/report.json"), + ("phase_directory", f"outputs/training/{self.model_name}/phases"), + ): + setattr(self, key, str((root / (getattr(self, key) or default)).resolve())) + if self.initialize_from: + self.initialize_from = str((root / self.initialize_from).resolve()) + if self.model is None and self.initialize_from is None: + raise ValueError("a run requires a model or initialize_from") + if len({phase.name for phase in self.phases}) != len(self.phases): + raise ValueError("phase names must be unique") + if any(phase.algorithm == "supervised" for phase in self.phases): + if self.global_training is None or not self.dataset_training: + raise ValueError( + "supervised phases require global_training and dataset_training" + ) + if self.phases[0].algorithm != "supervised" and self.initialize_from is None: + raise ValueError("policy-first unified runs require initialize_from") + return self diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 62187779..304b453f 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -1,6 +1,7 @@ """Policy alignment and standard model-component configuration.""" -from typing import Annotated, Literal +import math +from typing import Annotated, Any, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -10,7 +11,7 @@ class PolicyConfigBase(BaseModel): - model_config = ConfigDict(extra="forbid", frozen=True) + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) class ReturnDefinition(PolicyConfigBase): @@ -19,11 +20,71 @@ class ReturnDefinition(PolicyConfigBase): finite_horizon: int | None = Field(default=None, gt=0) +class ActionFieldConfig(PolicyConfigBase): + """Distribution and domain of one named model target.""" + + type: Literal["categorical", "real"] + values: list[DomainValue] = Field(default_factory=list) + scale: float | None = Field(default=None, gt=0, allow_inf_nan=False) + learn_scale: bool = False + min_scale: float = Field(default=1e-4, gt=0) + max_scale: float = Field(default=100.0, gt=0) + lower: float | None = Field(default=None, allow_inf_nan=False) + upper: float | None = Field(default=None, allow_inf_nan=False) + active_when: dict[str, list[DomainValue]] = Field(default_factory=dict) + inactive_value: DomainValue = 0.0 + # Fixed point masses allow exact endpoints alongside a continuous density. + atoms: dict[float, float] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_distribution(self): + if self.type == "real" and (self.scale is None or self.values): + raise ValueError("real actions require an explicit scale and no values") + if self.type == "categorical" and (self.scale is not None or self.learn_scale): + raise ValueError("categorical actions cannot configure a real scale") + if len(set(self.values)) != len(self.values): + raise ValueError("action labels must be unique") + if ( + self.lower is not None + and self.upper is not None + and self.lower >= self.upper + ): + raise ValueError("action lower bound must be below upper bound") + if self.min_scale > self.max_scale: + raise ValueError("min_scale must not exceed max_scale") + if ( + self.scale is not None + and not self.min_scale <= self.scale <= self.max_scale + ): + raise ValueError("scale must lie between min_scale and max_scale") + if self.atoms and self.type != "real": + raise ValueError("point-mass mixtures are real action distributions") + if ( + any( + not math.isfinite(value) + or not math.isfinite(probability) + or probability <= 0 + for value, probability in self.atoms.items() + ) + or sum(self.atoms.values()) >= 1 + ): + raise ValueError( + "atom probabilities must be positive and sum to less than one" + ) + if any( + (self.lower is not None and value < self.lower) + or (self.upper is not None and value > self.upper) + for value in self.atoms + ): + raise ValueError("point masses must lie within the declared action domain") + return self + + class PolicySchemaConfig(PolicyConfigBase): - alignment: Literal["current_observation_previous_action"] = ( + alignment: Literal["current_observation_previous_action", "model_inputs"] = ( "current_observation_previous_action" ) - observation_columns: list[str] = Field(min_length=1) + observation_columns: list[str] = Field(default_factory=list) static_columns: list[str] = Field(default_factory=list) categorical_columns: list[str] = Field(default_factory=list) # Numeric trajectory fields may also carry already encoded categorical IDs. @@ -40,31 +101,48 @@ class PolicySchemaConfig(PolicyConfigBase): action_values: list[DomainValue] = Field( default_factory=lambda: [0.0, 1.0], min_length=2 ) + actions: dict[str, ActionFieldConfig] = Field(default_factory=dict) + action_order: list[str] = Field(default_factory=list) + decision_position: int = Field(default=-1, le=-1) + # Existing input -> lagged action; no implicit features in model_inputs mode. + previous_action_inputs: dict[str, str] = Field(default_factory=dict) + initial_inputs: dict[str, Any] = Field(default_factory=dict) + input_schema: dict[str, Any] | None = None + input_statistics: dict[str, dict[str, float]] = Field(default_factory=dict) + input_encoding: Literal["raw", "model"] = "raw" return_conditioned: bool = True + initial_desired_return: float | None = Field(default=None, allow_inf_nan=False) returns: ReturnDefinition = Field(default_factory=ReturnDefinition) @model_validator(mode="after") def validate_names(self): + if self.action_order and ( + set(self.action_order) != set(self.actions) + or len(self.action_order) != len(self.actions) + ): + raise ValueError("action_order must name every action field exactly once") names = self.observation_columns + self.static_columns if any(not name for name in names) or len(names) != len(set(names)): raise ValueError( "policy observation/static columns must be nonempty and disjoint" ) reserved = {"__policy_previous_action", "__policy_return", "__policy_action"} - if reserved.intersection(names + [self.action_column]): + if self.alignment != "model_inputs" and reserved.intersection( + names + [self.action_column] + ): raise ValueError("policy feature names use reserved internal columns") for counts, columns in ( (self.observation_n_classes, self.observation_columns), (self.static_n_classes, self.static_columns), ): - if counts.keys() - set(columns): + if counts.keys() - set(columns) - set(self.previous_action_inputs): raise ValueError( "class counts must name declared observation/static columns" ) counts = {**self.observation_n_classes, **self.static_n_classes} if len(set(self.categorical_columns)) != len(self.categorical_columns) or set( self.categorical_columns - ) - set(names): + ) - set(names) - set(self.previous_action_inputs): raise ValueError( "categorical_columns must name unique observation/static features" ) @@ -87,6 +165,18 @@ def validate_names(self): ) return self + @property + def action_columns(self) -> list[str]: + return ( + (self.action_order or list(self.actions)) + if self.actions + else [self.action_column] + ) + + @property + def is_joint(self) -> bool: + return bool(self.actions) + @property def categorical_n_classes(self) -> dict[str, int]: return {**self.observation_n_classes, **self.static_n_classes} @@ -97,3 +187,8 @@ class CategoricalPolicyConfig(PolicySchemaConfig): context_length: int = Field(gt=0) interface_name: NonemptyStr + + +# General names for the shared contract; retain the original public names. +PolicyBinding = PolicySchemaConfig +PolicyConfig = CategoricalPolicyConfig diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index fde452be..1a268cdf 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -22,6 +22,7 @@ class PolicyInferenceConfig(PolicyConfigBase): project_root: str = "." model_path: str + execution_mode: Literal["prediction", "policy"] | None = None model_type: Literal["policy", "generative"] = "generative" model_interface: str | None = None data_path: str @@ -36,6 +37,7 @@ class PolicyInferenceConfig(PolicyConfigBase): desired_return: float | None = Field(default=None, allow_inf_nan=False) desired_return_column: str = "desired_return" previous_action_column: str | None = None + previous_action_columns: dict[str, str] = Field(default_factory=dict) sequence_id_column: str = "sequenceId" item_position_column: str = "itemPosition" rollout: PolicyRolloutConfig | None = None @@ -62,6 +64,8 @@ def policy_inference_config( values: dict, artifacts: dict[str, ModelArtifact] | None = None ) -> PolicyInferenceConfig | None: """Dispatch from the persisted interface contract, without training YAML.""" + if values.get("execution_mode") == "prediction": + return None artifacts = {} if artifacts is None else artifacts model_path = values.get("model_path") paths = [model_path] if isinstance(model_path, str) else model_path or [] diff --git a/src/sequifier/config/policy_optimization.py b/src/sequifier/config/policy_optimization.py index 184c6ff5..6a048004 100644 --- a/src/sequifier/config/policy_optimization.py +++ b/src/sequifier/config/policy_optimization.py @@ -3,6 +3,10 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from sequifier.config.components import ComponentSpec +from sequifier.config.optimizer_config import ( + OptimizerPlan, + resolve_plan_scheduler_arguments, +) from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class @@ -18,20 +22,26 @@ class OptimizationConfig(BaseModel): """One authored optimizer/scheduler configuration per trainable role.""" model_config = ConfigDict(extra="forbid", allow_inf_nan=False) - optimizer: ComponentSpec = Field(default_factory=default_optimizer) + optimizer: OptimizerPlan | ComponentSpec = Field( + default_factory=default_optimizer, union_mode="left_to_right" + ) scheduler: ComponentSpec = Field(default_factory=default_scheduler) learning_rate: float = Field(default=3e-4, gt=0) gradient_clip: float = Field(default=1.0, gt=0) @model_validator(mode="after") def validate_components(self): - if self.optimizer.name not in {"Adam", "AdamW", "SGD", "RMSprop"} or { - "lr", - "params", - }.intersection(self.optimizer.arguments): - raise ValueError( - "policy optimizers require a supported optimizer and phase learning rate" - ) + specs = ( + [group.optimizer for group in self.optimizer.groups] + if isinstance(self.optimizer, OptimizerPlan) + else [self.optimizer] + ) + for spec in specs: + get_optimizer_class(spec.name) + if {"lr", "params"}.intersection(spec.arguments): + raise ValueError( + "optimizer ownership and learning rates must be configured explicitly" + ) if ( self.scheduler.name in {"ReduceLROnPlateau", "SequentialLR", "ChainedScheduler"} @@ -53,11 +63,22 @@ def validate_budget(self, step_budget): ) def build(self, module): - optimizer = get_optimizer_class(self.optimizer.name)( + from sequifier.model.parameter_catalog import ParameterCatalog + from sequifier.training.optimization import create_optimizer + + optimizer = create_optimizer( + self.optimizer, [p for p in module.parameters() if p.requires_grad], - lr=self.learning_rate, - **self.optimizer.arguments, + self.learning_rate, + parameter_catalog=ParameterCatalog(module) + if isinstance(self.optimizer, OptimizerPlan) + else None, + ) + arguments = ( + resolve_plan_scheduler_arguments(self.optimizer, self.scheduler) + if isinstance(self.optimizer, OptimizerPlan) + else self.scheduler.arguments ) return optimizer, get_scheduler_class(self.scheduler.name)( - optimizer, **self.scheduler.arguments + optimizer, **arguments ) diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 24e099c7..2ba2672f 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -14,6 +14,8 @@ class PhaseConfig(BaseModel): model_config = ConfigDict(extra="forbid", allow_inf_nan=False) name: str handoff: Literal["best", "last"] = "best" + model_interface: str | None = None + freeze: list[str] = Field(default_factory=list) representation: CategoricalPolicyConfig | None = None optimization_roles: ClassVar[tuple[str, ...]] = ("actor",) optimization: dict[str, OptimizationConfig] = Field(default_factory=dict) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 07c5fd5e..8648d41d 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -16,6 +16,8 @@ class PolicyTableColumns(PolicyConfigBase): observations: dict[str, NonemptyStr] = Field(default_factory=dict) static_inputs: dict[str, NonemptyStr] = Field(default_factory=dict) action: NonemptyStr | None = None + actions: dict[str, NonemptyStr] = Field(default_factory=dict) + previous_actions: dict[str, NonemptyStr] = Field(default_factory=dict) reward: NonemptyStr | None = "reward" terminated: NonemptyStr | None = "terminated" truncated: NonemptyStr | None = "truncated" @@ -27,6 +29,7 @@ class PolicyTableColumns(PolicyConfigBase): class SequenceSourceConfig(PolicyConfigBase): + project_root: str = "." units: dict[str, str] | None = None paths: list[NonemptyStr] = Field(default_factory=list) manifest: NonemptyStr | None = None @@ -63,6 +66,7 @@ def resolve_sequence_source( source: SequenceSourceConfig, root: Path ) -> SequenceSourceConfig: values = source.model_dump() + values["project_root"] = str(root) values["paths"] = [str((root / path).resolve()) for path in source.paths] for key in ("manifest", "metadata_config_path", "static_data_path"): if values[key] is not None: diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 7b088e1e..dc914a2a 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -60,6 +60,7 @@ resolve_plan_scheduler_arguments, ) from sequifier.config.policy_config import ( + ActionFieldConfig, CategoricalPolicyConfig, NonemptyStr, PolicySchemaConfig, @@ -364,6 +365,7 @@ class GlobalTrainingSpecModel(BaseModel): read_format: Literal["csv", "parquet", "pt"] = "parquet" training_objective: str + forward_attention: Literal["causal", "bidirectional"] | None = None context_length: int = Field(gt=0) target_offset: int = Field(default=1, ge=0) window_stride: Optional[int] = Field(default=None, gt=0) @@ -606,6 +608,7 @@ class ModelInterfaceSpecModel(BaseModel): input_columns: list[str] = Field(default_factory=list) policy: PolicySchemaConfig | None = None + prediction_distributions: dict[str, ActionFieldConfig] = Field(default_factory=dict) static_conditioning: StaticConditioningConfig | None = None preconditioning_image_encoder: ImageEncoderConfig | None = None target_columns: list[str] = Field(..., min_length=1) @@ -746,9 +749,11 @@ class PolicyRunConfig(BaseModel): device: str = "cpu" seed: int = Field(default=1010, ge=0, lt=2**32) enforce_determinism: bool = True - model: ModelSpecModel - policy_interface: str - context_length: int = Field(gt=0) + model: ModelSpecModel | None = None + policy_binding: PolicySchemaConfig | None = None + execution_model: dict[str, Any] | None = None + policy_interface: str | None = None + context_length: int | None = Field(default=None, gt=0) units: dict[str, str] = Field(default_factory=dict) initialize_from: str | None = None normalization: FrozenNormalizationConfig | None = None @@ -788,25 +793,118 @@ def resolve(self): name: str((root / path).resolve()) for name, path in self.preferences.items() } - if set(self.model.interfaces) != {self.policy_interface}: + if self.initialize_from and self.execution_model is None: + from sequifier.artifacts.model_artifact import read_execution_artifact + + if Path(self.initialize_from).is_file(): + self.execution_model = read_execution_artifact( + self.initialize_from + ).model_config.values + elif self.resume.policy != "never" and Path(self.checkpoint_path).is_file(): + import json + + saved = torch.load( + self.checkpoint_path, map_location="cpu", weights_only=False + ) + representation = json.loads(saved["representation"]) + self.execution_model = representation.get("execution") + else: + raise FileNotFoundError(self.initialize_from) + if self.execution_model is not None: + from sequifier.artifacts.model_config import ( + resolved_config_from_model_config, + ) + from sequifier.rl.attachment import apply_binding, resolve_binding + + execution, selected = resolved_config_from_model_config( + self.execution_model, + device=self.device, + interface_name=self.policy_interface, + ) + self.policy_interface = selected + if ( + self.context_length is not None + and self.context_length != execution.global_training.context_length + ): + raise ValueError("attachment preserves the source context length") + self.context_length = execution.global_training.context_length + source_policy = execution.interface(selected).policy + binding = self.policy_binding or source_policy + if binding is None: + raise ValueError( + "supervised initialization requires policy_binding with named actions" + ) + if binding.is_joint: + if self.model is not None: + if ( + self.model.backbone.architecture + != execution.model.backbone.architecture + ): + raise ValueError( + "declared backbone differs from the initialization artifact" + ) + for name, declared in self.model.interfaces.items(): + actual = execution.model.interfaces.get(name) + if actual is None or any( + getattr(declared, key) != getattr(actual, key) + for key in ( + "input_columns", + "target_columns", + "ingestion", + "decoder", + ) + ): + raise ValueError( + f"declared interface {name!r} differs from the initialization artifact" + ) + fitted = resolve_binding(execution, selected, binding) + apply_binding(execution, selected, fitted) + self.model = execution.model + elif self.model is None: + model = execution.model.model_copy(deep=True) + model.interfaces[selected].input_columns = ( + binding.observation_columns + binding.static_columns + ) + model.interfaces[selected].target_columns = [binding.action_column] + self.model = model + if ( + self.model is None + or self.policy_interface is None + or self.context_length is None + ): + raise ValueError( + "model, policy_interface and context_length are required without an artifact" + ) + joint = self.policy.is_joint + if not joint and set(self.model.interfaces) != {self.policy_interface}: raise ValueError( "policy runs require exactly the selected policy_interface" ) spec = self.policy - if set(self.units) - set(spec.observation_columns + spec.static_columns): - raise ValueError("units must name policy observation/static fields") + if set(self.units) - set( + spec.observation_columns + spec.static_columns + spec.action_columns + ): + raise ValueError( + "units must name policy observation, static or action fields" + ) interface = self.model.interfaces[self.policy_interface] - if interface.input_columns != spec.observation_columns + spec.static_columns: + if ( + not joint + and interface.input_columns + != spec.observation_columns + spec.static_columns + ): raise ValueError( "model.interfaces inputs must match policy observations and static fields in order" ) - if interface.target_columns != [spec.action_column]: + if not joint and interface.target_columns != [spec.action_column]: raise ValueError("model.interfaces targets must contain the policy action") - if interface.decoder.support != 1 or interface.decoder.prediction_length != 1: + if not joint and ( + interface.decoder.support != 1 or interface.decoder.prediction_length != 1 + ): raise ValueError( "policy decoder requires support=1 and prediction_length=1" ) - if any( + if not joint and any( ( interface.categorical_hashing, interface.static_conditioning, @@ -817,7 +915,7 @@ def resolve(self): raise ValueError( "policy runs support scalar observation/static fields only" ) - if self.model.backbone.repository is not None: + if not joint and self.model.backbone.repository is not None: raise ValueError("policy initialization uses initialize_from") if self.context_length > self.model.backbone.architecture.max_context_length: raise ValueError( @@ -832,10 +930,25 @@ def resolve(self): if len(names) != len(set(names)): raise ValueError("policy phase names must be unique") for phase in self.phases: - if phase.algorithm != "policy_imitation" and spec.return_conditioned: + if ( + phase.model_interface is not None + and phase.model_interface != self.policy_interface + ): raise ValueError( - f"phases.{phase.name}: {phase.algorithm} requires an unconditioned actor" + "use a unified plan to switch policy interfaces between phases" ) + if phase.algorithm != "policy_imitation" and spec.return_conditioned: + if not joint: + raise ValueError( + f"phases.{phase.name}: legacy conditioned policies support imitation only" + ) + if ( + isinstance(phase, (PPOPhaseConfig, GRPOPhaseConfig)) + and spec.initial_desired_return is None + ): + raise ValueError( + "online return conditioning requires initial_desired_return" + ) refs = getattr(phase, "sources", []) + ( getattr(phase, "validation_sources", None) or [] ) @@ -869,9 +982,29 @@ def resolve(self): @property def policy(self) -> CategoricalPolicyConfig: + if ( + self.model is None + or self.policy_interface is None + or self.context_length is None + ): + raise ValueError( + "policy model, interface and context must be resolved first" + ) schema = self.model.interfaces[self.policy_interface].policy if schema is None: raise ValueError("policy_interface requires a categorical policy schema") + if schema.is_joint and self.execution_model is not None: + from sequifier.artifacts.model_config import ( + resolved_config_from_model_config, + ) + from sequifier.rl.attachment import resolve_binding + + execution, _ = resolved_config_from_model_config( + self.execution_model, + device=self.device, + interface_name=self.policy_interface, + ) + return resolve_binding(execution, self.policy_interface, schema) return CategoricalPolicyConfig( **schema.model_dump(), interface_name=self.policy_interface, @@ -1126,9 +1259,7 @@ def validate_model_name(cls, value): @model_validator(mode="after") @beartype def validate_relationships(self): - if self.global_training.training_objective == "policy" or any( - interface.policy is not None for interface in self.model.interfaces.values() - ): + if self.global_training.training_objective == "policy": raise ValueError( "policy training requires PolicyRunConfig; supervised and policy phases cannot be mixed" ) @@ -1317,6 +1448,7 @@ class ResolvedDatasetPart(BaseModel): class ResolvedModelInterface(BaseModel): policy: CategoricalPolicyConfig | None = None + prediction_distributions: dict[str, ActionFieldConfig] = Field(default_factory=dict) model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") depth_layouts: DepthLayoutRegistryModel = Field( @@ -1525,8 +1657,8 @@ def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: @dataclass(frozen=True) class LoadedTrainConfig: - config: SequifierConfig | PolicyRunConfig - resolved: ResolvedSequifierConfig | PolicyRunConfig + config: Any + resolved: Any metadata: dict[str, DatasetMetadata] @@ -1862,6 +1994,7 @@ def _resolve_interface( else None ), input_columns=spec.input_columns, + prediction_distributions=spec.prediction_distributions, static_conditioning=spec.static_conditioning, static_features_path=metadata.static_features_path, image_sources={ @@ -2409,6 +2542,18 @@ def load_train_config_with_source( if args: raise ValueError(f"Unsupported training CLI overrides: {sorted(args)}") + if "phases" in raw and ( + raw.get("run_type") == "unified" + or "policy_bindings" in raw + or "dataset_training" in raw + or "global_training" in raw + ): + from sequifier.config.learning_run import LearningRunConfig + + if sensitive or metadata_inline_values or skip_metadata: + raise ValueError("mixed learning plans require explicit source metadata") + authored = LearningRunConfig.model_validate(raw) + return LoadedTrainConfig(config=authored, resolved=authored, metadata={}) if "phases" in raw: if ( "training_plan" in raw @@ -2491,7 +2636,7 @@ def load_train_config_with_source( @beartype def load_train_config( config_path: str, args_config: dict[str, Any], skip_metadata: bool -) -> ResolvedSequifierConfig | PolicyRunConfig: +) -> Any: return load_train_config_with_source( config_path, args_config, skip_metadata ).resolved diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 4e28b426..2b62309e 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -1732,9 +1732,11 @@ PPO environment steps and GRPO groups are explicit per-phase budgets. Each phase selects `handoff: best | last` (default `best`). One frozen actor representation and causal history contract spans the run; all optimizer/critic state starts fresh at each handoff. Return conditioning supports imitation only. -Policy runs are local FP32 runs and cannot contain supervised forecasting phases. -The supervised `training_plan`/`dataset_training`/`global_training` schema above -remains independent. +Standalone legacy policy runs use their existing phase schema. Use +`run_type: unified` for supervised and policy phases in one plan, or +`initialize_from` plus `policy_binding.actions` to post-train an ordinary +supervised artifact. See the [unified learning guide](../supervised-policy-learning.md) +for typed actions, source bindings, prediction distributions, and recovery. See the [policy learning guide](../policy-learning.md) for full fields, objective formulas, environment capabilities, preference manifests, checkpoint boundaries, @@ -2348,7 +2350,12 @@ srun torchrun \ * **Data Access:** All nodes must have access to the same shared filesystem (e.g., NFS, GPFS) where the `project_root` and the sharded preprocessing output are stored. -# Categorical policy learning +# Policy learning + +For supervised checkpoint attachment, mixed real/categorical heads, full model +input schemas, and mixed supervised/policy plans, see the +[unified learning guide](supervised-policy-learning.md). The configuration below +documents the compatible single-categorical authoring surface. `sequifier train` and `sequifier.rl_api.run_policy_training` execute the same `PolicyRunConfig`. Its ordered `phases` may contain `policy_imitation`, `iql`, @@ -2356,12 +2363,12 @@ srun torchrun \ of its predecessor's selected actor. Optimizers, schedulers, critics, samplers, and environments start fresh at phase boundaries. -This release supports one fixed categorical action, scalar or categorical +This legacy surface supports one fixed categorical action, scalar or categorical observations/static fields, causal fixed-length histories, FP32, and local single-process CPU/CUDA/MPS execution. Return conditioning is available only for imitation. A run containing another algorithm requires an unconditioned -actor throughout. Forecasting and policy phases cannot share a training plan. -Text tokenizers, continuous/joint actions, images/depth, distributed training, +actor throughout. Mixed phases use the unified run configuration linked above. +For this legacy surface, text tokenizers, continuous/joint actions, images/depth, distributed training, vectorized environments, and reward models are outside this contract. ## Authoring a run diff --git a/src/sequifier/io/actions.py b/src/sequifier/io/actions.py new file mode 100644 index 00000000..3617ab81 --- /dev/null +++ b/src/sequifier/io/actions.py @@ -0,0 +1,129 @@ +"""Typed action conversion shared by datasets, collection, and algorithms.""" + +from collections.abc import Mapping + +import torch + +from sequifier.io.values import domain_scalar, numeric_scalar + + +def action_latent_key(name, config): + key = "__policy_latent:" + name + while key in config.actions: + key = "__policy_latent:" + key + return key + + +def action_component_key(name, config): + key = "__policy_component:" + name + while key in config.actions: + key = "__policy_component:" + key + return key + + +def encode_action(value, config): + if not config.is_joint: + value = domain_scalar(value, "action") + try: + return config.action_values.index(value) + except ValueError as exc: + raise ValueError( + f"action {value!r} is outside the fixed vocabulary" + ) from exc + if not isinstance(value, Mapping) or { + key for key in value if key in config.actions + } != set(config.actions): + raise ValueError("actions must contain exactly the declared named fields") + result = {} + for name, field in config.actions.items(): + item = domain_scalar(value[name], name) + if field.type == "categorical": + if item not in field.values: + raise ValueError(f"{name}: action is outside the executable vocabulary") + item = field.values.index(item) + else: + item = numeric_scalar(item, name) + result[name] = item + allowed = set(config.actions) + for name, field in config.actions.items(): + if field.type != "real": + continue + key = action_latent_key(name, config) + allowed.add(key) + component_key = action_component_key(name, config) + allowed.add(component_key) + if component_key in value and value[component_key] is not None: + component = numeric_scalar(value[component_key], component_key) + if ( + component != int(component) + or component < -2 + or component >= len(field.atoms) + ): + raise ValueError("invalid real mixture component") + result[component_key] = int(component) + if key in value and value[key] is not None: + result[key] = numeric_scalar(value[key], key) + if set(value) - allowed: + raise ValueError("action record contains unknown fields") + return result + + +def action_batch(values, config, *, device=None, encoded=True): + values = list(values) + if not encoded: + values = [encode_action(value, config) for value in values] + if not config.is_joint: + return torch.tensor(values, dtype=torch.long, device=device) + result = { + name: torch.tensor( + [value[name] for value in values], + dtype=torch.long + if field.type == "categorical" + else ( + torch.float32 + if device is not None and torch.device(device).type == "mps" + else torch.float64 + ), + device=device, + ) + for name, field in config.actions.items() + } + latent_keys = { + action_latent_key(name, config) + for name, field in config.actions.items() + if field.type == "real" + } + for key in latent_keys: + result[key] = torch.tensor( + [value.get(key, float("nan")) for value in values], + dtype=torch.float32 + if device is not None and torch.device(device).type == "mps" + else torch.float64, + device=device, + ) + for name, field in config.actions.items(): + if field.type == "real": + key = action_component_key(name, config) + result[key] = torch.tensor( + [value.get(key, -2) for value in values], + dtype=torch.long, + device=device, + ) + return result + + +def action_record(actions, index=0): + if isinstance(actions, dict): + return {name: value[index].item() for name, value in actions.items()} + return int(actions[index]) + + +def decode_action(action, config): + if not config.is_joint: + return {config.action_column: config.action_values[action]} + return { + name: field.values[int(action[name])] + if field.type == "categorical" + else float(action[name]) + for name, field in config.actions.items() + } diff --git a/src/sequifier/io/episode_corpus.py b/src/sequifier/io/episode_corpus.py index d02bb600..a392c8d8 100644 --- a/src/sequifier/io/episode_corpus.py +++ b/src/sequifier/io/episode_corpus.py @@ -68,11 +68,11 @@ def __init__(self, sources, config, cache_size=8): if episode.previous_action is not None: action_index(episode.previous_action, config) for column, values in episode.observations.items(): - if column not in categorical: + if not config.is_joint and column not in categorical: for value in values: numeric_scalar(value, column) for column, value in episode.static_inputs.items(): - if column not in categorical: + if not config.is_joint and column not in categorical: numeric_scalar(value, column) self.metadata.append( EpisodeMetadata( diff --git a/src/sequifier/io/episode_features.py b/src/sequifier/io/episode_features.py index f76e25a2..6b9d5861 100644 --- a/src/sequifier/io/episode_features.py +++ b/src/sequifier/io/episode_features.py @@ -96,6 +96,15 @@ def validate_normalization(self, statistics): """Check FP32 transforms episode-by-episode without constructing windows.""" if self._validated_statistics == statistics: return + if self.config.is_joint: + from sequifier.io.policy_window import PolicyWindowView + + view = PolicyWindowView(self.config) + for episode in self.episodes: + for step in range(len(episode.actions)): + view.episode_inputs(episode, step, None) + self._validated_statistics = {k: dict(v) for k, v in statistics.items()} + return for episode, length in zip(self.episodes, self.lengths): for column in self.config.observation_columns + self.config.static_columns: if column in self.config.categorical_n_classes: diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py index 662595dd..082ca393 100644 --- a/src/sequifier/io/iql_dataset.py +++ b/src/sequifier/io/iql_dataset.py @@ -4,8 +4,9 @@ import torch +from sequifier.io.actions import action_batch from sequifier.io.batch import SequifierInputs -from sequifier.io.policy_window import PolicyWindowView, action_index +from sequifier.io.policy_window import PolicyWindowView from sequifier.io.tensors import TensorRecord from sequifier.io.values import numeric_scalar @@ -14,7 +15,7 @@ class IQLBatch(TensorRecord): current: SequifierInputs successor: SequifierInputs - action: torch.Tensor + action: torch.Tensor | dict[str, torch.Tensor] reward: torch.Tensor bootstrap: torch.Tensor @@ -23,7 +24,7 @@ class IQLTransitionDataset: """Index eligible decisions while retaining the source's bounded episode cache.""" def __init__(self, decisions): - if decisions.config.return_conditioned: + if decisions.config.return_conditioned and not decisions.config.is_joint: raise ValueError("IQL requires an unconditioned actor") self.decisions = decisions self.view = PolicyWindowView(decisions.config) @@ -107,9 +108,8 @@ def __getitem__(self, index): return IQLBatch( current=current, successor=next_inputs, - action=torch.tensor( - [action_index(episode.actions[step], self.decisions.config)], - dtype=torch.long, + action=action_batch( + [episode.actions[step]], self.decisions.config, encoded=False ), reward=torch.tensor([transitions.rewards[step]], dtype=torch.float32), bootstrap=torch.tensor( diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index 68ddcb81..d2bd2136 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -204,6 +204,18 @@ def imitation_example(dataset, index) -> SequifierBatch: inputs = PolicyWindowView(dataset.config).episode_inputs( episode, step, returns, action_ids ) + if dataset.config.is_joint: + from sequifier.io.actions import action_batch + + actions = action_batch([action_ids[step]], dataset.config) + metadata = dict(inputs.metadata) + metadata["target_valid_mask"] = torch.zeros_like(inputs.attention_valid_mask) + metadata["target_valid_mask"][0, -1] = True + return SequifierBatch( + inputs.inputs, + {name: value[:, None] for name, value in actions.items()}, + metadata, + ) targets = torch.zeros((1, dataset.config.context_length), dtype=torch.int64) targets[0, -1] = action_global_ids(dataset.config)[action_ids[step]] metadata = dict(inputs.metadata) diff --git a/src/sequifier/io/policy_materialization.py b/src/sequifier/io/policy_materialization.py new file mode 100644 index 00000000..c033b128 --- /dev/null +++ b/src/sequifier/io/policy_materialization.py @@ -0,0 +1,92 @@ +"""Materialize policy histories using the persisted model execution schema.""" + +import torch + +from sequifier.io.actions import decode_action +from sequifier.io.batch import SequifierInputs +from sequifier.model.execution_schema import ExecutionSchema + + +def encode_value(value, column, config, *, force_raw=False): + if isinstance(value, (list, tuple)): + return [ + encode_value(item, column, config, force_raw=force_raw) for item in value + ] + mapping = config.id_maps.get(column) + if mapping and (force_raw or config.input_encoding == "raw"): + if value in mapping: + return mapping[value] + if str(value) in mapping: + return mapping[str(value)] + from sequifier.special_tokens import SPECIAL_TOKEN_IDS + + special = { + label: number for number, label in SPECIAL_TOKEN_IDS.labels_by_id.items() + } + if value in special and special[value] < config.categorical_n_classes[column]: + return special[value] + unknown = SPECIAL_TOKEN_IDS.unknown + if unknown in mapping.values(): + return unknown + raise ValueError(f"{column}: value is outside the persisted categorical codec") + return value + + +def materialize_history(config, observations, previous_actions, static): + schema = ExecutionSchema.from_dict(config.input_schema) + rows = observations[-config.context_length :] + previous = previous_actions[-config.context_length :] + padding = config.context_length - len(rows) + features, metadata = {}, {} + for descriptor in schema.inputs: + key = descriptor.key + dtype = getattr(torch, descriptor.dtype) + if key == "attention_valid_mask": + metadata[key] = torch.tensor([[False] * padding + [True] * len(rows)]) + continue + if key in config.static_columns: + if key not in static: + raise ValueError(f"missing static model input {key!r}") + value = torch.as_tensor( + encode_value(static[key], key, config), dtype=dtype + ).unsqueeze(0) + else: + shape = (1, config.context_length, *descriptor.shape[2:]) + value = torch.zeros(shape, dtype=dtype) + for index, row in enumerate(rows): + if key in config.previous_action_inputs: + action = previous[index] + if action is None: + if key not in config.initial_inputs: + raise ValueError( + f"{key}: lagged action requires initial_inputs at episode start" + ) + raw = config.initial_inputs[key] + else: + raw = decode_action(action, config)[ + config.previous_action_inputs[key] + ] + else: + if key not in row: + raise ValueError(f"missing temporal model input {key!r}") + raw = row[key] + feedback = key in config.previous_action_inputs + encoded = torch.as_tensor( + encode_value(raw, key, config, force_raw=feedback), dtype=dtype + ) + if ( + feedback + and config.input_encoding == "model" + and key in config.input_statistics + ): + from sequifier.io.preprocessing import normalize_values + + encoded = normalize_values(encoded, config.input_statistics[key]) + if tuple(encoded.shape) != tuple(shape[2:]): + raise ValueError( + f"{key}: payload shape differs from the execution schema" + ) + value[0, padding + index] = encoded + (metadata if descriptor.role == "metadata" else features)[key] = value + schema.validate(features, metadata) + return SequifierInputs(features, metadata) diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py index 6f3c7380..b21c5f10 100644 --- a/src/sequifier/io/policy_window.py +++ b/src/sequifier/io/policy_window.py @@ -1,6 +1,7 @@ """Policy alignment over the shared named-tensor and model-window contracts.""" from collections.abc import Mapping, Sequence +from typing import cast import torch @@ -20,17 +21,31 @@ def normalize_policy_features(features, metadata, interface): features = dict(features) mask = metadata["attention_valid_mask"] for column, statistics in interface.selected_columns_statistics.items(): - features[column] = normalize_values(features[column], statistics).masked_fill( - ~mask, 0.0 - ) - if interface.policy.return_conditioned: + if column not in features or ( + interface.policy.is_joint and not interface.normalize_real_columns + ): + continue + if interface.policy.is_joint and interface.policy.input_encoding == "model": + continue + value = normalize_values(features[column], statistics) + if value.ndim >= 2 and value.shape[:2] == mask.shape: + valid = mask.reshape(*mask.shape, *([1] * (value.ndim - 2))) + value = value.masked_fill(~valid, 0.0) + features[column] = value + if interface.policy.return_conditioned and not interface.policy.is_joint: features[DESIRED_RETURN] = ( features[DESIRED_RETURN] / interface.policy.returns.scale ) return features -def action_index(value, config: CategoricalPolicyConfig) -> int: +def action_index( + value, config: CategoricalPolicyConfig +) -> int | dict[str, int | float]: + if config.is_joint: + from sequifier.io.actions import encode_action + + return encode_action(value, config) value = domain_scalar(value, "recorded action") try: return config.action_values.index(value) @@ -51,6 +66,10 @@ def action_target(config: CategoricalPolicyConfig) -> str: def encode_feature(value, column: str, config: CategoricalPolicyConfig): + if config.is_joint: + from sequifier.io.policy_materialization import encode_value + + return encode_value(value, column, config) mapping = config.id_maps.get(column) if mapping is not None: try: @@ -95,9 +114,16 @@ def episode_inputs( self, episode, step: int, returns, action_ids=None ) -> SequifierInputs: """Expose one causal decision from a validated episode in either lifecycle.""" + if self.config.return_conditioned and returns is None: + from sequifier.io.episode_features import return_labels + + length = episode.return_length(self.config.returns.finite_horizon) + if length is None: + raise ValueError("conditioned likelihood requires a known reward tail") + returns = return_labels(episode, length, self.config.returns.gamma) start = max(0, step + 1 - self.config.context_length) initial = ( - len(self.config.action_values) + (None if self.config.is_joint else len(self.config.action_values)) if episode.starts_episode else action_index(episode.previous_action, self.config) ) @@ -124,7 +150,7 @@ def successor_inputs(self, episode, step: int, successor) -> SequifierInputs: ] + [successor] previous = [ ( - len(self.config.action_values) + (None if self.config.is_joint else len(self.config.action_values)) if episode.starts_episode else action_index(episode.previous_action, self.config) ) @@ -132,16 +158,47 @@ def successor_inputs(self, episode, step: int, successor) -> SequifierInputs: else action_index(episode.actions[position - 1], self.config) for position in range(start, step + 2) ] - return self.inputs(observations, previous, None, episode.static_inputs) + budgets = None + if self.config.return_conditioned: + from sequifier.io.episode_features import return_labels + from sequifier.trajectory.returns import remaining_return + + length = episode.return_length(self.config.returns.finite_horizon) + if length is None: + raise ValueError("conditioned successors require a known reward tail") + returns = return_labels(episode, length, self.config.returns.gamma) + following = remaining_return( + returns[step], + episode.transitions.rewards[step], + self.config.returns.gamma, + ) + budgets = returns[start : step + 1] + [following] + return self.inputs(observations, previous, budgets, episode.static_inputs) def inputs( self, observations: Sequence[Mapping], - previous_actions: Sequence[int], + previous_actions: Sequence[int | dict[str, int | float] | None], desired_returns: Sequence[float] | None, static_inputs: Mapping, ) -> SequifierInputs: config = self.config + if config.is_joint: + from sequifier.io.policy_materialization import materialize_history + + if config.return_conditioned: + if desired_returns is None or len(desired_returns) != len(observations): + raise ValueError( + "return-conditioned histories require one return per observation" + ) + observations = [ + {**row, DESIRED_RETURN: value} + for row, value in zip(observations, desired_returns) + ] + return materialize_history( + config, observations, previous_actions, static_inputs + ) + previous_actions = cast(Sequence[int], previous_actions) length = len(observations) if not length or len(previous_actions) != length: raise ValueError( diff --git a/src/sequifier/io/sequence.py b/src/sequifier/io/sequence.py index 2b9a6f74..19d35642 100644 --- a/src/sequifier/io/sequence.py +++ b/src/sequifier/io/sequence.py @@ -8,6 +8,7 @@ BaseModel, ConfigDict, Field, + JsonValue, StrictBool, StrictInt, model_validator, @@ -42,7 +43,7 @@ class TransitionColumns(BaseModel): terminated: tuple[StrictBool, ...] | None = None truncated: tuple[StrictBool, ...] | None = None # None at a position means the true successor was not observed. Never a reset. - successors: tuple[dict[str, DomainValue] | None, ...] | None = None + successors: tuple[dict[str, JsonValue] | None, ...] | None = None bootstrap_allowed: tuple[StrictBool, ...] | None = None def at(self, index: int) -> dict: @@ -59,11 +60,11 @@ class SequenceEpisode(BaseModel): episode_id: str = Field(min_length=1) item_positions: tuple[StrictInt, ...] = Field(min_length=1) - observations: dict[str, tuple[DomainValue, ...]] = Field(min_length=1) - actions: tuple[DomainValue, ...] - static_inputs: dict[str, DomainValue] = Field(default_factory=dict) + observations: dict[str, tuple[JsonValue, ...]] = Field(min_length=1) + actions: tuple[DomainValue | dict[str, DomainValue], ...] + static_inputs: dict[str, JsonValue] = Field(default_factory=dict) starts_episode: StrictBool = True - previous_action: DomainValue | None = None + previous_action: DomainValue | dict[str, DomainValue] | None = None group_id: str | None = None transitions: TransitionColumns | None = None diff --git a/src/sequifier/io/tensors.py b/src/sequifier/io/tensors.py index 640e2dac..34840d40 100644 --- a/src/sequifier/io/tensors.py +++ b/src/sequifier/io/tensors.py @@ -28,9 +28,19 @@ class TensorRecord: """Mixin for dataclasses containing tensors, dictionaries and optional fields.""" def to(self: Record, device: str | torch.device) -> Record: + target = torch.device(device) return cast( Record, - _map_tensors(self, lambda value: value.to(device, non_blocking=True)), + _map_tensors( + self, + lambda value: value.to( + target, + dtype=torch.float32 + if target.type == "mps" and value.dtype == torch.float64 + else value.dtype, + non_blocking=True, + ), + ), ) def pin_memory(self: Record) -> Record: diff --git a/src/sequifier/io/values.py b/src/sequifier/io/values.py index ebdb8e93..80bec9c0 100644 --- a/src/sequifier/io/values.py +++ b/src/sequifier/io/values.py @@ -3,7 +3,7 @@ import math from collections.abc import Mapping from numbers import Integral, Real -from typing import Annotated, TypeAlias +from typing import Annotated, Any, TypeAlias from pydantic import BeforeValidator, FiniteFloat, StrictInt, StrictStr @@ -25,6 +25,21 @@ def numeric_scalar(value, label: str) -> float: return number +def portable_value(value): + """Own JSON-compatible copies of array, tensor, and nested input payloads.""" + if hasattr(value, "tolist"): + value = value.tolist() + if isinstance(value, dict): + return {key: portable_value(item) for key, item in value.items()} + if isinstance(value, (tuple, list)): + return [portable_value(item) for item in value] + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("model payloads must contain finite values") + if value is None or isinstance(value, (str, bool, int, float)): + return value + raise ValueError(f"unsupported model payload type: {type(value).__name__}") + + def domain_scalar(value, label: str): if isinstance(value, str): return value @@ -47,3 +62,7 @@ def numeric_fields(values, expected: set[str], label: str) -> dict[str, float]: key: numeric_scalar(value, f"{label}.{key}") for key, value in domain_fields(values, expected, label).items() } + + +def portable_fields(values: Mapping[str, Any]) -> dict[str, Any]: + return {name: portable_value(value) for name, value in values.items()} diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index a0947374..1d5938c4 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -19,7 +19,7 @@ from sequifier.model.layers import RMSNorm from sequifier.model.network import ComposableTransformerNetwork, ModelInterfaceModule from sequifier.model.static_conditioning import StaticConditionEncoder -from sequifier.objectives import CausalObjective, create_objective +from sequifier.objectives import BERTObjective, CausalObjective, create_objective from sequifier.special_tokens import resolve_categorical_decoder_ids from sequifier.typechecking import beartype @@ -392,6 +392,14 @@ def _build_composable_network( preconditioning_image_encoder=image_prefix_encoder, prefix_projection=prefix_projection, ) + from sequifier.model.prediction_distribution import ( + PredictionScales, + prediction_scale_fields, + ) + + distributions = prediction_scale_fields(interface) + if distributions: + routes[name].prediction_scales = PredictionScales(distributions).to(device) routes[name].depth_layouts = interface.depth_layouts routes[name].input_columns = tuple(interface.input_columns) routes[name].input_n_classes = interface.n_classes @@ -429,13 +437,20 @@ def _build_composable_network( interface.ingestion.initialization, ) + attention_objective = objective + if config.global_training.forward_attention is not None: + attention_objective = ( + CausalObjective + if config.global_training.forward_attention == "causal" + else BERTObjective + ) network = ComposableTransformerNetwork( backbone=backbone, interfaces=routes, - attention_mask_policy=objective.build_attention_mask_policy( + attention_mask_policy=attention_objective.build_attention_mask_policy( config.global_training.context_length ), - attention_mask_policy_with_prefix=objective.build_attention_mask_policy( + attention_mask_policy_with_prefix=attention_objective.build_attention_mask_policy( config.global_training.context_length + 1 ), context_length=config.global_training.context_length, diff --git a/src/sequifier/model/parameter_catalog.py b/src/sequifier/model/parameter_catalog.py index 43e308d2..1e671a38 100644 --- a/src/sequifier/model/parameter_catalog.py +++ b/src/sequifier/model/parameter_catalog.py @@ -92,6 +92,8 @@ def __init__(self, model: nn.Module): @beartype def _component(name: str) -> ParameterComponent: parts = name.split(".") + while len(parts) > 1 and (parts[0] == "network" or parts[0].isdigit()): + parts = parts[1:] root = parts[0] route_component = parts[2] if root == "interfaces" and len(parts) > 2 else root if route_component in { @@ -102,7 +104,7 @@ def _component(name: str) -> ParameterComponent: "prefix_projection", }: return "ingestion" - if route_component == "decoder": + if route_component in {"decoder", "prediction_scales"}: return "decoder" return "backbone" diff --git a/src/sequifier/model/prediction_distribution.py b/src/sequifier/model/prediction_distribution.py new file mode 100644 index 00000000..394dd355 --- /dev/null +++ b/src/sequifier/model/prediction_distribution.py @@ -0,0 +1,197 @@ +"""Scalar prediction distributions, independent of the learning objective.""" + +import math + +import torch +from torch import nn +from torch.nn import functional as F + + +def prediction_scale_fields(interface): + """Retain existing scale modules when a task temporarily fixes exploration.""" + fields = dict(interface.prediction_distributions) + if interface.policy is not None: + for name, field in interface.policy.actions.items(): + if name not in fields or field.learn_scale or not fields[name].learn_scale: + fields[name] = field + return fields + + +class PredictionScales(nn.Module): + """Optional global exploration parameters owned by the model artifact.""" + + def __init__(self, fields): + super().__init__() + self.log_scales = nn.ParameterDict( + { + name: nn.Parameter(torch.tensor(math.log(fields[name].scale))) + for name in sorted(fields) + if fields[name].learn_scale + } + ) + + def ensure_fields(self, fields): + for name in sorted(fields): + field = fields[name] + if field.learn_scale and name not in self.log_scales: + self.log_scales[name] = nn.Parameter( + torch.tensor(math.log(field.scale)) + ) + + def scale(self, name, field): + if not field.learn_scale: + return field.scale + return ( + self.log_scales[name] + .clamp(math.log(field.min_scale), math.log(field.max_scale)) + .exp() + ) + + +class RealPredictionDistribution: + """A Gaussian, optionally mapped bijectively onto an open real domain. + + Values and densities use physical coordinates. Explicit point masses can + represent endpoints; continuous samples retain transform latents for replay. + """ + + def __init__(self, location, scale, field): + self.dtype = torch.float32 if location.device.type == "mps" else torch.float64 + self.location = location.to(self.dtype) + self.scale = torch.as_tensor(scale, device=location.device).to(self.dtype) + self.field = field + if ( + not torch.isfinite(self.location).all() + or not torch.isfinite(self.scale).all() + or (self.scale <= 0).any() + ): + raise ValueError( + "real prediction locations and positive scales must be finite" + ) + self.normal = torch.distributions.Normal(self.location, self.scale) + + def _transform(self, latent): + lower, upper = self.field.lower, self.field.upper + if lower is not None and upper is not None: + return lower + (upper - lower) * torch.sigmoid(latent) + if lower is not None: + return lower + F.softplus(latent) + if upper is not None: + return upper - F.softplus(latent) + return latent + + def mode(self): + # Deterministic location; a transformed location need not be its mode. + return self._transform(self.location) + + @torch.no_grad() + def sample(self, generator=None): + device = generator.device if generator is not None else self.location.device + noise = torch.randn( + self.location.shape, + device=device, + dtype=self.dtype, + generator=generator, + ).to(self.location.device) + self.sampled_latent = self.location + self.scale * noise + value = self._transform(self.sampled_latent) + self.sampled_component = torch.full_like(value, -1, dtype=torch.long) + if self.field.atoms: + choice = torch.rand( + self.location.shape, device=device, generator=generator + ).to(value.device) + cumulative = 0.0 + for index, (atom, probability) in enumerate( + sorted(self.field.atoms.items()) + ): + selected = (choice >= cumulative) & (choice < cumulative + probability) + value = torch.where(selected, value.new_tensor(atom), value) + self.sampled_component = torch.where( + selected, + self.sampled_component.new_tensor(index), + self.sampled_component, + ) + cumulative += probability + return value + + def log_prob(self, value, latent=None, component=None): + value = value.to(self.dtype) + if not torch.isfinite(value).all(): + raise ValueError("real actions must be finite") + atoms = torch.zeros_like(value, dtype=torch.bool) + atom_score = torch.zeros_like(value) + if component is not None and ( + component.shape != value.shape + or (component < -2).any() + or (component >= len(self.field.atoms)).any() + ): + raise ValueError("invalid real mixture component") + for index, (atom, probability) in enumerate(sorted(self.field.atoms.items())): + selected = value == atom + if component is not None: + if ((component == index) & ~selected).any(): + raise ValueError( + "recorded mixture component differs from its action" + ) + selected &= (component == -2) | (component == index) + atoms |= selected + atom_score = torch.where( + selected, value.new_tensor(math.log(probability)), atom_score + ) + supplied = torch.zeros_like(atoms) if latent is None else ~torch.isnan(latent) + if latent is not None: + latent = latent.to(self.dtype) + if ( + latent.shape != value.shape + or not torch.isfinite(latent[supplied]).all() + ): + raise ValueError( + "real transform latents must be finite and match actions" + ) + comparable = supplied & ~atoms + if not torch.equal(self._transform(latent[comparable]), value[comparable]): + raise ValueError( + "recorded real latent does not reconstruct its executed action" + ) + lower, upper = self.field.lower, self.field.upper + invert = ~supplied & ~atoms + if lower is not None and (value[invert] <= lower).any(): + raise ValueError( + "real endpoints require a declared atom or a recorded transform latent" + ) + if upper is not None and (value[invert] >= upper).any(): + raise ValueError( + "real endpoints require a declared atom or a recorded transform latent" + ) + safe = torch.where(invert, value, self._transform(torch.zeros_like(value))) + if lower is not None and upper is not None: + fraction = (safe - lower) / (upper - lower) + inverse = fraction.log() - torch.log1p(-fraction) + elif lower is not None or upper is not None: + distance = safe - lower if lower is not None else upper - safe + inverse = distance + torch.log(-torch.expm1(-distance)) + else: + inverse = safe + z = torch.where(supplied, latent, inverse) if latent is not None else inverse + z = torch.where(atoms, torch.zeros_like(z), z) + if lower is not None and upper is not None: + jacobian = math.log(upper - lower) + F.logsigmoid(z) + F.logsigmoid(-z) + elif lower is not None or upper is not None: + jacobian = F.logsigmoid(z) + else: + jacobian = 0.0 + score = ( + self.normal.log_prob(z) + - jacobian + + math.log1p(-sum(self.field.atoms.values())) + ) + return torch.where(atoms, atom_score, score) + + def entropy(self): + if ( + self.field.lower is not None + or self.field.upper is not None + or self.field.atoms + ): + raise ValueError("transformed entropy requires a sampling estimator") + return self.normal.entropy() diff --git a/src/sequifier/rl/attachment.py b/src/sequifier/rl/attachment.py new file mode 100644 index 00000000..94614b02 --- /dev/null +++ b/src/sequifier/rl/attachment.py @@ -0,0 +1,150 @@ +"""Attach a policy task to a resolved model without rebuilding its interface.""" + +from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig +from sequifier.model.execution_schema import ExecutionSchema + + +def resolve_binding(execution, interface_name, binding): + interface = execution.interface(interface_name) + schema = ExecutionSchema.from_interface( + interface, execution.global_training.context_length + ) + if ( + -binding.decision_position + > execution.global_training.context_length - interface.decoder.support + 1 + ): + raise ValueError("decision_position lies outside the source decoder output") + values = binding.model_dump() + actions = dict(binding.actions) + if not actions: + raise ValueError( + "model attachment requires named actions with explicit real scales" + ) + for name, field in actions.items(): + if name not in interface.target_columns: + raise ValueError(f"{name}: policy action is not a model target") + if field.type != interface.target_column_types[name]: + raise ValueError(f"{name}: action type differs from the model") + if field.type == "categorical" and not field.values: + inverse = {v: k for k, v in interface.id_maps.get(name, {}).items()} + actions[name] = field.model_copy( + update={ + "values": [ + inverse.get(i, i) for i in interface.target_decoder_ids[name] + ] + } + ) + static = [ + item.key + for item in schema.inputs + if item.role == "static_feature" or item.key == "__preconditioning_image" + ] + temporal = [ + item.key + for item in schema.inputs + if item.key not in static and item.key != "attention_valid_mask" + ] + conditioned = ( + binding.return_conditioned + if "__policy_return" in temporal + and "return_conditioned" in binding.model_fields_set + else False + ) + values.update( + actions=actions, + action_order=[name for name in interface.target_columns if name in actions], + alignment="model_inputs", + return_conditioned=conditioned, + observation_columns=[ + name + for name in temporal + if name not in binding.previous_action_inputs + and (name != "__policy_return" or not conditioned) + ], + static_columns=list(static), + observation_n_classes={ + k: n for k, n in interface.n_classes.items() if k in temporal + }, + static_n_classes={k: n for k, n in interface.n_classes.items() if k in static}, + categorical_columns=[k for k in interface.categorical_columns if k in temporal], + id_maps={ + k: v + for k, v in interface.id_maps.items() + if k in set(temporal) | set(static) + }, + input_schema=schema.to_dict(), + input_statistics={ + name: stats + for name, stats in interface.selected_columns_statistics.items() + if name in set(temporal) | set(static) and interface.normalize_real_columns + }, + context_length=execution.global_training.context_length, + interface_name=interface_name, + ) + for column in binding.previous_action_inputs: + if column not in temporal: + raise ValueError(f"{column}: previous action binding is not a model input") + return CategoricalPolicyConfig.model_validate(values) + + +def apply_binding(execution, interface_name, policy): + """Persist distribution modules even when a later task selects fewer heads.""" + interface = execution.interface(interface_name) + from sequifier.model.prediction_distribution import prediction_scale_fields + + distributions = prediction_scale_fields(interface) + for name, field in policy.actions.items(): + if ( + name not in distributions + or field.learn_scale + or not distributions[name].learn_scale + ): + distributions[name] = field + interface.prediction_distributions = distributions + interface.policy = policy + authored = execution.model.interfaces[interface_name] + authored.prediction_distributions = distributions + authored.policy = PolicySchemaConfig.model_validate( + policy.model_dump(include=set(PolicySchemaConfig.model_fields)) + ) + + +def attach_policy(network, execution, interface_name, binding): + from sequifier.rl.policy import CategoricalPolicy + + original_parameters = {name for name, _ in network.named_parameters()} + policy = resolve_binding(execution, interface_name, binding) + apply_binding(execution, interface_name, policy) + actor = CategoricalPolicy.from_network(network, execution, interface_name) + actor.transfer_report = { + "mode": "adapt" + if binding.previous_action_inputs + or any( + field.lower is not None or field.upper is not None + for field in binding.actions.values() + ) + or any( + field.type == "categorical" + and set(actor.action_decoder_ids(name)) + != set(actor.interface.target_decoder_ids[name]) + for name, field in policy.actions.items() + ) + else "attach", + "interface": interface_name, + "reused_parameters": [ + name + for name, _ in network.named_parameters() + if name in original_parameters + ], + "new_parameters": [ + name + for name, _ in network.named_parameters() + if name not in original_parameters + ], + "source_preprocessing": "preserved", + "action_support": { + name: field.model_dump(mode="json") + for name, field in policy.actions.items() + }, + } + return actor diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 727ef972..1b5112f2 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -20,8 +20,6 @@ def phase_actor_path(directory, index, name): def validate_policy_weights(policy: CategoricalPolicy) -> None: for name, value in policy.network.state_dict().items(): - if value.is_floating_point() and value.dtype != torch.float32: - raise ValueError(f"policy tensor {name!r} requires FP32 precision") if not torch.isfinite(value).all(): raise ValueError(f"policy tensor {name!r} must be finite") statistics = policy.interface.selected_columns_statistics @@ -39,6 +37,7 @@ def save_policy( provenance={ **(metadata or {}), "policy_units": dict(getattr(policy, "units", {})), + "transfer": getattr(policy, "transfer_report", None), }, state_dict={ name: value.detach().cpu() @@ -54,6 +53,7 @@ def load_policy( *, device: str = "cpu", interface_name: str | None = None, + binding=None, ) -> CategoricalPolicy: if isinstance(path, ModelArtifact): network, config, artifact = restore_model_artifact( @@ -64,16 +64,25 @@ def load_policy( path, device=device, interface_name=interface_name ) if interface_name is None: - candidates = [ - name - for name, dataset in config.dataset_training.items() - if dataset.interface.policy is not None - ] + candidates = ( + list(config.interface_names) + if binding is not None + else [ + name + for name, dataset in config.dataset_training.items() + if dataset.interface.policy is not None + ] + ) if len(candidates) != 1: raise ValueError("Select one policy model_interface") interface_name = candidates[0] assert interface_name is not None - policy = CategoricalPolicy.from_network(network, config, interface_name) + if binding is not None and binding.is_joint: + from sequifier.rl.attachment import attach_policy + + policy = attach_policy(network, config, interface_name, binding) + else: + policy = CategoricalPolicy.from_network(network, config, interface_name) policy.units = dict(artifact.metadata.provenance.get("policy_units", {})) validate_policy_weights(policy) policy.eval() diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index f7a98c28..eaa82679 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -6,7 +6,8 @@ import torch -from sequifier.rl.distributions import CategoricalActionDistribution, action_generator +from sequifier.io.actions import action_record, decode_action +from sequifier.rl.distributions import action_generator from sequifier.rl.environment import validate_reset, validate_step from sequifier.rl.history import PolicyHistory from sequifier.trajectory.returns import remaining_return @@ -22,7 +23,7 @@ def stream_seed(seed, stream, reset_count): @dataclass(frozen=True) class PolicyDecision: domain_action: dict - action: int + action: int | dict log_probability: float entropy: float value: float @@ -49,6 +50,12 @@ def refresh(self): def reset(self, result, episode_id, desired_return=None): result = validate_reset(result, self.units) + if self.actor.config.return_conditioned and desired_return is None: + desired_return = self.actor.config.initial_desired_return + if desired_return is None: + raise ValueError( + "return-conditioned collection needs an initial desired return" + ) self.history.reset( result.observation, result.static_inputs, episode_id, desired_return ) @@ -65,9 +72,7 @@ def decide(self, value=None, *, deterministic=False): if value is not None: value.eval() features, metadata = actor.model_inputs(self.inputs) - distribution = CategoricalActionDistribution( - actor.prepared_logits(features, metadata)[:, -1] - ) + distribution = actor.distribution(self.inputs) action = distribution.select(deterministic, self.generator) estimate = ( value.prepared_value(features, metadata) @@ -76,12 +81,16 @@ def decide(self, value=None, *, deterministic=False): ) if not torch.isfinite(estimate).all(): raise ValueError("non-finite collection value") - index = int(action[0]) + index = action_record(action) return PolicyDecision( - {actor.config.action_column: actor.config.action_values[index]}, + decode_action(index, actor.config), index, float(distribution.log_prob(action)[0]), - float(distribution.entropy()[0]), + float( + distribution.entropy(self.generator)[0] + if actor.config.is_joint + else distribution.entropy()[0] + ), float(estimate[0]), ) diff --git a/src/sequifier/rl/distributions.py b/src/sequifier/rl/distributions.py index 283903c4..1fbad8a3 100644 --- a/src/sequifier/rl/distributions.py +++ b/src/sequifier/rl/distributions.py @@ -13,9 +13,9 @@ def action_generator(device: torch.device, seed: int) -> torch.Generator: class CategoricalActionDistribution: def __init__(self, logits: Tensor): logits = logits.float() - if logits.ndim < 2 or logits.shape[-1] < 2 or not torch.isfinite(logits).all(): + if logits.ndim < 2 or logits.shape[-1] < 1 or not torch.isfinite(logits).all(): raise ValueError( - "categorical policy logits must be finite with at least two actions" + "categorical policy logits must be finite with at least one action" ) self.logits = logits self.distribution = torch.distributions.Categorical(logits=self.logits) diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py index 424eea18..c9ff0040 100644 --- a/src/sequifier/rl/environment.py +++ b/src/sequifier/rl/environment.py @@ -3,22 +3,22 @@ import json from dataclasses import dataclass, field from importlib import import_module -from typing import Literal, Mapping, Protocol +from typing import Any, Literal, Mapping, Protocol -from sequifier.io.values import DomainValue, numeric_scalar +from sequifier.io.values import DomainValue, numeric_scalar, portable_fields @dataclass(frozen=True) class OnlineResetResult: - observation: Mapping[str, DomainValue] - static_inputs: Mapping[str, DomainValue] = field(default_factory=dict) + observation: Mapping[str, Any] + static_inputs: Mapping[str, Any] = field(default_factory=dict) units: Mapping[str, str] = field(default_factory=dict) @dataclass(frozen=True) class OnlineStepResult: # None is permitted only at a true terminal, where no bootstrap is requested. - observation: Mapping[str, DomainValue] | None + observation: Mapping[str, Any] | None reward: float terminated: bool = False truncated: bool = False @@ -143,7 +143,11 @@ def validate_reset(result, units): raise ValueError( "online reset requires an initial observation with the frozen units" ) - return result + return OnlineResetResult( + portable_fields(result.observation), + portable_fields(result.static_inputs), + dict(result.units), + ) def validate_step(result, units): @@ -164,7 +168,14 @@ def validate_step(result, units): ) if result.success is not None and not isinstance(result.success, bool): raise ValueError("adapter success must be boolean when defined") - return result + return OnlineStepResult( + portable_fields(result.observation) if result.observation is not None else None, + float(result.reward), + result.terminated, + result.truncated, + dict(result.units), + result.success, + ) def require_seeded_reset(adapter): diff --git a/src/sequifier/rl/grpo_buffer.py b/src/sequifier/rl/grpo_buffer.py index af342914..226ca87c 100644 --- a/src/sequifier/rl/grpo_buffer.py +++ b/src/sequifier/rl/grpo_buffer.py @@ -5,6 +5,7 @@ import torch +from sequifier.io.actions import action_batch from sequifier.io.batch import SequifierInputs, collate_model_inputs from sequifier.rl.history import history_inputs @@ -17,7 +18,7 @@ class GRPOTrajectory: episode_id: str frames: tuple context_length: int - actions: tuple[int, ...] + actions: tuple[int | dict, ...] rewards: tuple[float, ...] old_log_probabilities: tuple[float, ...] reference_log_probabilities: tuple[float, ...] @@ -41,8 +42,8 @@ def batches(self, actor, chunk_size): [self.inputs(actor.config, t) for t in range(start, end)] ), ).to(actor.device), - torch.tensor( - self.actions[start:end], dtype=torch.long, device=actor.device + action_batch( + self.actions[start:end], actor.config, device=actor.device ), torch.tensor( self.old_log_probabilities[start:end], diff --git a/src/sequifier/rl/grpo_collector.py b/src/sequifier/rl/grpo_collector.py index 860fd1d9..b007ef4c 100644 --- a/src/sequifier/rl/grpo_collector.py +++ b/src/sequifier/rl/grpo_collector.py @@ -6,6 +6,7 @@ import torch from sequifier.fingerprints import fingerprint +from sequifier.io.actions import action_batch from sequifier.rl.collection import PolicyCursor, stream_seed from sequifier.rl.environment import require_group_start, validate_start_token from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer @@ -87,11 +88,16 @@ def collect_groups( raise ValueError("GRPO group exceeded max_buffer_bytes") inputs = cursor.inputs decision = cursor.decide() + used += 4 * len( + json.dumps(decision.action, allow_nan=False).encode() + ) + if used > phase.max_buffer_bytes: + raise ValueError("GRPO actions exceeded max_buffer_bytes") reference_score = float( reference.evaluate_actions( inputs, - torch.tensor( - [decision.action], dtype=torch.long, device=actor.device + action_batch( + [decision.action], actor.config, device=actor.device ), )[0] ) diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py index 1076965e..9f93ade2 100644 --- a/src/sequifier/rl/history.py +++ b/src/sequifier/rl/history.py @@ -2,6 +2,7 @@ import json from collections import deque +from copy import deepcopy from dataclasses import dataclass from sequifier.io.policy_window import PolicyWindowView, encode_feature @@ -11,7 +12,7 @@ @dataclass(frozen=True) class PolicyFrame: observation: tuple - previous_action: int + previous_action: int | dict | None static_inputs: tuple episode_id: str step_id: int @@ -89,7 +90,7 @@ def _frame( "policy frame requires an episode ID and nonnegative integer step" ) action_count = len(self.spec.action_values) - if ( + if not self.spec.is_joint and ( not isinstance(previous_action, int) or isinstance(previous_action, bool) or not ( @@ -99,6 +100,23 @@ def _frame( ) ): raise ValueError("policy start token is allowed only on the first decision") + if self.spec.is_joint: + if set(observation) != set(self.spec.observation_columns) or set( + static + ) != set(self.spec.static_columns): + raise ValueError( + "environment payload does not match the model input binding" + ) + frame = PolicyFrame( + tuple(deepcopy(observation).items()), + deepcopy(previous_action), + tuple(deepcopy(static).items()), + episode_id, + step_id, + desired_return, + ) + history_inputs(self.spec, [frame]) + return frame observation = domain_fields( observation, set(self.spec.observation_columns), "policy observation" ) @@ -132,7 +150,7 @@ def _frame( def reset(self, observation, static_inputs, episode_id, desired_return=None): frame = self._frame( observation, - len(self.spec.action_values), + None if self.spec.is_joint else len(self.spec.action_values), static_inputs, episode_id, 0, @@ -144,7 +162,7 @@ def reset(self, observation, static_inputs, episode_id, desired_return=None): def advance(self, action, successor, desired_return=None): current = self.current - if not 0 <= action < len(self.spec.action_values): + if not self.spec.is_joint and not 0 <= action < len(self.spec.action_values): raise ValueError("history action is outside the actor vocabulary") frame = self._frame( successor, diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 5c546a99..4696bf31 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -11,12 +11,13 @@ from sequifier.config.policy_infer_config import PolicyInferenceConfig from sequifier.config.policy_train_config import PolicyTableColumns from sequifier.helpers import configure_determinism, normalize_path, write_data +from sequifier.io.actions import action_record, decode_action, encode_action from sequifier.io.batch import SequifierInputs, collate_model_inputs from sequifier.io.policy_window import PolicyWindowView from sequifier.io.sequence import ordered_sequence_rows from sequifier.io.values import domain_scalar from sequifier.rl.checkpoint import load_policy -from sequifier.rl.distributions import CategoricalActionDistribution, action_generator +from sequifier.rl.distributions import action_generator from sequifier.rl.environment import load_online_adapter from sequifier.rl.runtime import collect_policy_rollout from sequifier.rollout.history import numeric_scalar @@ -94,12 +95,13 @@ def infer_rollout(policy, data, config): seed = SeedHistory( observations={ column: [0.0] * (spec.context_length - 1) - + [domain_scalar(row[column], column)] + + [row[column] if spec.is_joint else domain_scalar(row[column], column)] for column in spec.observation_columns }, attention_valid_mask=[False] * (spec.context_length - 1) + [True], static_inputs={ - column: domain_scalar(row[column], column) for column in spec.static_columns + column: row[column] if spec.is_joint else domain_scalar(row[column], column) + for column in spec.static_columns }, ) budget = desired_return(config, spec, row) @@ -130,6 +132,9 @@ def infer_rollout(policy, data, config): def policy_windows(data, spec, config): # Rows without a sequence ID are independent single-observation decisions. # Within sequences, recorded actions determine the causal previous-action input. + if spec.is_joint: + yield from joint_policy_windows(data, spec, config) + return indexed = data.with_row_index("__policy_row") groups = ( indexed.partition_by(config.sequence_id_column, maintain_order=True) @@ -201,18 +206,23 @@ def predict_policy(policy, data, config): inputs = cast(SequifierInputs, collate_model_inputs(histories)).to( policy.device ) - distribution = CategoricalActionDistribution(policy(inputs)[:, -1, :]) - actions = distribution.select(config.deterministic, generator).cpu().tolist() + distribution = policy.distribution(inputs) + sampled = distribution.select(config.deterministic, generator) + actions = [action_record(sampled, i) for i in range(len(rows))] probabilities = ( distribution.distribution.probs.cpu().tolist() - if config.output_probabilities + if config.output_probabilities and not spec.is_joint else None ) for index, (row_index, action) in enumerate(zip(rows, actions)): output = { "__policy_row": row_index, - spec.action_column: spec.action_values[action], + **decode_action(action, spec), } + if spec.is_joint and config.output_probabilities: + output["action_log_probability"] = float( + distribution.log_prob(sampled)[index] + ) if probabilities is not None: output.update( { @@ -233,3 +243,50 @@ def predict_policy(policy, data, config): else predictions ) return predictions + + +def joint_policy_windows(data, spec, config): + indexed = data.with_row_index("__policy_row") + groups = ( + indexed.partition_by(config.sequence_id_column, maintain_order=True) + if config.sequence_id_column in data.columns + else [indexed.slice(i, 1) for i in range(len(indexed))] + ) + view = PolicyWindowView(spec) + for group in groups: + group = ordered_sequence_rows(group, config.item_position_column) + rows = group.to_dicts() + previous, observations = [], [] + for index, row in enumerate(rows): + if spec.previous_action_inputs: + if index: + prior = {name: rows[index - 1][name] for name in spec.actions} + previous.append(encode_action(prior, spec)) + elif row.get(config.item_position_column, 0) != 0: + if not config.previous_action_columns: + raise ValueError( + "fragment inference requires previous_action_columns" + ) + previous.append( + encode_action( + { + name: row[column] + for name, column in config.previous_action_columns.items() + }, + spec, + ) + ) + else: + previous.append(None) + else: + previous.append(None) + observations.append({name: row[name] for name in spec.observation_columns}) + yield ( + row["__policy_row"], + view.inputs( + observations, + previous, + None, + {name: row[name] for name in spec.static_columns}, + ), + ) diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 636531e1..4281949f 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -68,7 +68,11 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): actor, phase = context.actor.eval(), context.phase data = context.data.prepare_transitions() value, q1, q2, targets = make_iql_models( - actor.execution_config, actor.interface_name, len(actor.config.action_values) + actor.execution_config, + actor.interface_name, + actor.config.actions + if actor.config.is_joint + else len(actor.config.action_values), ) for module in (value, q1, q2, *targets): module.eval() diff --git a/src/sequifier/rl/joint_distribution.py b/src/sequifier/rl/joint_distribution.py new file mode 100644 index 00000000..a582c295 --- /dev/null +++ b/src/sequifier/rl/joint_distribution.py @@ -0,0 +1,265 @@ +"""Joint sampling and recorded-prefix scoring over existing model decoders.""" + +import torch + +from sequifier.io.actions import ( + action_component_key, + action_latent_key, + action_record, + decode_action, +) +from sequifier.io.preprocessing import denormalize_values, normalize_values +from sequifier.model.prediction_distribution import RealPredictionDistribution +from sequifier.rl.distributions import CategoricalActionDistribution + + +class JointActionDistribution: + """The model owns all weights; this object owns one encoded input window.""" + + def __init__(self, actor, inputs): + self.actor = actor + self.features, self.metadata = actor.model_inputs(inputs) + self.representation = actor.network.encode( + self.features, self.metadata, interface_name=actor.interface_name + ) + self.batch, self.time = self.representation.shape[:2] + + def _teachers(self, actions): + interface = self.actor.interface + teachers = {} + for name in interface.target_columns: + categorical = interface.target_column_types[name] == "categorical" + fill = interface.target_decoder_ids[name][0] if categorical else 0.0 + values = torch.full( + (self.batch, self.time), + fill, + dtype=torch.long if categorical else torch.float32, + device=self.representation.device, + ) + if name in actions: + action = actions[name] + if categorical: + ids = self.actor.action_decoder_ids(name) + action = torch.tensor(ids, device=values.device)[action.long()] + elif ( + interface.normalize_real_columns + and name in interface.selected_columns_statistics + ): + action = normalize_values( + action, interface.selected_columns_statistics[name] + ) + values[:, self.actor.config.decision_position] = action.to(values.dtype) + teachers[name] = values + return teachers + + def _parameters(self, actions): + return self.actor.network.decode_representation( + self.representation, + teacher_targets=self._teachers(actions), + teacher_valid_mask=self.metadata["attention_valid_mask"], + interface_name=self.actor.interface_name, + ) + + def _field(self, name, parameters): + actor, field = self.actor, self.actor.config.actions[name] + output = parameters[name][:, actor.config.decision_position] + if field.type == "categorical": + indices = [ + actor.interface.target_decoder_ids[name].index(i) + for i in actor.action_decoder_ids(name) + ] + return CategoricalActionDistribution(output[:, indices]) + location = output.squeeze(-1) + stats = actor.interface.selected_columns_statistics.get(name) + if stats is not None and actor.interface.normalize_real_columns: + location = denormalize_values(location, stats) + return RealPredictionDistribution( + location, + actor.network.resolve_interface( + actor.interface_name + ).prediction_scales.scale(name, field), + field, + ) + + def _active(self, name, actions): + mask = torch.ones( + self.batch, dtype=torch.bool, device=self.representation.device + ) + for predecessor, labels in self.actor.config.actions[name].active_when.items(): + if predecessor not in actions: + raise ValueError( + "conditional activity requires a preceding action field" + ) + field = self.actor.config.actions[predecessor] + allowed = ( + [field.values.index(label) for label in labels] + if field.type == "categorical" + else labels + ) + mask &= torch.isin( + actions[predecessor], torch.tensor(allowed, device=mask.device) + ) + return mask + + def select(self, deterministic, generator): + actions = {} + for name in self.actor.action_order: + distribution = self._field(name, self._parameters(actions)) + value = ( + distribution.mode() if deterministic else distribution.sample(generator) + ) + field = self.actor.config.actions[name] + inactive = ( + field.values.index(field.inactive_value) + if field.type == "categorical" and field.active_when + else (0 if field.type == "categorical" else field.inactive_value) + ) + active = self._active(name, actions) + actions[name] = torch.where(active, value, value.new_tensor(inactive)) + if isinstance(distribution, RealPredictionDistribution): + latent = ( + distribution.location + if deterministic + else distribution.sampled_latent + ) + actions[action_latent_key(name, self.actor.config)] = latent + actions[action_component_key(name, self.actor.config)] = ( + torch.full_like(value, -1, dtype=torch.long) + if deterministic + else distribution.sampled_component + ) + return actions + + def sample(self, generator=None): + return self.select(False, generator) + + def mode(self): + return self.select(True, None) + + def log_prob(self, actions): + fields = {key for key in actions if key in self.actor.config.actions} + if fields != set(self.actor.config.actions): + raise ValueError("joint likelihood requires every action field") + allowed = set(self.actor.config.actions) + for name, field in self.actor.config.actions.items(): + if field.type == "real": + allowed.update( + ( + action_latent_key(name, self.actor.config), + action_component_key(name, self.actor.config), + ) + ) + if set(actions) - allowed: + raise ValueError("joint action contains unknown fields") + parameters = self._parameters(actions) + score = self.representation.new_zeros( + self.batch, + dtype=torch.float32 + if self.representation.device.type == "mps" + else torch.float64, + ) + for name in self.actor.action_order: + value = actions[name] + if value.shape != (self.batch,): + raise ValueError(f"{name}: expected one action per decision") + distribution = self._field(name, parameters) + active = self._active(name, actions) + field = self.actor.config.actions[name] + inactive = ( + field.values.index(field.inactive_value) + if field.type == "categorical" and field.active_when + else (0 if field.type == "categorical" else field.inactive_value) + ) + if (value[~active] != inactive).any(): + raise ValueError(f"{name}: inactive action differs from its point mass") + safe = torch.where(active, value, distribution.mode().detach()) + latent = actions.get(action_latent_key(name, self.actor.config)) + if ( + isinstance(distribution, RealPredictionDistribution) + and latent is not None + ): + latent = torch.where(active, latent, distribution.location.detach()) + mixture = actions.get(action_component_key(name, self.actor.config)) + if mixture is not None: + mixture = torch.where(active, mixture, -1) + component = distribution.log_prob( + safe, latent=latent, component=mixture + ) + else: + component = distribution.log_prob(safe) + score = score + torch.where(active, component, 0.0) + return score.float() + + @property + def entropy_is_exact(self): + return not self.actor.autoregressive_actions and all( + not field.active_when + and field.lower is None + and field.upper is None + and not field.atoms + for field in self.actor.config.actions.values() + ) + + def entropy(self, generator=None): + if self.entropy_is_exact: + parameters = self._parameters({}) + return ( + torch.stack( + [ + self._field(name, parameters).entropy() + for name in self.actor.action_order + ] + ) + .sum(dim=0) + .float() + ) + # Score-function estimator includes the distribution of sampled prefixes. + # Its value estimates -E log p; its surrogate has the entropy gradient. + with torch.no_grad(): + actions = self.sample(generator) + logp = self.log_prob(actions) + surrogate = -(logp.detach() + 1) * logp + return -logp.detach() + surrogate - surrogate.detach() + + def kl(self, reference, generator=None): + if self.actor.config.actions != reference.actor.config.actions: + raise ValueError( + "KL requires matching action measures and physical transforms" + ) + if not self.actor.autoregressive_actions and not any( + field.active_when for field in self.actor.config.actions.values() + ): + current, frozen = self._parameters({}), reference._parameters({}) + values = [] + for name, field in self.actor.config.actions.items(): + p, q = self._field(name, current), reference._field(name, frozen) + if isinstance(p, CategoricalActionDistribution) and isinstance( + q, CategoricalActionDistribution + ): + values.append( + torch.distributions.kl_divergence( + p.distribution, q.distribution + ) + ) + else: + if not isinstance(p, RealPredictionDistribution) or not isinstance( + q, RealPredictionDistribution + ): + raise ValueError("reference distribution families differ") + values.append( + (1 - sum(field.atoms.values())) + * torch.distributions.kl_divergence(p.normal, q.normal) + ) + return torch.stack(values).sum(dim=0).float() + with torch.no_grad(): + actions = self.sample(generator) + logp = self.log_prob(actions) + delta = logp - reference.log_prob(actions) + surrogate = (delta.detach() + 1) * logp + return delta.detach() + surrogate - surrogate.detach() + + def domain_actions(self, actions): + return [ + decode_action(action_record(actions, i), self.actor.config) + for i in range(self.batch) + ] diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 39a41347..7e7458d8 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -45,8 +45,18 @@ def _initialize_actor(config, data): config.initialize_from, device=config.device, interface_name=config.policy_interface, + binding=config.policy if config.policy.is_joint else None, ) validate_declared_policy(config.policy, source.config, "initialize_from.policy") + if source.config.is_joint: + from sequifier.artifacts.model_config import validate_execution_attachment + from sequifier.artifacts.model_export import model_execution_config + + validate_execution_attachment( + config.execution_model, model_execution_config(source.execution_config) + ) + source.units = dict(config.units) + return source actor = build_actor( config, source.config, source.interface.selected_columns_statistics ) @@ -55,6 +65,23 @@ def _initialize_actor(config, data): ) actor.network.load_state_dict(source.network.state_dict(), strict=True) else: + if config.policy.is_joint: + from sequifier.artifacts.model_config import ( + resolved_config_from_model_config, + ) + + execution, _ = resolved_config_from_model_config( + config.execution_model, + device=config.device, + interface_name=config.policy_interface, + ) + return build_actor( + config, + config.policy, + execution.interface( + config.policy_interface + ).selected_columns_statistics, + ) first = data[0] fitting = first.fitting_dataset() if first is not None else None fitted = ( @@ -134,6 +161,7 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: adapter_contract, { "path": config.initialize_from, + "transfer": getattr(actor, "transfer_report", None), "weight_hash": weight_fingerprint(actor.network.state_dict()), }, clone_weights(actor.network), diff --git a/src/sequifier/rl/models.py b/src/sequifier/rl/models.py index e0278108..9b61e913 100644 --- a/src/sequifier/rl/models.py +++ b/src/sequifier/rl/models.py @@ -10,8 +10,54 @@ from sequifier.model.factory import build_transformer_network +class ActionEncoder(nn.Module): + """Fuse typed categorical embeddings and standardized real projections.""" + + def __init__(self, fields, width, statistics): + super().__init__() + self.fields = {name: fields[name] for name in sorted(fields)} + self.statistics = statistics + self.encoders = nn.ModuleDict( + { + name: nn.Embedding(len(field.values), width) + if field.type == "categorical" + else nn.Linear(1, width) + for name, field in self.fields.items() + } + ) + self.fusion = nn.Linear(len(fields) * (width + 1), width) + + def forward(self, actions): + from sequifier.io.preprocessing import normalize_values + + pieces = [] + for name, field in self.fields.items(): + value = actions[name] + active = torch.ones_like(value, dtype=torch.bool) + for predecessor, labels in field.active_when.items(): + parent = self.fields[predecessor] + allowed = ( + [parent.values.index(label) for label in labels] + if parent.type == "categorical" + else labels + ) + active &= torch.isin( + actions[predecessor], torch.tensor(allowed, device=value.device) + ) + if field.type == "real": + value = value.float() + if name in self.statistics: + value = normalize_values(value, self.statistics[name]) + value = value[:, None] + encoded = self.encoders[name](value) + pieces.extend((encoded * active[:, None], active.float()[:, None])) + return self.fusion(torch.cat(pieces, dim=-1)) + + class HistoryCritic(nn.Module): - def __init__(self, execution_config, interface_name: str, action_count: int | None): + def __init__( + self, execution_config, interface_name: str, action_count: int | dict | None + ): super().__init__() self.network = build_transformer_network( execution_config, device=torch.device(execution_config.device) @@ -24,7 +70,17 @@ def __init__(self, execution_config, interface_name: str, action_count: int | No parameter.requires_grad_(False) width = self.network.dim_model self.action_embedding = ( - nn.Embedding(action_count, width) if action_count else None + ActionEncoder( + action_count, + width, + self.interface.selected_columns_statistics + if self.interface.normalize_real_columns + else {}, + ) + if isinstance(action_count, dict) + else nn.Embedding(action_count, width) + if action_count + else None ) self.head = nn.Sequential( nn.Linear(width * (2 if action_count else 1), width), @@ -33,7 +89,11 @@ def __init__(self, execution_config, interface_name: str, action_count: int | No ) self.to(execution_config.device) - def forward(self, history: SequifierInputs, action: torch.Tensor | None = None): + def forward( + self, + history: SequifierInputs, + action: torch.Tensor | dict[str, torch.Tensor] | None = None, + ): if (action is None) != (self.action_embedding is None): raise ValueError("Q requires a current action and V forbids one") features = normalize_policy_features( @@ -50,7 +110,7 @@ def prepared_value(self, features, metadata, action=None): if self.action_embedding is not None: if action is None: raise ValueError("Q requires a current action") - if ( + if not isinstance(action, dict) and ( action.dtype != torch.long or action.ndim != 1 or action.shape[0] != state.shape[0] @@ -60,7 +120,7 @@ def prepared_value(self, features, metadata, action=None): return self.head(state).squeeze(-1).float() -def make_iql_models(execution_config, interface_name: str, action_count: int): +def make_iql_models(execution_config, interface_name: str, action_count: int | dict): value = HistoryCritic(execution_config, interface_name, None) q1 = HistoryCritic(execution_config, interface_name, action_count) q2 = HistoryCritic(execution_config, interface_name, action_count) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 86a8d1a5..786c069d 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -26,7 +26,9 @@ ) from sequifier.io.preprocessing import normalization_statistics from sequifier.model.execution_schema import ExecutionSchema +from sequifier.model.prediction_distribution import PredictionScales from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.rl.joint_distribution import JointActionDistribution if TYPE_CHECKING: from sequifier.model.network import ComposableTransformerNetwork @@ -39,9 +41,27 @@ def policy_execution_config( *, global_training=None, statistics=None, + execution_model=None, **run_options, ): """Resolve policy alignment directly into the ordinary execution contract.""" + if config.is_joint: + if execution_model is None: + raise ValueError( + "joint policies require a resolved model execution contract" + ) + from sequifier.artifacts.model_config import resolved_config_from_model_config + + execution, _ = resolved_config_from_model_config( + execution_model, device=device, interface_name=config.interface_name + ) + from sequifier.rl.attachment import apply_binding + + apply_binding(execution, config.interface_name, config) + for key, value in run_options.items(): + if key in type(execution).model_fields: + setattr(execution, key, value) + return execution columns = config.observation_columns + config.static_columns + [PREVIOUS_ACTION] if config.return_conditioned: columns.append(DESIRED_RETURN) @@ -131,7 +151,7 @@ def from_network(cls, network, execution_config, interface_name: str): raise ValueError( f"Model interface {interface_name!r} has no policy alignment" ) - if interface.target_decoder_ids[ + if not interface.policy.is_joint and interface.target_decoder_ids[ interface.target_columns[0] ] != action_global_ids(interface.policy): raise ValueError("policy decoder IDs must exactly match executable actions") @@ -140,6 +160,8 @@ def from_network(cls, network, execution_config, interface_name: str): policy.execution_config = execution_config policy.interface_name = interface_name policy.network = network + if policy.config.is_joint: + policy._resolve_actions() return policy @property @@ -154,9 +176,122 @@ def device(self) -> torch.device: def backbone(self): return self.network.backbone + def _resolve_actions(self): + from sequifier.model.decoders import AutoregressiveTransformerDecoderBranch + + route = self.network.resolve_interface(self.interface_name) + prerequisites = { + name: set(field.active_when) for name, field in self.config.actions.items() + } + for branch in route.decoder.branches.values(): + targets = [ + name for name in branch.target_columns if name in self.config.actions + ] + if isinstance(branch, AutoregressiveTransformerDecoderBranch) and targets: + last = max(branch.target_columns.index(name) for name in targets) + if set(branch.target_columns[: last + 1]) - set(targets): + raise ValueError( + "autoregressive actions must include all preceding decoder targets" + ) + for index, name in enumerate(targets): + prerequisites[name].update(targets[:index]) + pending = [ + name for name in self.interface.target_columns if name in prerequisites + ] + if set(pending) != set(self.config.actions): + raise ValueError("policy actions must name existing model targets") + order = [] + while pending: + ready = next( + (name for name in pending if prerequisites[name] <= set(order)), None + ) + if ready is None: + raise ValueError( + "action activity/decoder dependencies are cyclic or unknown" + ) + order.append(ready) + pending.remove(ready) + self.action_order = tuple(order) + self.autoregressive_actions = any( + isinstance(branch, AutoregressiveTransformerDecoderBranch) + and len( + [name for name in branch.target_columns if name in self.config.actions] + ) + > 1 + for branch in route.decoder.branches.values() + ) + for index, name in enumerate(order): + field = self.config.actions[name] + if field.type != self.interface.target_column_types[name]: + raise ValueError(f"{name}: action type differs from the model target") + if set(field.active_when) - set(order[:index]): + raise ValueError( + "activity conditions must reference earlier action fields" + ) + if field.type == "categorical": + self.action_decoder_ids(name) + if field.active_when and field.inactive_value not in field.values: + raise ValueError( + f"{name}: inactive categorical value must be executable" + ) + for predecessor, labels in field.active_when.items(): + parent = self.config.actions[predecessor] + if parent.type == "categorical" and any( + label not in parent.values for label in labels + ): + raise ValueError( + f"{name}: activity condition uses an unknown category" + ) + if not hasattr(route, "prediction_scales"): + route.prediction_scales = PredictionScales(self.config.actions).to( + self.device + ) + else: + route.prediction_scales.ensure_fields(self.config.actions) + route.prediction_scales.to(self.device) + + def action_decoder_ids(self, name): + field = self.config.actions[name] + mapping = self.interface.id_maps.get(name, {}) + ids = [] + for label in field.values: + key = ( + label + if isinstance(label, (str, int)) + else int(label) + if label.is_integer() + else str(label) + ) + global_id = mapping.get(key, mapping.get(str(label), mapping.get(str(key)))) + if global_id is None: + if ( + isinstance(label, int) + and label in self.interface.target_decoder_ids[name] + ): + global_id = label + else: + raise ValueError( + f"{name}: action label {label!r} is absent from the model codec" + ) + if global_id not in self.interface.target_decoder_ids[name]: + raise ValueError(f"{name}: action is absent from decoder support") + ids.append(global_id) + if not ids or len(ids) != len(set(ids)): + raise ValueError( + f"{name}: categorical actions need distinct executable decoder IDs" + ) + return ids + def set_normalization_statistics( self, statistics: Mapping[str, Mapping[str, float]] ) -> None: + if self.config.is_joint: + self.interface.selected_columns_statistics = ( + normalization_statistics(statistics) + if self.interface.normalize_real_columns + else {name: dict(values) for name, values in statistics.items()} + ) + return expected = ( set(self.config.observation_columns + self.config.static_columns) - self.config.categorical_n_classes.keys() @@ -185,31 +320,48 @@ def model_inputs( raise ValueError("frozen input transforms produced non-finite values") return features, inputs.metadata - def forward(self, inputs: SequifierInputs) -> Tensor: + def forward(self, inputs: SequifierInputs) -> Tensor | dict[str, Tensor]: return self.prepared_logits(*self.model_inputs(inputs)) - def prepared_logits(self, features, metadata) -> Tensor: + def prepared_logits(self, features, metadata) -> Tensor | dict[str, Tensor]: """Execute features already checked and normalized by model_inputs.""" - logits = self.network( + outputs = self.network( features, metadata, interface_name=self.interface_name - ).logits[self.interface.target_columns[0]] + ).logits + if self.config.is_joint: + return { + name: value.masked_fill( + ~metadata["attention_valid_mask"][:, -value.shape[1] :, None], 0.0 + ) + for name, value in outputs.items() + } + logits = outputs[self.interface.target_columns[0]] return logits.masked_fill( ~metadata["attention_valid_mask"][:, -logits.shape[1] :, None], 0.0 ) - def distribution(self, inputs: SequifierInputs) -> CategoricalActionDistribution: - return CategoricalActionDistribution(self(inputs)) + def distribution( + self, inputs: SequifierInputs + ) -> CategoricalActionDistribution | JointActionDistribution: + if self.config.is_joint: + return JointActionDistribution(self, inputs) + return CategoricalActionDistribution(self(inputs)[:, -1]) def act(self, inputs: SequifierInputs, *, deterministic: bool, generator): - distribution = CategoricalActionDistribution(self(inputs)[:, -1]) + distribution = self.distribution(inputs) action = distribution.select(deterministic, generator) return action, distribution.log_prob(action) - def evaluate_actions(self, inputs: SequifierInputs, actions: Tensor) -> Tensor: - return CategoricalActionDistribution(self(inputs)[:, -1]).log_prob(actions) + def evaluate_actions( + self, inputs: SequifierInputs, actions: Tensor | dict[str, Tensor] + ) -> Tensor: + return self.distribution(inputs).log_prob(actions) def validate_inputs(self, inputs: SequifierInputs) -> None: - expected = set(self.interface.input_columns) + schema = ExecutionSchema.from_interface( + self.interface, self.config.context_length + ) + expected = {item.key for item in schema.inputs if item.role != "metadata"} if set(inputs.inputs) != expected: raise ValueError( "policy inputs must match the persisted named feature schema" @@ -236,9 +388,15 @@ def validate_inputs(self, inputs: SequifierInputs) -> None: for value in [*inputs.inputs.values(), *inputs.metadata.values()] ): raise ValueError("policy features and masks must be on the model device") + if self.config.is_joint: + return ids = action_global_ids(self.config) allowed = torch.tensor(ids + [ids[-1] + 1], device=self.device) if not torch.isin(inputs.inputs[PREVIOUS_ACTION], allowed).all(): raise ValueError( "previous actions must be executable action IDs or the input-only start token" ) + + +# Backward-compatible name for the original single-categorical API. +Policy = CategoricalPolicy diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py index 7ce40a69..2cad9520 100644 --- a/src/sequifier/rl/ppo_buffer.py +++ b/src/sequifier/rl/ppo_buffer.py @@ -8,6 +8,7 @@ import torch from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.io.actions import action_batch, decode_action from sequifier.io.atomic import write_json_atomic from sequifier.io.batch import SequifierInputs from sequifier.io.tensors import collate_tensor_records @@ -18,8 +19,8 @@ class PPODecision: frame: int successor: int | None - action: int - domain_action: str | float | int + action: int | dict + domain_action: str | float | int | dict old_log_probability: float old_value: float entropy: float @@ -81,9 +82,9 @@ def batch(self, indices, device): [self.window(self.decisions[index].frame) for index in indices] ), ).to(device), - torch.tensor( + action_batch( [self.decisions[index].action for index in indices], - dtype=torch.long, + self.spec, device=device, ), torch.tensor( @@ -135,7 +136,7 @@ def archive(self, path, *, include_scores): decisions.append(record) payload = { "format": "sequifier.ppo-collection", - "version": 1, + "version": 2 if self.spec.is_joint else 1, "policy_version": self.policy_version, "policy": self.spec.model_dump(mode="json"), "frames": [frame.state_dict() for frame in self.frames], @@ -147,10 +148,9 @@ def archive(self, path, *, include_scores): def load_collection_record(path): """Read the independent archival format without changing v1 episode readers.""" payload = json.loads(Path(path).read_text(encoding="utf-8")) - if ( - payload.get("format") != "sequifier.ppo-collection" - or payload.get("version") != 1 - ): + if payload.get("format") != "sequifier.ppo-collection" or payload.get( + "version" + ) not in {1, 2}: raise ValueError("expected version-1 PPO collection record") from sequifier.config.policy_config import CategoricalPolicyConfig from sequifier.rl.history import PolicyFrame @@ -165,8 +165,13 @@ def load_collection_record(path): step["policy_version"] != payload["policy_version"] or (step["episode_id"], step["step_id"]) != (frame.episode_id, frame.step_id) - or not 0 <= step["action"] < len(spec.action_values) - or step["domain_action"] != spec.action_values[step["action"]] + or (not spec.is_joint and not 0 <= step["action"] < len(spec.action_values)) + or step["domain_action"] + != ( + decode_action(step["action"], spec) + if spec.is_joint + else spec.action_values[step["action"]] + ) or step["bootstrap"] != (not step["terminated"]) ): raise ValueError("collection record violates the decision contract") @@ -241,12 +246,13 @@ def __init__(self, spec, version, horizon, initial_frames, max_bytes): ) self.used_bytes = sum(frame_storage_bytes(frame) for frame in self.frames) self.sealed = False + self.decision_bytes = 0 def append_frame(self, frame): if self.sealed: raise ValueError("PPO buffer is sealed") cost = frame_storage_bytes(frame) - if self.used_bytes + cost + len(self.decisions) * 1024 > self.max_bytes: + if self.used_bytes + cost + self.decision_bytes > self.max_bytes: raise ValueError("PPO history exceeded its declared memory budget") if len(self.frames) >= 2 * self.horizon + self.spec.context_length: raise ValueError("PPO history exceeded its bounded frame capacity") @@ -259,8 +265,11 @@ def append(self, decision): raise ValueError("PPO buffer is sealed or full") if decision.policy_version != self.version: raise ValueError("policy generation changed during collection") + self.decision_bytes += 1024 + 4 * len( + json.dumps(asdict(decision), allow_nan=False).encode() + ) self.decisions.append(decision) - if self.used_bytes + len(self.decisions) * 1024 > self.max_bytes: + if self.used_bytes + self.decision_bytes > self.max_bytes: raise ValueError("PPO decisions exceeded the memory budget") def seal(self, gamma, gae_lambda, normalize): @@ -275,5 +284,5 @@ def seal(self, gamma, gae_lambda, normalize): tuple(self.decisions), advantages, targets, - max(self.estimated_bytes, self.used_bytes + len(self.decisions) * 1024), + max(self.estimated_bytes, self.used_bytes + self.decision_bytes), ) diff --git a/src/sequifier/rl/ppo_collector.py b/src/sequifier/rl/ppo_collector.py index 520d4958..7834c3d8 100644 --- a/src/sequifier/rl/ppo_collector.py +++ b/src/sequifier/rl/ppo_collector.py @@ -96,7 +96,9 @@ def collect(self, horizon, policy_version): current_reference, successor_reference, decision.action, - self.actor.config.action_values[decision.action], + decision.domain_action + if self.actor.config.is_joint + else self.actor.config.action_values[decision.action], decision.log_probability, decision.value, decision.entropy, diff --git a/src/sequifier/rl/ppo_update.py b/src/sequifier/rl/ppo_update.py index 85089e4a..cebc73ff 100644 --- a/src/sequifier/rl/ppo_update.py +++ b/src/sequifier/rl/ppo_update.py @@ -4,8 +4,6 @@ import torch -from sequifier.rl.distributions import CategoricalActionDistribution - def ppo_objectives( log_probability, @@ -118,7 +116,7 @@ def update_ppo(session, value, buffer): history, actions, old_logp, old_v, advantages, target = buffer.batch( indices, actor.device ) - distribution = CategoricalActionDistribution(actor(history)[:, -1]) + distribution = actor.distribution(history) policy_loss, value_loss, metrics = ppo_objectives( distribution.log_prob(actions), old_logp, diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 11c0500c..3f2a7971 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -149,6 +149,7 @@ def build_actor(config, fitted, statistics): config.model, config.device, statistics=statistics, + execution_model=config.execution_model, project_root=config.project_root, model_name=config.model_name, seed=config.seed, @@ -222,10 +223,12 @@ def preflight_optimization(config, actor, data): """Resolve constructors and authored scheduler budgets before creating outputs.""" budgets = [] for phase, part in zip(config.phases, data): + for path in phase.freeze: + actor.network.get_submodule(path) size = len(part.training) if part is not None else None budget = phase.optimizer_step_budget(size) for optimization in phase.optimization_configs().values(): optimization.validate_budget(budget) - optimization.build(actor) + optimization.build(actor.network) budgets.append(budget) return budgets diff --git a/src/sequifier/rl/references.py b/src/sequifier/rl/references.py index 20695a0b..a5fac4c5 100644 --- a/src/sequifier/rl/references.py +++ b/src/sequifier/rl/references.py @@ -54,7 +54,10 @@ def _artifact(self, path): checksum = file_checksum(path) if path not in self.artifacts or self.checksums[path] != checksum: source = load_policy( - path, device="cpu", interface_name=self.config.policy_interface + path, + device="cpu", + interface_name=self.config.policy_interface, + binding=self.config.policy if self.config.policy.is_joint else None, ) self.recovery.representation.validate_actor( source, self.config.units, f"reference.{path}" diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 2d5a1c67..17f70488 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -82,7 +82,15 @@ def load(cls, path, config, plan, data_identity): "policy resume requires a version-5 or version-6 policy run checkpoint" ) try: - require_equal(plan, payload["plan"], "resume.plan") + saved_plan = payload["plan"] + if saved_plan != plan: + from sequifier.config.train_config import PolicyRunConfig + + upgraded = PolicyRunConfig.model_validate( + {**saved_plan, "resume": config.resume.model_dump()} + ) + saved_plan = upgraded.model_dump(mode="json", exclude={"resume"}) + require_equal(plan, saved_plan, "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") if payload["format_version"] == 5: records = payload.pop("completed_artifacts") diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 17d9615b..59f84cf6 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -58,9 +58,16 @@ class ActorRepresentation: @classmethod def from_actor(cls, actor, units): + from pydantic_core import to_jsonable_python + + from sequifier.artifacts.model_export import model_execution_config + return cls( json.dumps( { + "execution": to_jsonable_python( + model_execution_config(actor.execution_config) + ), "policy": actor.config.model_dump(mode="json"), "model": actor.execution_config.model.model_dump(mode="json"), "statistics": actor.interface.selected_columns_statistics, @@ -84,7 +91,10 @@ def identity(self): return hashlib.sha256(self.canonical.encode()).hexdigest() def validate_actor(self, actor, units, field="actor"): - require_equal(self.values, type(self).from_actor(actor, units).values, field) + actual = type(self).from_actor(actor, units).values + if "execution" not in self.values: + actual.pop("execution") + require_equal(self.values, actual, field) @dataclass(frozen=True) @@ -206,6 +216,15 @@ class PhaseSession: def __init__(self, context, *, modules=None, frozen=None): self.context = context + actor = context.actor + if not hasattr(actor, "_phase_trainability"): + actor._phase_trainability = { + name: p.requires_grad for name, p in actor.network.named_parameters() + } + for name, parameter in actor.network.named_parameters(): + parameter.requires_grad_(actor._phase_trainability[name]) + for path in context.phase.freeze: + actor.network.get_submodule(path).requires_grad_(False) self.modules = {"actor": context.actor.network, **(modules or {})} self.networks = {**self.modules, **(frozen or {})} self.configs = context.phase.optimization_configs() diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index b53b93d2..eefa39ba 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -7,6 +7,7 @@ from sequifier.config.policy_infer_config import PolicyRolloutConfig from sequifier.config.rollout_config import InputBinding +from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.values import numeric_scalar from sequifier.rl.collection import PolicyCursor from sequifier.rl.environment import ( @@ -53,6 +54,8 @@ def collect_policy_rollout( ) if not spec.return_conditioned and desired_return is not None: raise ValueError("behavior cloning has no desired-return input") + if spec.return_conditioned and desired_return is None: + desired_return = spec.initial_desired_return returns = ( [numeric_scalar(desired_return, "desired_return")] if spec.return_conditioned @@ -103,12 +106,29 @@ def collect_policy_rollout( successor = ( dict(result.observation) if result.observation is not None else None ) + action_metadata = ( + decision.action if isinstance(decision.action, dict) else {} + ) transitions.append( Transition( episode_id=episode_id, step_id=step_id, observation=dict(current.observation), predictions=decision.domain_action, + action_latents={ + name: action_metadata[action_latent_key(name, spec)] + for name, field in spec.actions.items() + if field.type == "real" + } + if spec.is_joint + else {}, + action_components={ + name: action_metadata[action_component_key(name, spec)] + for name, field in spec.actions.items() + if field.type == "real" + } + if spec.is_joint + else {}, external_inputs=successor or {}, reward=result.reward, successor_observation=successor, @@ -121,14 +141,17 @@ def collect_policy_rollout( break last = transitions[-1] return Trajectory( - format="sequifier.scalar-policy-trajectory", + format="sequifier.policy-trajectory" + if spec.is_joint + else "sequifier.scalar-policy-trajectory", + version=2 if spec.is_joint else 1, episode_id=episode_id, seed=seed, bindings=[ InputBinding(input=name, source="environment", key=name) for name in spec.observation_columns ], - prediction_columns=[spec.action_column], + prediction_columns=spec.action_columns, transitions=transitions, collection_cutoff=not (last.terminated or last.truncated), total_return=math.fsum(step.reward for step in transitions), diff --git a/src/sequifier/rl/scoring.py b/src/sequifier/rl/scoring.py index f5ebf030..2156721e 100644 --- a/src/sequifier/rl/scoring.py +++ b/src/sequifier/rl/scoring.py @@ -4,8 +4,9 @@ import torch +from sequifier.io.actions import action_batch from sequifier.io.batch import SequifierInputs, collate_model_inputs -from sequifier.io.policy_window import PolicyWindowView, action_index +from sequifier.io.policy_window import PolicyWindowView def sequence_log_probabilities(actor, episode, start, end, chunk_size): @@ -20,10 +21,11 @@ def sequence_log_probabilities(actor, episode, start, end, chunk_size): [view.episode_inputs(episode, t, None) for t in positions] ), ).to(actor.device) - actions = torch.tensor( - [action_index(episode.actions[t], actor.config) for t in positions], - dtype=torch.long, + actions = action_batch( + [episode.actions[t] for t in positions], + actor.config, device=actor.device, + encoded=False, ) scores = actor.evaluate_actions(inputs, actions).float() if not torch.isfinite(scores).all(): diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 4de386d1..7f85f6e8 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -13,6 +13,9 @@ def action_nll(actor, batch): batch = batch.to(actor.device) + if actor.config.is_joint: + actions = {name: value[:, -1] for name, value in batch.targets.items()} + return -actor.evaluate_actions(batch.model_inputs, actions).float() labels = batch.targets[action_target(actor.config)][:, -1] ids = torch.tensor(action_global_ids(actor.config), device=actor.device) actions = (labels[:, None] == ids[None, :]).long().argmax(dim=-1) diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index cf4e6691..859a091c 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -1,7 +1,10 @@ """Local categorical policy learning, inference, and environment contracts.""" from sequifier.config.policy_config import ( + ActionFieldConfig, CategoricalPolicyConfig, + PolicyBinding, + PolicyConfig, PolicySchemaConfig, ReturnDefinition, ) @@ -45,7 +48,7 @@ OnlineStepResult, ) from sequifier.rl.learner import run_policy_training -from sequifier.rl.policy import CategoricalPolicy +from sequifier.rl.policy import CategoricalPolicy, Policy from sequifier.rl.ppo_buffer import load_collection_record from sequifier.rl.ppo_evaluation import evaluate_policy from sequifier.rl.run_state import ( @@ -120,3 +123,5 @@ "source_episodes", "split_by_episode", ] + +__all__ += ["ActionFieldConfig", "Policy", "PolicyBinding", "PolicyConfig"] diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 6a8cc732..41f7e3f5 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -150,6 +150,8 @@ def build( config: Any, execution: ExecutionEnvironment, integrations: IntegrationManager, + *, + initial_artifact=None, ) -> TrainingRun: strategy = self._strategy(config, execution) random_manager = RandomStateManager(execution.device) @@ -174,7 +176,18 @@ def build( logger=logger, ) network = built.network - if loaded is None: + if loaded is None and initial_artifact is not None: + from sequifier.artifacts.model_artifact import read_execution_artifact + from sequifier.artifacts.model_export import model_execution_config + + artifact = read_execution_artifact(initial_artifact) + from sequifier.artifacts.model_config import validate_execution_attachment + + validate_execution_attachment( + artifact.model_config.values, model_execution_config(config) + ) + network.load_state_dict(artifact.model_state_dict, strict=True) + if loaded is None and initial_artifact is None: revision = select_revision(config.model.backbone, config.project_root) if revision is not None: load_revision(network.backbone, revision) diff --git a/src/sequifier/templates/train-policy-from-supervised.yaml b/src/sequifier/templates/train-policy-from-supervised.yaml new file mode 100644 index 00000000..25c40414 --- /dev/null +++ b/src/sequifier/templates/train-policy-from-supervised.yaml @@ -0,0 +1,19 @@ +project_root: . +model_name: posttrained +device: cpu +initialize_from: models/supervised/model.pt +policy_interface: main +policy_binding: + actions: + mode: {type: categorical, values: ["off", "on"]} + quantity: {type: real, scale: 0.5} +sources: + demonstrations: + paths: [data/demonstrations.parquet] + format: table +phases: + - name: imitation + algorithm: policy_imitation + sources: [demonstrations] + epochs: 5 +resume: {policy: if_exists, mode: reset_environments} diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 713c892d..f07818be 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -19,6 +19,7 @@ ModelArtifact, load_execution_model, ) +from sequifier.config.learning_run import LearningRunConfig # noqa: E402 from sequifier.config.train_config import PolicyRunConfig # noqa: E402 from sequifier.config.train_config import load_train_config # noqa: E402 from sequifier.config.train_config import ( # noqa: E402 @@ -163,14 +164,22 @@ def _mp_train_worker_wrapper( @beartype def run_training( - config: TrainModel | PolicyRunConfig, + config: TrainModel | PolicyRunConfig | LearningRunConfig, *, integration_specs: tuple[IntegrationSpec, ...] = (), integration_instances: tuple[Any, ...] = (), semantic_optimizer_grouping: bool = False, -) -> None: +) -> Any: """Launch canonical service-composed training locally or across workers.""" + if isinstance(config, LearningRunConfig): + if integration_specs or integration_instances or semantic_optimizer_grouping: + raise ValueError( + "mixed learning plans currently use phase-local native integration settings" + ) + from sequifier.training.coordinator import run_learning_plan + + return run_learning_plan(config) if isinstance(config, PolicyRunConfig): if integration_specs or integration_instances or semantic_optimizer_grouping: raise ValueError( diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index ef706314..78b578f9 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -62,6 +62,10 @@ def _resume_settings( execution = copy.deepcopy(dict(model_config or model_execution_config(config))) execution.pop("embedding_layer_names", None) + execution.setdefault( + "forward_attention", + "bidirectional" if execution["training_objective"] == "bert" else "causal", + ) # Initialization is strict run provenance, compared from resolved training # configs rather than older portable artifacts that omitted it. @@ -88,6 +92,7 @@ def strip_provenance(value: Any) -> Any: # project directory containing the same splits. interface.pop("split_paths", None) interface.setdefault("depth_layouts", {}) + interface.setdefault("prediction_distributions", {}) interface.setdefault("tensor_payload_version", 1) training = config.global_training datasets = { diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py new file mode 100644 index 00000000..70678c0c --- /dev/null +++ b/src/sequifier/training/coordinator.py @@ -0,0 +1,390 @@ +"""Atomic phase coordination around the native supervised and policy runners. + +The envelope commits an immutable incoming artifact before starting a phase. +Native checkpoints own in-phase optimizer, sampler, and environment recovery. +Completing a phase publishes its selected artifact before advancing the envelope. +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import torch + +from sequifier.artifacts.model_artifact import ( + build_model_artifact, + read_execution_artifact, + save_model_artifact, +) +from sequifier.config.learning_run import LearningRunConfig +from sequifier.config.train_config import ( + PolicyRunConfig, + SequifierConfig, + resolve_sequifier_config, +) +from sequifier.fingerprints import fingerprint +from sequifier.io.atomic import atomic_save, write_json_atomic +from sequifier.trajectory.sources import file_checksum + + +@dataclass(frozen=True) +class LearningRunResult: + model_path: str + checkpoint_path: str + report_path: str + phases: tuple[dict, ...] + + +def _supervised_config(config, phase, index, incoming, resume): + from sequifier.artifacts.model_config import resolved_config_from_model_config + from sequifier.config.metadata import load_dataset_metadata + + model = config.model + source = None + forward_attention = None + if incoming is not None: + source, _ = resolved_config_from_model_config( + read_execution_artifact(incoming).model_config.values, device=config.device + ) + model = source.model + forward_attention = source.global_training.forward_attention or ( + "bidirectional" + if source.global_training.training_objective == "bert" + else "causal" + ) + training = config.global_training.model_copy(deep=True) + if training.distributed: + raise ValueError("mixed learning plans currently require local phase execution") + if forward_attention is not None: + training.forward_attention = forward_attention + from sequifier.config.components import ResumeConfig + + training.resume = ResumeConfig(policy="if_exists" if resume else "never") + authored = SequifierConfig.model_validate( + { + "project_root": config.project_root, + "model_name": f"{config.model_name}-phase-{index:03d}-{phase.name}", + "device": config.device, + "seed": config.seed + index, + "model": model.model_dump(), + "global_training": training.model_dump(), + "dataset_training": { + name: value.model_dump() + for name, value in config.dataset_training.items() + }, + "training_plan": {"phases": [phase.model_dump(exclude={"handoff"})]}, + "evaluation": config.evaluation.model_dump() if config.evaluation else None, + "export_pt": True, + "export_onnx": config.export_onnx, + } + ) + metadata = { + f"{name}.{part_name}": load_dataset_metadata( + str(Path(config.project_root) / part.metadata_config_path) + ) + for name, dataset in authored.dataset_training.items() + for part_name, part in dataset.parts.items() + } + resolved = resolve_sequifier_config(authored, metadata) + if source is not None: + # Preserve policy bindings and their model-owned scale modules on return + # to supervised learning. Dataset resolution owns the data/window views. + for dataset in resolved.dataset_training.values(): + previous = source.interface(dataset.model_interface) + dataset.interface.policy = previous.policy + from sequifier.artifacts.model_config import execution_config + + existing = { + dataset.model_interface for dataset in resolved.dataset_training.values() + } + for name in source.interface_names: + if name in existing: + continue + interface = source.interface(name) + placeholder = execution_config( + source.model, {name: interface}, resolved.global_training + ).dataset_training[name] + key = name + while key in resolved.dataset_training: + key += "_unused" + resolved.dataset_training[key] = placeholder.model_copy( + update={"name": key} + ) + if incoming is not None: + from sequifier.artifacts.model_config import validate_execution_attachment + from sequifier.artifacts.model_export import model_execution_config + + validate_execution_attachment( + read_execution_artifact(incoming).model_config.values, + model_execution_config(resolved), + ) + return resolved + + +def _run_supervised(config, phase, index, incoming, destination, resume): + from sequifier.helpers import configure_determinism, configure_logger + from sequifier.integration import IntegrationManager + from sequifier.runtime.builder import RunBuilder + from sequifier.runtime.context import ExecutionEnvironment + from sequifier.training.engine import TrainingEngine + + resolved = _supervised_config(config, phase, index, incoming, resume) + configure_logger(config.project_root, resolved.model_name, 0) + configure_determinism( + config.seed + index, resolved.global_training.enforce_determinism + ) + environment = ExecutionEnvironment( + rank=0, + local_rank=0, + world_size=1, + device=torch.device(config.device), + distributed=False, + ) + integrations = IntegrationManager(rank=0, world_size=1, distributed=False) + run = RunBuilder().build( + resolved, environment, integrations, initial_artifact=incoming + ) + try: + result = TrainingEngine().run(run) + if phase.handoff == "best" and run.state.best_model_state_dict: + run.network.load_state_dict(run.state.best_model_state_dict, strict=True) + save_model_artifact(build_model_artifact(run.network, resolved), destination) + return { + "completion_reason": result.completion_reason, + "optimizer_steps": run.state.optimizer_step, + "handoff": phase.handoff, + } + finally: + run.distributed.finalize() + + +def _policy_config( + config, phase, index, incoming, destination, resume, execution_model=None +): + if incoming is None and execution_model is None: + raise ValueError("a policy phase requires a model execution contract") + interface = phase.model_interface or config.policy_interface + if interface is None: + if execution_model is not None: + interfaces = execution_model["interfaces"] + else: + if incoming is None: + raise ValueError("policy initialization requires a model artifact") + interfaces = read_execution_artifact(incoming).model_config.values[ + "interfaces" + ] + if len(interfaces) != 1: + raise ValueError("select policy_interface or a phase model_interface") + interface = next(iter(interfaces)) + return PolicyRunConfig.model_validate( + { + "project_root": config.project_root, + "model_name": f"{config.model_name}-phase-{index:03d}-{phase.name}", + "device": config.device, + "seed": config.seed + index, + "initialize_from": incoming, + "policy_interface": interface, + "execution_model": execution_model, + "policy_binding": config.policy_bindings[interface].model_dump( + exclude_unset=True + ) + if interface in config.policy_bindings + else None, + "sources": { + name: value.model_dump() for name, value in config.sources.items() + }, + "preferences": config.preferences, + "units": config.units, + "phases": [phase.model_dump()], + "validation_fraction": config.validation_fraction, + "episode_cache_size": config.episode_cache_size, + "resume": { + "policy": "if_exists" if resume else "never", + "mode": config.resume.mode, + }, + "model_path": str(destination), + } + ) + + +def _run_policy(config, phase, index, incoming, destination, resume): + from sequifier.rl.learner import run_policy_training + + run = _policy_config(config, phase, index, incoming, destination, resume) + return run_policy_training(run).phases[0].as_dict() + + +def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: + checkpoint = Path(config.checkpoint_path) + plan = config.model_dump(mode="json", exclude={"resume"}) + identity = fingerprint(plan) + state: dict[str, Any] = { + "artifact_type": "sequifier_learning_run", + "format_version": 1, + "plan": identity, + "completed": [], + "incoming": config.initialize_from, + "incoming_sha256": file_checksum(config.initialize_from) + if config.initialize_from + else None, + "active": False, + } + if config.resume.policy != "never" and checkpoint.is_file(): + state = torch.load(checkpoint, map_location="cpu", weights_only=False) + if ( + state.get("artifact_type") != "sequifier_learning_run" + or state.get("format_version") != 1 + or state.get("plan") != identity + ): + raise ValueError("learning run checkpoint differs from the requested plan") + elif config.resume.policy == "required": + raise FileNotFoundError(checkpoint) + if not 0 <= len(state["completed"]) <= len(config.phases): + raise ValueError("learning run checkpoint exceeds the phase plan") + if ( + state["incoming"] + and file_checksum(state["incoming"]) != state["incoming_sha256"] + ): + raise ValueError("committed incoming model artifact changed") + # Validate model/source/environment contracts for the complete plan first. + from sequifier.artifacts.model_export import model_execution_config + from sequifier.rl.preparation import RunInputs, preflight_environments, prepare_data + + execution_model: dict[str, Any] | None = ( + read_execution_artifact(state["incoming"]).model_config.values + if state["incoming"] + else None + ) + dependencies = {} + for index, phase in enumerate(config.phases): + if phase.algorithm == "supervised": + resolved = _supervised_config( + config, phase, index, state["incoming"], False + ) + execution_model = model_execution_config(resolved) + for dataset in resolved.dataset_training.values(): + for part in dataset.parts.values(): + path = str(Path(config.project_root) / part.metadata_config_path) + dependencies[path] = file_checksum(path) + else: + destination = ( + Path(config.phase_directory) / f"{index:03d}-{phase.name}" / "model.pt" + ) + run = _policy_config( + config, + phase, + index, + state["incoming"], + destination, + False, + execution_model, + ) + inputs = RunInputs.resolve(run) + dependencies.update(inputs.identity) + preflight_environments(run) + prepare_data(run, run.policy, inputs) + if run.execution_model is None or run.policy_interface is None: + raise ValueError("a policy phase has no resolved model interface") + execution_model = dict(run.execution_model) + execution_model["interfaces"] = dict(execution_model["interfaces"]) + selected = dict(execution_model["interfaces"][run.policy_interface]) + selected["policy"] = run.policy.model_dump(mode="json") + execution_model["interfaces"][run.policy_interface] = selected + if "dependencies" in state and state["dependencies"] != dependencies: + raise ValueError("learning plan source identities changed since its checkpoint") + state["dependencies"] = dependencies + protected = {Path(path).resolve() for path in dependencies} + if config.initialize_from: + protected.add(Path(config.initialize_from).resolve()) + outputs = [ + Path(config.checkpoint_path), + Path(config.model_path), + Path(config.report_path), + ] + if len(set(outputs)) != len(outputs): + raise ValueError("learning run output paths must be distinct") + for output in outputs: + if output in protected or any( + output in source.parents or source in output.parents for source in protected + ): + raise ValueError(f"learning output conflicts with an input: {output}") + directory = Path(config.phase_directory) + if any(directory == source or directory in source.parents for source in protected): + raise ValueError("phase_directory overlaps a learning input") + for index in range(len(state["completed"]), len(config.phases)): + phase = config.phases[index] + incoming = state["incoming"] + if incoming and file_checksum(incoming) != state["incoming_sha256"]: + raise ValueError( + "incoming model artifact changed since the committed boundary" + ) + destination = ( + Path(config.phase_directory) / f"{index:03d}-{phase.name}" / "model.pt" + ) + resuming = state["active"] and config.resume.policy != "never" + if not state["active"]: + state["active"] = True + atomic_save(checkpoint, state) + receipt_path = destination.with_name("completed.json") + import json + + receipt = ( + json.loads(receipt_path.read_text()) + if resuming and receipt_path.is_file() + else None + ) + phase_identity = fingerprint( + { + "phase": phase.model_dump(mode="json"), + "incoming": state["incoming_sha256"], + } + ) + if receipt is not None: + if receipt["identity"] != phase_identity or receipt[ + "sha256" + ] != file_checksum(destination): + raise ValueError("completed phase artifact or receipt changed") + report = receipt["report"] + else: + runner = _run_supervised if phase.algorithm == "supervised" else _run_policy + report = runner(config, phase, index, incoming, destination, resuming) + write_json_atomic( + receipt_path, + { + "identity": phase_identity, + "sha256": file_checksum(destination), + "report": report, + }, + ) + state["completed"].append( + { + "name": phase.name, + "algorithm": phase.algorithm, + "model_path": str(destination), + "report": report, + } + ) + state.update( + incoming=str(destination), + incoming_sha256=file_checksum(destination), + active=False, + ) + atomic_save(checkpoint, state) + final = read_execution_artifact(state["incoming"]) + save_model_artifact(final, config.model_path) + write_json_atomic( + config.report_path, + { + "format": "sequifier.learning-run-report", + "version": 1, + "plan_fingerprint": identity, + "phases": state["completed"], + "model_path": config.model_path, + }, + ) + return LearningRunResult( + config.model_path, + config.checkpoint_path, + config.report_path, + tuple(state["completed"]), + ) diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 252b2db6..f5b8661c 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -681,6 +681,7 @@ def flush_metrics( ) raise + run.checkpoints.save(CheckpointRequest("latest", last_identity), run) run.distributed.barrier() last_state = run.distributed.capture_model_state(run.network) best_state = state.best_model_state_dict or last_state diff --git a/src/sequifier/training/loss.py b/src/sequifier/training/loss.py index 9be8562a..5396a742 100644 --- a/src/sequifier/training/loss.py +++ b/src/sequifier/training/loss.py @@ -254,7 +254,33 @@ def calculate( ) target_for_loss = target_for_loss.masked_fill(excluded, 0) criterion = dataset.criteria[target] - if isinstance(criterion, AccurateCrossEntropyLoss): + if isinstance(criterion, torch.nn.GaussianNLLLoss): + from sequifier.io.preprocessing import denormalize_values + from sequifier.model.prediction_distribution import ( + RealPredictionDistribution, + ) + + field = interface.prediction_distributions.get(target) + if field is None and interface.policy is not None: + field = interface.policy.actions.get(target) + if field is None or field.type != "real": + raise ValueError( + f"{target}: GaussianNLLLoss requires a real prediction_distributions entry" + ) + route = network.resolve_interface(dataset.interface_name) + location, observed = ( + logits_for_loss[flat_mask], + target_for_loss[flat_mask], + ) + statistics = interface.selected_columns_statistics.get(target) + if statistics is not None and interface.normalize_real_columns: + location = denormalize_values(location, statistics) + observed = denormalize_values(observed, statistics) + distribution = RealPredictionDistribution( + location, route.prediction_scales.scale(target, field), field + ) + sums[target] = -distribution.log_prob(observed).sum() + elif isinstance(criterion, AccurateCrossEntropyLoss): sums[target] = criterion( logits_for_loss[flat_mask], target_for_loss[flat_mask] ).sum() diff --git a/src/sequifier/training/optimization.py b/src/sequifier/training/optimization.py index e740fca3..4427ae29 100644 --- a/src/sequifier/training/optimization.py +++ b/src/sequifier/training/optimization.py @@ -33,6 +33,19 @@ from sequifier.optimizers.plan import CompositeOptimizer, route_optimizer_parameters +def create_optimizer(spec, parameters, learning_rate, *, parameter_catalog=None): + """Build an optimizer with the same ownership rules for every learner.""" + if isinstance(spec, OptimizerPlan): + if parameter_catalog is None: + raise ValueError("An optimizer plan requires a parameter catalog.") + return CompositeOptimizer( + route_optimizer_parameters(spec, parameter_catalog, parameters) + ) + return get_optimizer_class(spec.name)( + parameters, lr=learning_rate, **spec.arguments + ) + + @dataclass(frozen=True) class SchedulerPolicy: step_on: str @@ -87,21 +100,12 @@ def create( parameter_catalog: ParameterCatalog | None = None, phase_epochs: int | None = None, ) -> "OptimizationRuntime": - if isinstance(training.optimizer, OptimizerPlan): - if parameter_catalog is None: - raise ValueError("An optimizer plan requires a parameter catalog.") - optimizer = CompositeOptimizer( - route_optimizer_parameters( - training.optimizer, parameter_catalog, parameters - ) - ) - else: - optimizer_class = get_optimizer_class(training.optimizer.name) - optimizer = optimizer_class( - parameters, - lr=training.learning_rate, - **training.optimizer.arguments, - ) + optimizer = create_optimizer( + training.optimizer, + parameters, + training.learning_rate, + parameter_catalog=parameter_catalog, + ) scheduler_class = get_scheduler_class(training.scheduler.name) scheduler_arguments = dict(training.scheduler.arguments) if isinstance(training.optimizer, OptimizerPlan): diff --git a/src/sequifier/training_api.py b/src/sequifier/training_api.py index 4e6d8887..ef862776 100644 --- a/src/sequifier/training_api.py +++ b/src/sequifier/training_api.py @@ -1,7 +1,9 @@ """Stable update-aware primitives for training integrations.""" +from sequifier.config.learning_run import LearningRunConfig from sequifier.integration.contexts import StepIdentity from sequifier.integration.controls import TrainingDirective +from sequifier.training.coordinator import LearningRunResult, run_learning_plan from sequifier.training.distributed_strategy import ( DistributedDataParallelStrategy, DistributedStrategy, @@ -21,4 +23,7 @@ "StepIdentity", "StepResult", "TrainingDirective", + "LearningRunConfig", + "LearningRunResult", + "run_learning_plan", ] diff --git a/src/sequifier/trajectory/export.py b/src/sequifier/trajectory/export.py index a821a9ea..33fbf311 100644 --- a/src/sequifier/trajectory/export.py +++ b/src/sequifier/trajectory/export.py @@ -7,6 +7,7 @@ from pathlib import Path from sequifier.config.policy_train_config import PolicyTableColumns +from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.sequence import TransitionColumns from sequifier.trajectory.importers import source_episodes from sequifier.trajectory.sources import file_checksum, source_paths @@ -54,9 +55,38 @@ def import_sequences(config) -> Path: - config.policy.categorical_n_classes.keys() ) action_type = ( - pa.string() if isinstance(config.policy.action_values[0], str) else pa.float64() + pa.struct( + [ + pa.field( + name, + pa.string() + if field.type == "categorical" and isinstance(field.values[0], str) + else pa.float64(), + ) + for name, field in config.policy.actions.items() + ] + + [ + pa.field(action_latent_key(name, config.policy), pa.float64()) + for name, field in config.policy.actions.items() + if field.type == "real" + ] + + [ + pa.field(action_component_key(name, config.policy), pa.int64()) + for name, field in config.policy.actions.items() + if field.type == "real" + ] + ) + if config.policy.is_joint + else ( + pa.string() + if isinstance(config.policy.action_values[0], str) + else pa.float64() + ) ) - types = {name: pa.float64() for name in real | {"__reward"}} + types = { + name: pa.float64() + for name in ({"__reward"} if config.policy.is_joint else real | {"__reward"}) + } types.update( {name: pa.bool_() for name in ("__terminated", "__truncated", "__bootstrap")} ) diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 1f49ed46..40b0b73e 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -5,6 +5,7 @@ from sequifier.config.policy_config import PolicySchemaConfig from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.sequence import SequenceEpisode, TransitionColumns from sequifier.trajectory.schema import Trajectory, validate_trajectory from sequifier.trajectory.sources import ResolvedSequenceSource, resolve_source @@ -17,13 +18,14 @@ def sequence_from_trajectory( ) -> SequenceEpisode: episode = validate_trajectory(trajectory) if ( - episode.format != "sequifier.scalar-policy-trajectory" + episode.format + not in {"sequifier.scalar-policy-trajectory", "sequifier.policy-trajectory"} and action_semantics != "executed_predictions" ): raise ValueError( "forecast sources require explicit action_semantics='executed_predictions'" ) - if episode.prediction_columns != [policy.action_column]: + if episode.prediction_columns != policy.action_columns: raise ValueError("recorded actions do not match the policy action column") steps = episode.transitions return SequenceEpisode( @@ -33,7 +35,22 @@ def sequence_from_trajectory( column: tuple(step.observation[column] for step in steps) for column in episode.seed.observations }, - actions=tuple(step.predictions[policy.action_column] for step in steps), + actions=tuple( + { + **step.predictions, + **{ + action_latent_key(name, policy): value + for name, value in step.action_latents.items() + }, + **{ + action_component_key(name, policy): value + for name, value in step.action_components.items() + }, + } + if policy.is_joint + else step.predictions[policy.action_column] + for step in steps + ), static_inputs=episode.seed.static_inputs, transitions=TransitionColumns( **{ diff --git a/src/sequifier/trajectory/prepared.py b/src/sequifier/trajectory/prepared.py index 2d82029d..a7624045 100644 --- a/src/sequifier/trajectory/prepared.py +++ b/src/sequifier/trajectory/prepared.py @@ -22,14 +22,14 @@ def prepared_table( from sequifier.io.pt_payload import load_pt_payload, tabular_tensor_batch metadata = load_dataset_metadata(source.metadata_config_path) - if metadata.depth_layouts: - raise ValueError( - "policy table import currently supports scalar prepared columns" - ) + from sequifier.config.depth_layout import depth_mask_metadata_key + rows = {} for path in paths: if path.suffix.lower() == ".pt": - batch = load_pt_payload(path, n_classes=metadata.n_classes) + batch = load_pt_payload( + path, layouts=metadata.depth_layouts, n_classes=metadata.n_classes + ) else: data = read_data(str(path), path.suffix.lower().lstrip(".")) batch = tabular_tensor_batch( @@ -55,9 +55,15 @@ def prepared_table( position = int(starts[row]) + offset key = (int(ids[row]), position) values = { - column: tensor[row, offset].item() + column: tensor[row, offset].tolist() for column, tensor in sequences.items() } + values.update( + { + depth_mask_metadata_key(name): mask[row, offset].tolist() + for name, mask in batch.depth_valid_masks.items() + } + ) record = { "sequenceId": key[0], "itemPosition": position, diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index 3742cb35..bb56a437 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -9,6 +9,7 @@ ConfigDict, Field, FiniteFloat, + JsonValue, StrictInt, TypeAdapter, model_validator, @@ -24,9 +25,9 @@ class Record(BaseModel): class SeedHistory(Record): - observations: dict[str, list[DomainValue]] + observations: dict[str, list[JsonValue]] attention_valid_mask: list[bool] - static_inputs: dict[str, DomainValue] + static_inputs: dict[str, JsonValue] @model_validator(mode="after") def validate_window(self): @@ -51,11 +52,13 @@ def validate_window(self): class Transition(Record): episode_id: str step_id: int = Field(ge=0) - observation: dict[str, DomainValue] + observation: dict[str, JsonValue] predictions: dict[str, DomainValue] - external_inputs: dict[str, DomainValue] + external_inputs: dict[str, JsonValue] reward: FiniteFloat - successor_observation: dict[str, DomainValue] | None + action_latents: dict[str, FiniteFloat] = Field(default_factory=dict) + action_components: dict[str, StrictInt] = Field(default_factory=dict) + successor_observation: dict[str, JsonValue] | None terminated: bool truncated: bool bootstrap_allowed: bool @@ -63,9 +66,11 @@ class Transition(Record): class Trajectory(Record): format: Literal[ - "sequifier.numeric-forecast-trajectory", "sequifier.scalar-policy-trajectory" + "sequifier.numeric-forecast-trajectory", + "sequifier.scalar-policy-trajectory", + "sequifier.policy-trajectory", ] = "sequifier.numeric-forecast-trajectory" - version: Literal[1] = 1 + version: Literal[1, 2] = 1 episode_id: str = Field(min_length=1) seed: SeedHistory bindings: list[InputBinding] @@ -108,7 +113,11 @@ def validate_temporal_contract(self): ) if transition.successor_observation is None: if ( - self.format != "sequifier.scalar-policy-trajectory" + self.format + not in { + "sequifier.scalar-policy-trajectory", + "sequifier.policy-trajectory", + } or not transition.terminated or transition.external_inputs ): @@ -157,6 +166,10 @@ def validate_temporal_contract(self): self.total_return, expected_return, rel_tol=1e-12, abs_tol=1e-12 ): raise ValueError("total_return does not equal the sum of rewards") + if self.format == "sequifier.policy-trajectory": + if self.version != 2: + raise ValueError("joint policy trajectories require version 2") + _validate_joint_execution(self) if self.format == "sequifier.scalar-policy-trajectory": if ( sum(self.seed.attention_valid_mask) != 1 @@ -241,3 +254,72 @@ def validate_trajectory(value: Trajectory | dict) -> Trajectory: return Trajectory.model_validate( value.model_dump() if isinstance(value, Trajectory) else value ) + + +def _validate_joint_execution(trajectory): + from sequifier.config.policy_config import CategoricalPolicyConfig + from sequifier.io.actions import encode_action + + execution = trajectory.metadata.get("policy_execution") + if not isinstance(execution, dict): + raise ValueError("policy trajectories require execution metadata") + spec = CategoricalPolicyConfig.model_validate(execution.get("config")) + if trajectory.prediction_columns != spec.action_columns: + raise ValueError("trajectory action fields differ from the binding") + if set(trajectory.seed.observations) != set(spec.observation_columns) or set( + trajectory.seed.static_inputs + ) != set(spec.static_columns): + raise ValueError("trajectory inputs differ from the model binding") + scores = TypeAdapter(list[FiniteFloat]).validate_python( + execution.get("action_log_probabilities"), strict=True + ) + previous = execution.get("previous_action_ids") + if ( + len(scores) != len(trajectory.transitions) + or not isinstance(previous, list) + or len(previous) != len(scores) + ): + raise ValueError("trajectory scores/history must match decision count") + expected = None + for index, step in enumerate(trajectory.transitions): + real_fields = { + name for name, field in spec.actions.items() if field.type == "real" + } + if (set(step.action_latents) | set(step.action_components)) - real_fields: + raise ValueError( + "trajectory transform metadata must name real action fields" + ) + recorded_previous = previous[index] + if isinstance(recorded_previous, dict): + recorded_previous = { + key: value + for key, value in recorded_previous.items() + if key in spec.actions + } + if recorded_previous != expected: + raise ValueError("previous actions must lag executed actions") + expected = encode_action(step.predictions, spec) + if not isinstance(execution.get("deterministic"), bool): + raise ValueError("trajectory must declare its selection mode") + budgets = TypeAdapter(list[FiniteFloat]).validate_python( + execution.get("remaining_returns"), strict=True + ) + if spec.return_conditioned: + if len(budgets) != len(scores) + 1 or budgets[0] != execution.get( + "initial_desired_return" + ): + raise ValueError( + "return budgets must include every successor and the initial budget" + ) + for index, step in enumerate(trajectory.transitions): + if not math.isclose( + budgets[index + 1], + remaining_return(budgets[index], step.reward, spec.returns.gamma), + rel_tol=1e-12, + abs_tol=1e-12, + ): + raise ValueError( + "return budgets violate the recorded reward arithmetic" + ) + elif budgets: + raise ValueError("an unconditioned policy cannot record return budgets") diff --git a/src/sequifier/trajectory/tables.py b/src/sequifier/trajectory/tables.py index f893e2ae..7708783c 100644 --- a/src/sequifier/trajectory/tables.py +++ b/src/sequifier/trajectory/tables.py @@ -10,6 +10,7 @@ from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata from sequifier.config.policy_config import PolicySchemaConfig from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.io.images import ImageStore from sequifier.io.preprocessing import denormalize_values from sequifier.io.sequence import ( SequenceEpisode, @@ -89,6 +90,10 @@ def model_decoder(metadata, action_column, action_values): ) def decode(value, column): + if isinstance(value, (list, tuple)): + return [decode(item, column) for item in value] + if isinstance(value, bool) or isinstance(value, dict): + return value value = domain_scalar(value, column) if metadata is not None: if column in inverse_maps: @@ -98,6 +103,13 @@ def decode(value, column): or value != int(value) or int(value) not in inverse ): + from sequifier.special_tokens import SPECIAL_TOKEN_IDS + + if ( + isinstance(value, int) + and value in SPECIAL_TOKEN_IDS.labels_by_id + ): + return SPECIAL_TOKEN_IDS.labels_by_id[value] raise ValueError(f"{column}: encoded ID has no domain label") value = inverse[int(value)] if column == action_column: @@ -135,6 +147,7 @@ class TableEpisodeReference: metadata: DatasetMetadata | None = None prepared: bool = False static_table: IndexedTable | None = None + image_store: ImageStore | None = None def load(self) -> SequenceEpisode: columns = self.source.columns @@ -154,7 +167,15 @@ def load(self) -> SequenceEpisode: raise ValueError("prepared episode contains no split-owned decisions") first = decisions[columns.item_position][0] previous = data.filter(pl.col(columns.item_position) == first - 1) - if len(previous): + if len(previous) and self.policy.is_joint: + mapping = columns.actions or { + name: name for name in self.policy.action_columns + } + preceding = { + name: decode(previous[column][0], name) + for name, column in mapping.items() + } + elif len(previous): preceding = decode( previous[columns.action or self.policy.action_column][0], columns.action or self.policy.action_column, @@ -163,6 +184,26 @@ def load(self) -> SequenceEpisode: static_data = None if self.static_table is not None: static_data = self.static_table.load(self.sequence_id) + if self.image_store is not None: + import torch + + images, masks = self.image_store.gather( + torch.tensor([int(self.sequence_id)]), + torch.tensor([data[columns.item_position].to_list()]), + torch.ones((1, len(data)), dtype=torch.bool), + ) + for name, value in {**images, **masks}.items(): + if name == "__preconditioning_image": + if static_data is None: + data = data.with_columns( + pl.Series(name, [value[0].tolist()] * len(data)) + ) + else: + static_data = static_data.with_columns( + pl.Series(name, [value[0].tolist()]) + ) + else: + data = data.with_columns(pl.Series(name, value[0].tolist())) return table_episode( data, self.policy, @@ -185,10 +226,25 @@ def table_features(data, policy, columns, static_data): ) ) action_column = columns.action or policy.action_column + action_columns = ( + ( + columns.actions + or ( + {name: name for name in policy.action_columns} + if columns.action is None + else {name: action_column for name in policy.action_columns} + ) + ) + if policy.is_joint + else {policy.action_column: action_column} + ) inputs = set(observations.values()) if static_data is None: inputs.update(static_inputs.values()) - if action_column in inputs or columns.previous_action == action_column: + if ( + set(action_columns.values()) & inputs + or columns.previous_action in action_columns.values() + ): raise ValueError( "current action labels require distinct input/previous-action columns" ) @@ -199,7 +255,9 @@ def table_features(data, policy, columns, static_data): raise ValueError("table mappings must name declared policy features") required = ( inputs - | {columns.item_position, action_column} + | {columns.item_position} + | set(action_columns.values()) + | set(columns.previous_actions.values()) | set(columns.successors.values()) ) required.update( @@ -235,12 +293,46 @@ def table_episode( ) -> SequenceEpisode: """Canonical table-to-episode validation shared by training and inference.""" data = ordered_sequence_rows(data, columns.item_position) + if policy.is_joint and decode is domain_scalar: + + def identity_decode(value, column): + return value + + decode = identity_decode + if data.is_empty(): raise ValueError("sequence source changed while loading") action_column = columns.action or policy.action_column + action_columns = ( + ( + columns.actions + or ( + {name: name for name in policy.action_columns} + if columns.action is None + else {name: action_column for name in policy.action_columns} + ) + ) + if policy.is_joint + else {policy.action_column: action_column} + ) observations, static_inputs = table_features(data, policy, columns, static_data) + def canonical_action(value): + if value is None or not policy.is_joint: + return value + return { + name: entry + for name, entry in value.items() + if name in policy.actions or entry is not None + } + def values(column, decoder=decode): + if ( + policy.is_joint + and policy.input_encoding == "model" + and column in observations.values() + ): + return tuple(data[column].to_list()) return tuple(decoder(value, column) for value in data[column]) def optional_values(column, decoder=decode): @@ -253,17 +345,54 @@ def optional_values(column, decoder=decode): return values(column, decoder) def constant(entries, label): - unique = set(entries) - if len(unique) != 1: + entries = list(entries) + if not entries or any(value != entries[0] for value in entries[1:]): raise ValueError(f"{label} must remain constant within an episode") - return next(iter(unique)) + return entries[0] positions = values(columns.item_position, lambda value, _: value) - actions = values(action_column) + if policy.is_joint: + if set(action_columns) != set(policy.actions): + raise ValueError("table action mappings must cover every action field") + actions = tuple( + canonical_action(dict(row[columns.action])) + if columns.action is not None and not columns.actions + else { + name: decode(row[column], name) + for name, column in action_columns.items() + } + for row in data.iter_rows(named=True) + ) + if columns.previous_actions: + if set(columns.previous_actions) != set(policy.actions): + raise ValueError( + "previous action mappings must cover every action field" + ) + previous = tuple( + {name: row[column] for name, column in columns.previous_actions.items()} + for row in data.iter_rows(named=True) + ) + preceding = ( + None + if all(value is None for value in previous[0].values()) + else previous[0] + ) + if previous[1:] != actions[:-1]: + raise ValueError( + "previous actions must lag recorded actions by one decision" + ) + else: + actions = values(action_column) if columns.previous_action is not None: preceding_actions = values( columns.previous_action, - lambda value, _: None if value is None else decode(value, action_column), + lambda value, _: None + if value is None + else ( + canonical_action(value) + if policy.is_joint + else decode(value, action_column) + ), ) if preceding_actions[1:] != actions[:-1]: raise ValueError( @@ -277,7 +406,12 @@ def constant(entries, label): static_data = data static = { name: constant( - (decode(value, column) for value in static_data[column]), + ( + value + if policy.is_joint and policy.input_encoding == "model" + else decode(value, column) + for value in static_data[column] + ), f"static input {name}", ) for name, column in static_inputs.items() @@ -307,7 +441,9 @@ def constant(entries, label): raise ValueError("a missing successor must mask all observation fields") successors.append( { - name: decode(row[column], observations[name]) + name: row[column] + if policy.is_joint and policy.input_encoding == "model" + else decode(row[column], observations[name]) for name, column in columns.successors.items() } if all(present) @@ -367,9 +503,26 @@ def table_episodes(source, policy, paths, *, prepared=False): if source.metadata_config_path else None ) + image_store = None + if ( + prepared + and policy.is_joint + and metadata is not None + and (metadata.image_sources or metadata.preconditioning_image is not None) + ): + image_store = ImageStore( + metadata.image_sources, metadata.preconditioning_image, source.project_root + ) return [ TableEpisodeReference( - source, policy, identity, table, metadata, prepared, static_table + source, + policy, + identity, + table, + metadata, + prepared, + static_table, + image_store, ) for identity in table.ranges ] From afe3e998226aa4ebb2e5dbf66c9a2647625aba8c Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 10 Nov 2026 16:42:13 +0100 Subject: [PATCH 24/71] clean up config surface --- .pre-commit-config.yaml | 2 +- documentation/config-capability-inventory.md | 648 +++++++++++ documentation/configs/infer.md | 89 +- documentation/configs/train.md | 367 +++--- .../configs/import-policy-sequences.yaml | 20 +- .../example_inputs/configs/infer-policy.yaml | 13 +- .../example_inputs/configs/infer-ppo-toy.yaml | 7 +- .../configs/rollout-ppo-toy.yaml | 19 +- .../example_inputs/configs/train-bc-toy.yaml | 72 +- .../configs/train-iql-from-table.yaml | 111 +- .../example_inputs/configs/train-iql-toy.yaml | 92 +- .../configs/train-multiphase-control.yaml | 197 ++-- .../configs/train-policy-from-table.yaml | 78 +- .../configs/train-return-conditioned.yaml | 88 +- .../example_inputs/iql_toy_control.py | 71 +- .../example_inputs/policy_control_data.py | 2 +- .../example_inputs/ppo_toy_control.py | 16 +- documentation/policy-learning.md | 197 ++-- ...blic-config-surface-implementation-plan.md | 368 ++++++ documentation/rollout.md | 29 +- documentation/supervised-policy-learning.md | 129 +- hooks/consolidate_docs.py | 2 + release-notes/unreleased-phase-ownership.md | 24 + release-notes/unreleased-policy-training.md | 16 +- ...nreleased-supervised-policy-unification.md | 8 +- src/sequifier/artifacts/model_artifact.py | 8 +- src/sequifier/artifacts/model_config.py | 14 +- .../canonical_hyperparameter_search_config.py | 16 +- src/sequifier/config/components.py | 1 + src/sequifier/config/composition.py | 4 +- src/sequifier/config/infer_config.py | 112 +- src/sequifier/config/inference_common.py | 16 + src/sequifier/config/learning_run.py | 97 -- src/sequifier/config/optimizer_config.py | 4 +- src/sequifier/config/policy_config.py | 129 +- src/sequifier/config/policy_infer_config.py | 112 +- src/sequifier/config/policy_optimization.py | 112 +- src/sequifier/config/policy_plan.py | 49 +- src/sequifier/config/rollout_config.py | 13 +- .../config/sequence_import_config.py | 47 +- src/sequifier/config/sources.py | 27 + src/sequifier/config/train_config.py | 863 +++++--------- src/sequifier/config/training_resolution.py | 653 +++++++++++ src/sequifier/consolidated-docs.md | 1034 ++++++++++++----- src/sequifier/helpers.py | 3 +- src/sequifier/infer.py | 13 +- src/sequifier/io/actions.py | 12 - src/sequifier/io/episode_corpus.py | 11 +- src/sequifier/io/episode_features.py | 48 +- src/sequifier/io/folder_dataset.py | 2 +- src/sequifier/io/iql_dataset.py | 2 - src/sequifier/io/policy_dataset.py | 32 +- src/sequifier/io/policy_materialization.py | 4 +- src/sequifier/io/policy_window.py | 175 +-- src/sequifier/io/preference_dataset.py | 10 +- src/sequifier/io/prepared_policy_data.py | 4 +- .../io/sequifier_dataset_from_file.py | 2 +- src/sequifier/make.py | 26 +- src/sequifier/rl/attachment.py | 23 +- src/sequifier/rl/checkpoint.py | 12 +- src/sequifier/rl/collection.py | 6 +- src/sequifier/rl/history.py | 62 +- src/sequifier/rl/inference.py | 119 +- src/sequifier/rl/iql.py | 4 +- src/sequifier/rl/joint_distribution.py | 18 + src/sequifier/rl/learner.py | 99 +- src/sequifier/rl/offline_data.py | 10 +- src/sequifier/rl/policy.py | 184 +-- src/sequifier/rl/ppo_buffer.py | 18 +- src/sequifier/rl/ppo_collector.py | 4 +- src/sequifier/rl/ppo_evaluation.py | 5 +- src/sequifier/rl/preparation.py | 4 +- src/sequifier/rl/references.py | 2 +- src/sequifier/rl/run_checkpoint.py | 8 +- src/sequifier/rl/run_config.py | 35 + src/sequifier/rl/run_state.py | 71 +- src/sequifier/rl/runtime.py | 30 +- src/sequifier/rl/training.py | 8 +- src/sequifier/rl_api.py | 23 +- src/sequifier/runtime/builder.py | 59 +- src/sequifier/sequifier.py | 9 + src/sequifier/templates/infer-policy.yaml | 8 +- src/sequifier/templates/train-dpo.yaml | 89 +- src/sequifier/templates/train-grpo.yaml | 90 +- src/sequifier/templates/train-iql.yaml | 87 +- .../train-policy-from-supervised.yaml | 32 +- src/sequifier/templates/train-policy.yaml | 83 +- src/sequifier/templates/train-ppo.yaml | 102 +- .../templates/train-supervised-to-policy.yaml | 104 ++ src/sequifier/templates/train-supervised.yaml | 57 + src/sequifier/train.py | 35 +- src/sequifier/training/checkpoint_service.py | 8 - src/sequifier/training/coordinator.py | 108 +- src/sequifier/training/engine.py | 24 +- src/sequifier/training/optimization.py | 15 +- src/sequifier/training/runtime.py | 6 +- src/sequifier/training/scheduler_budgets.py | 111 ++ src/sequifier/training_api.py | 5 +- src/sequifier/trajectory/export.py | 69 +- src/sequifier/trajectory/importers.py | 10 +- src/sequifier/trajectory/schema.py | 82 +- src/sequifier/trajectory/tables.py | 132 +-- src/sequifier/visualize_training.py | 2 - 103 files changed, 5376 insertions(+), 3016 deletions(-) create mode 100644 documentation/config-capability-inventory.md create mode 100644 documentation/public-config-surface-implementation-plan.md create mode 100644 release-notes/unreleased-phase-ownership.md create mode 100644 src/sequifier/config/inference_common.py delete mode 100644 src/sequifier/config/learning_run.py create mode 100644 src/sequifier/config/sources.py create mode 100644 src/sequifier/config/training_resolution.py create mode 100644 src/sequifier/rl/run_config.py create mode 100644 src/sequifier/templates/train-supervised-to-policy.yaml create mode 100644 src/sequifier/templates/train-supervised.yaml create mode 100644 src/sequifier/training/scheduler_budgets.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ccabc291..028352ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,5 +64,5 @@ repos: name: Consolidate configuration documentation entry: python hooks/consolidate_docs.py language: python - files: ^(README\.md|documentation/configs/(preprocess|train|infer|hyperparameter-search)\.md|documentation/policy-learning\.md|documentation/commands/visualize-training\.md|documentation/training/multi-gpu-training\.md|src/sequifier/consolidated-docs\.md)$ + files: ^(README\.md|documentation/configs/(preprocess|train|infer|hyperparameter-search)\.md|documentation/(policy-learning|supervised-policy-learning|rollout)\.md|documentation/commands/visualize-training\.md|documentation/training/multi-gpu-training\.md|src/sequifier/consolidated-docs\.md)$ pass_filenames: false diff --git a/documentation/config-capability-inventory.md b/documentation/config-capability-inventory.md new file mode 100644 index 00000000..180d7a12 --- /dev/null +++ b/documentation/config-capability-inventory.md @@ -0,0 +1,648 @@ +# Configuration capability inventory + +Capability baseline: `082225833e1d744e662d094008bcc44268ab868a`. Layout reference: `2d01d407`. + +The phase/route redesign supersedes baseline placements below. Training batching +and optimization belong to phases, inference validates the selected model route, +and rollout declarations use `environment`. See the current +[training](configs/train.md), [inference](configs/infer.md), and +[phase/route tests](../../sequifier-test/local/unit/test_phase_route_contracts.py). + +This inventory records every directly declared field in the baseline run, policy, inference, source, rollout and recovery models. Shared component behavior remains owned by the existing component models. Internal facts have resolved owners; duplicate routing spellings have one authored owner. Test references below are acceptance coverage, not support for old config syntax. + +## `train_config.GlobalTrainingSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `read_format` | global_training.read_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `training_objective` | global_training.training_objective | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `forward_attention` | global_training.forward_attention | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `context_length` | global_training.context_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `target_offset` | global_training.target_offset | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `window_stride` | global_training.window_stride | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `inference_batch_size` | global_training.inference_batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `batch_size` | training_plan.phases[].batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `accumulation_steps` | training_plan.phases[].accumulation_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `learning_rate` | training_plan.phases[].optimization..learning_rate | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `optimizer` | training_plan.phases[].optimization..optimizer | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `scheduler` | training_plan.phases[].optimization..scheduler | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `scheduler_step_on` | training_plan.phases[].optimization..scheduler_step_on | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `reset_optimization_on_phase` | training_plan.phases[].optimizer_state (reset by default; validated continuation) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `gradient_clip` | training_plan.phases[].optimization..gradient_clip | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `bert_spec` | global_training.bert_spec | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `next_occurrence_config` | global_training.next_occurrence_config | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `device_max_concat_length` | global_training.device_max_concat_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `log_interval` | global_training.log_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `early_stopping_epochs` | global_training.early_stopping_epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `save_interval_epochs` | global_training.save_interval_epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `save_latest_interval_minutes` | global_training.save_latest_interval_minutes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `save_interval_minutes` | global_training.save_interval_minutes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `save_interval_batches` | global_training.save_interval_batches | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `save_interval_val_loss` | global_training.save_interval_val_loss | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `calculate_validation_loss_on_initialization` | global_training.calculate_validation_loss_on_initialization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `resume` | global_training.resume | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `enforce_determinism` | global_training.enforce_determinism | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `distributed` | global_training.distributed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `load_full_data_to_ram` | global_training.load_full_data_to_ram | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_ram_gb` | global_training.max_ram_gb | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `world_size` | global_training.world_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `num_workers` | global_training.num_workers | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `backend` | global_training.backend | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `layer_type_dtypes` | global_training.layer_type_dtypes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `layer_autocast` | global_training.layer_autocast | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `data_parallelism` | global_training.data_parallelism | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `fsdp_cpu_offload` | global_training.fsdp_cpu_offload | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `torch_compile` | global_training.torch_compile | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `float32_matmul_precision` | global_training.float32_matmul_precision | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.ModelInterfaceSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `input_columns` | model.interfaces..input_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `policy` | policy_bindings. | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `prediction_distributions` | model.interfaces..prediction_distributions | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `static_conditioning` | model.interfaces..static_conditioning | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `preconditioning_image_encoder` | model.interfaces..preconditioning_image_encoder | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `target_columns` | model.interfaces..target_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `categorical_hashing` | model.interfaces..categorical_hashing | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `categorical_decoder_special_tokens` | model.interfaces..categorical_decoder_special_tokens | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `feature_layout` | model.interfaces..feature_layout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `ingestion` | model.interfaces..ingestion | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `decoder` | model.interfaces..decoder | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.ModelSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `backbone` | backbone | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `interfaces` | interfaces | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## Policy run fields (authored through `SequifierConfig`) + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_name` | model_name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `enforce_determinism` | global_training.enforce_determinism | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model` | model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `policy_binding` | policy_bindings. | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `execution_model` | resolved model execution | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `policy_interface` | dataset/phase.model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `context_length` | global_training.context_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `units` | policy_bindings..units | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `initialize_from` | initialize_from | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `normalization` | global_training.normalization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `sources` | dataset_training..parts. (sequence) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `preferences` | dataset_training..parts. (preferences) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `phases` | training_plan.phases | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `validation_fraction` | global_training.validation_fraction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `episode_cache_size` | global_training.episode_cache_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `resume` | global_training.resume | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_path` | model_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `report_path` | report_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `checkpoint_path` | global_training.resume.checkpoint_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `phase_directory` | phase_directory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.DatasetPartSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `metadata_config_path` | metadata_config_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `image_path` | image_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `preconditioning_image_path` | preconditioning_image_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `file_order` | file_order | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.DatasetFreezingSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `backbone` | backbone | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `ingestion` | ingestion | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `ingestion_adapter` | ingestion_adapter | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `decoder` | decoder | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.DatasetTrainingSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `model_interface` | model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `parts` | parts | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `criterion` | criterion | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `class_weights` | class_weights | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `loss_weights` | loss_weights | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `class_share_log_columns` | class_share_log_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `freeze` | freeze | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.TrainingSourceSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `weight` | weight | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `batches_per_selection` | batches_per_selection | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.TrainingPhaseSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `algorithm` | algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `name` | name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `epochs` | epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `mode` | mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `selection` | selection | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `sources` | sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.TrainingPlanModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `curriculum_training` | curriculum_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `curriculum_column` | curriculum_column | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `phases` | phases | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.EvaluationMonitorSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `metric` | metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `mode` | mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.EvaluationSpecModel` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `sources` | sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `monitor` | monitor | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `train_config.SequifierConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_name` | model_name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `global_training` | global_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model` | model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `dataset_training` | dataset_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `training_plan` | training_plan | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `evaluation` | evaluation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `export_generative_model` | export_generative_model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `export_embedding_model` | export_embedding_model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `embedding_layer_names` | embedding_layer_names | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `export_onnx` | export_onnx | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `export_pt` | export_pt | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `export_with_dropout` | export_with_dropout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## Mixed run fields (authored through `SequifierConfig`) + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `run_type` | removed dispatch hint; phase algorithms determine execution | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_name` | model_name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model` | model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `initialize_from` | initialize_from | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `global_training` | global_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `dataset_training` | dataset_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `evaluation` | evaluation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `policy_interface` | dataset/phase.model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `policy_bindings` | policy_bindings | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `sources` | dataset_training..parts. (sequence) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `preferences` | dataset_training..parts. (preferences) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `units` | policy_bindings..units | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `phases` | training_plan.phases | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `resume` | global_training.resume | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `validation_fraction` | global_training.validation_fraction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `episode_cache_size` | global_training.episode_cache_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `checkpoint_path` | global_training.resume.checkpoint_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_path` | model_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `report_path` | report_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `phase_directory` | phase_directory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `export_onnx` | export_onnx | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_config.ReturnDefinition` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `gamma` | gamma | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `scale` | scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `finite_horizon` | finite_horizon | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_config.ActionFieldConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `type` | type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `values` | values | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `scale` | scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `learn_scale` | learn_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `min_scale` | min_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_scale` | max_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `lower` | lower | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `upper` | upper | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `active_when` | active_when | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `inactive_value` | inactive_value | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `atoms` | atoms | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_config.PolicySchemaConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `alignment` | resolved from explicit model inputs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `observation_columns` | model input columns / resolved binding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `static_columns` | model static conditioning / resolved binding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `categorical_columns` | metadata types / resolved binding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `observation_n_classes` | metadata.n_classes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `static_n_classes` | metadata.n_classes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `id_maps` | metadata.id_maps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `categorical_encoding` | resolved fixed codec convention | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `action_column` | named actions key | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `action_values` | actions..values | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `actions` | policy_bindings..actions | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `action_order` | resolved decoder/activity dependency order | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `decision_position` | policy_bindings..decision_position | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `previous_action_inputs` | policy_bindings..previous_action_inputs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `initial_inputs` | policy_bindings..initial_inputs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `input_schema` | resolved ExecutionSchema | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `input_statistics` | metadata / resolved input statistics | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `input_encoding` | dataset part encoding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `return_conditioned` | policy_bindings..return_conditioned | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `initial_desired_return` | policy_bindings..initial_desired_return | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `returns` | policy_bindings..returns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_config.CategoricalPolicyConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `context_length` | resolved model context | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `interface_name` | resolved interface name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.PhaseConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `name` | training_plan.phases[].name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `handoff` | training_plan.phases[].handoff | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_interface` | training_plan.phases[].model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `freeze` | training_plan.phases[].freeze | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `representation` | resolved binding identity | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `optimization` | training_plan.phases[].optimization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `evaluation_interval` | training_plan.phases[].evaluation_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `checkpoint_interval` | training_plan.phases[].checkpoint_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.BatchedPhaseConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `batch_size` | training_plan.phases[].batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `accumulation_steps` | training_plan.phases[].accumulation_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.OfflinePhaseConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `sources` | training_plan.phases[].sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `validation_sources` | training_plan.phases[].validation_sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.PolicyImitationPhaseConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `epochs` | training_plan.phases[].epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `monitor` | training_plan.phases[].monitor | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `direction` | training_plan.phases[].direction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.IQLPhaseConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `monitor` | training_plan.phases[].monitor | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `direction` | training_plan.phases[].direction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `name` | training_plan.phases[].name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `updates` | training_plan.phases[].updates | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `batch_size` | training_plan.phases[].batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `gamma` | training_plan.phases[].gamma | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `expectile` | training_plan.phases[].expectile | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `inverse_temperature` | training_plan.phases[].inverse_temperature | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_weight` | training_plan.phases[].max_weight | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `target_update_rate` | training_plan.phases[].target_update_rate | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `evaluation_interval` | training_plan.phases[].evaluation_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `checkpoint_interval` | training_plan.phases[].checkpoint_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.OnlineEnvironmentConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `factory` | factory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `config` | config | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `protocol_version` | fixed by adapter type; checked against implementation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `implementation_id` | implementation_id | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `units` | units | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.FrozenNormalizationConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `statistics` | statistics | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.EvaluationEpisodesConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `environment` | environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seeds` | seeds | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_episode_steps` | max_episode_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `timeout_seconds` | timeout_seconds | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `selection` | selection | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.OnlineEvaluationConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `metric` | evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `direction` | evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.PolicyResumeConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `policy` | global_training.resume.policy | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `mode` | global_training.resume.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.PPOPhaseConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `name` | training_plan.phases[].name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `environment` | training_plan.phases[].environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `evaluation` | training_plan.phases[].evaluation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `normalization` | training_plan.phases[].normalization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed_policy` | training_plan.phases[].seed_policy | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `total_env_steps` | training_plan.phases[].total_env_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `rollout_length` | training_plan.phases[].rollout_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `allow_partial_rollout` | training_plan.phases[].allow_partial_rollout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `optimization_epochs` | training_plan.phases[].optimization_epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `minibatch_size` | training_plan.phases[].minibatch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `accumulation_steps` | training_plan.phases[].accumulation_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `gamma` | training_plan.phases[].gamma | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `gae_lambda` | training_plan.phases[].gae_lambda | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `clip_epsilon` | training_plan.phases[].clip_epsilon | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `value_clip` | training_plan.phases[].value_clip | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `value_coefficient` | training_plan.phases[].value_coefficient | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `entropy_coefficient` | training_plan.phases[].entropy_coefficient | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `target_kl` | training_plan.phases[].target_kl | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `normalize_advantages` | training_plan.phases[].normalize_advantages | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `reward_scale` | training_plan.phases[].reward_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `archive_directory` | training_plan.phases[].archive_directory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `archive_scores` | training_plan.phases[].archive_scores | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_buffer_bytes` | training_plan.phases[].max_buffer_bytes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.ReferenceConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `path` | path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.DPOPhaseConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `preferences` | training_plan.phases[].preferences | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `validation_preferences` | training_plan.phases[].validation_preferences | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `updates` | training_plan.phases[].updates | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `beta` | training_plan.phases[].beta | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `scoring_chunk_size` | training_plan.phases[].scoring_chunk_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `reference` | training_plan.phases[].reference | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `monitor` | training_plan.phases[].monitor | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `direction` | training_plan.phases[].direction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.GRPOEvaluationConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `metric` | evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `direction` | evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_plan.GRPOPhaseConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `scoring_chunk_size` | training_plan.phases[].scoring_chunk_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `environment` | training_plan.phases[].environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `evaluation` | training_plan.phases[].evaluation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `normalization` | training_plan.phases[].normalization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `reference` | training_plan.phases[].reference | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `total_groups` | training_plan.phases[].total_groups | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `groups_per_update` | training_plan.phases[].groups_per_update | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `group_size` | training_plan.phases[].group_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_steps` | training_plan.phases[].max_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_env_steps` | training_plan.phases[].max_env_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_buffer_bytes` | training_plan.phases[].max_buffer_bytes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `optimization_epochs` | training_plan.phases[].optimization_epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `clip_epsilon` | training_plan.phases[].clip_epsilon | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `advantage_epsilon` | training_plan.phases[].advantage_epsilon | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `kl_coefficient` | training_plan.phases[].kl_coefficient | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `old_policy_target_kl` | training_plan.phases[].old_policy_target_kl | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `reward_scale` | training_plan.phases[].reward_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `archive_directory` | training_plan.phases[].archive_directory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_train_config.PolicyTableColumns` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `sequence_id` | dataset part columns.sequence_id | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `item_position` | dataset part columns.item_position | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `observations` | dataset part columns.observations | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `static_inputs` | dataset part columns.static_inputs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `action` | dataset part columns.action | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `actions` | dataset part columns.actions | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `previous_actions` | dataset part columns.previous_actions | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `reward` | dataset part columns.reward | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `terminated` | dataset part columns.terminated | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `truncated` | dataset part columns.truncated | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `previous_action` | dataset part columns.previous_action | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `successors` | dataset part columns.successors | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `bootstrap_allowed` | dataset part columns.bootstrap_allowed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `group` | dataset part columns.group | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `episode_id` | dataset part columns.episode_id | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_train_config.SequenceSourceConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `project_root` | resolved root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `units` | dataset_training..parts..units | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `paths` | dataset_training..parts..paths | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `manifest` | dataset_training..parts..manifest | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `format` | dataset_training..parts..format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `columns` | dataset_training..parts..columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `episode_end` | dataset_training..parts..episode_end | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `episode_namespace` | dataset_training..parts..episode_namespace | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `metadata_config_path` | dataset_training..parts..metadata_config_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `encoding` | dataset_training..parts..encoding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `static_data_path` | dataset_training..parts..static_data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `split_index` | dataset_training..parts..split_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `action_semantics` | dataset_training..parts..action_semantics | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_infer_config.PolicyRolloutConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `environment` | rollout.environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `trajectory_path` | rollout.trajectory_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed_index` | rollout.seed_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_episode_steps` | rollout.max_episode_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_infer_config.PolicyInferenceConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_path` | model_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `execution_mode` | model_type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_type` | model_type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_interface` | model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `data_path` | data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `read_format` | read_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `write_format` | write_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `output_path` | output_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `output_probabilities` | output_probabilities | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `inference_batch_size` | inference_batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `enforce_determinism` | runtime deterministic and policy selection | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `desired_return` | desired_return | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `desired_return_column` | desired_return_column | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `previous_action_column` | previous_action_columns. | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `previous_action_columns` | previous_action_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `sequence_id_column` | sequence_id_column | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `item_position_column` | item_position_column | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `rollout` | rollout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `autoregressive` | explicit rollout for environment interaction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `generation_steps` | rollout.max_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `rollout_config.InputBinding` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `input` | input | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `key` | key | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `rollout_config.AdapterConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `factory` | factory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `config` | config | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `protocol_version` | fixed by adapter type; checked against implementation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `rollout_config.RolloutConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `mode` | rollout.type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `environment` | rollout.environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `bindings` | rollout.bindings | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `trajectory_path` | rollout.trajectory_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed_index` | rollout.seed_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `max_episode_steps` | rollout.max_episode_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `sequence_import_config.SequenceImportConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `operation` | sequence import operation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `project_root` | sequence import project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `source` | sequence import source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `policy` | policy_binding + source/model metadata | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `output_path` | sequence import output_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `episodes_per_shard` | sequence import episodes_per_shard | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `policy_optimization.OptimizationConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `optimizer` | optimizer | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `scheduler` | scheduler | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `learning_rate` | learning_rate | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `gradient_clip` | gradient_clip | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `infer_config._InferenceConfigBase` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `execution_mode` | execution_mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `preprocessing_data_path` | preprocessing_data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `metadata_config_path` | metadata_config_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_path` | model_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_type` | model_type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `training_objective` | training_objective | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `data_path` | data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `training_config_path` | training_config_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `dataset` | dataset | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `part` | part | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `model_interface` | model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `read_format` | read_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `write_format` | write_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `input_columns` | input_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `static_columns` | static_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `static_features_path` | static_features_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `image_path` | image_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `preconditioning_image_path` | preconditioning_image_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `target_columns` | target_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `column_data_types` | column_data_types | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `target_column_types` | target_column_types | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `enforce_determinism` | deterministic | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `output_probabilities` | output_probabilities | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `decode_categories` | decode_categories | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `context_length` | context_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `target_offset` | target_offset | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `window_stride` | window_stride | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `prediction_length` | prediction_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `decoder_support` | decoder_support | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `prediction_alignment_steps` | prediction_alignment_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `inference_batch_size` | inference_batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `sample_from_distribution_columns` | sample_from_distribution_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `infer_with_dropout` | infer_with_dropout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `rollout` | rollout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `autoregressive` | autoregressive | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `generation_steps` | generation_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `infer_config.InferenceConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `data_path` | data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `column_data_types` | column_data_types | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `target_column_types` | target_column_types | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## `components.ResumeConfig` + +| Baseline field | New owner / behavior | Acceptance coverage | +| --- | --- | --- | +| `policy` | global_training.resume.policy | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `checkpoint_path` | global_training.resume.checkpoint_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | + +## Operation and combination coverage + +In addition to field coverage, acceptance tests exercise same-design recovery, native supervised execution and initialization, mixed supervised/imitation/supervised handoff, single categorical and return-conditioned policies, IQL/PPO/DPO/GRPO updates, raw/manifest/prepared sources, explicit and automatic holdouts, custom inference outputs and probabilities, and both environment protocols. Model component, distribution, HPO and source-import cases are covered in their focused suites. + +Completion requires all referenced tests to pass and every supported baseline combination to retain an execution path. Rejection of already supported combinations is a regression, even if individual fields still parse. diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index ddb52701..c9b0d0a1 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -117,7 +117,7 @@ fragments or define the same field twice. | --- | --- | --- | | `model_path` | Required | Model path, or a list of compatible model paths. | | `model_type` | Required | `generative` or `embedding`. | -| `device` | Required | ONNX: `cpu` or `cuda`; PT also supports `mps`. ONNX CUDA requires an installed runtime exposing `CUDAExecutionProvider`. | +| `device` | `cpu` | ONNX: `cpu` or `cuda`; PT also supports `mps`. ONNX CUDA requires an installed runtime exposing `CUDAExecutionProvider`. | | `project_root` | Required | Base for project paths; normally `.`. | | `data_path` | Metadata test/last split | Input file or folder. Usually required with artifact-only inference because artifacts do not store split paths. | | `preprocessing_data_path` | `null` | Derives the generated metadata path. | @@ -168,7 +168,7 @@ stored folder window if the remaining layout and semantics match. | `decode_categories` | `true` | Decode categorical predictions to their original values. | | `sample_from_distribution_columns` | `null` | Sample these categorical targets instead of using argmax. | | `infer_with_dropout` | `false` | Enable dropout at inference; ONNX also requires dropout-preserving export. | -| `deterministic` | `false` | Request deterministic PyTorch algorithms. | +| `enforce_determinism` | `false` | Request deterministic PyTorch algorithms. | | `seed` | `1010` | Random seed. | | `autoregressive` | `false` | Feed predictions back for multi-step generation. | | `generation_steps` | `null` | Required positive step count when autoregressive is enabled. | @@ -190,10 +190,12 @@ its stored windows rather than this first-window reduction. ## CLI overrides `--data-path`, `--input-columns`, `--metadata-config-path`, `--model-path`, -`--seed`, `--dataset`, `--part`, and `--model-interface` override YAML values. +`--seed`, `--dataset`, `--part`, `--model-interface`, `--model-type`, +`--enforce-determinism`, and `--action-selection` override YAML values. `--randomize` takes precedence over `--seed`. `--skip-metadata` skips external metadata loading; inline metadata, a selected training route, or a self-describing -PT artifact must then provide the required metadata. +PT artifact must then provide the required metadata. `--skip-metadata` is +forecast-only and is rejected on policy inference. ## Outputs @@ -267,13 +269,72 @@ ORT installation. Outputs stay indexed by outer item coordinates. ## Policy model interfaces -`sequifier infer` recognizes a policy interface in a portable PyTorch actor. -Use the selected actor export rather than the policy run checkpoint. Use raw -CSV/Parquet observation rows and optional recorded action histories; alignment, vocabulary, and normalization come from the artifact. -`model_type: generative` and `model_type: policy` are accepted. Environment -execution uses a policy `rollout` section with `environment` (a version-3 adapter -supporting `seeded_reset`), `trajectory_path`, `seed_index`, and optional -`max_episode_steps`. Set `autoregressive: true` and `generation_steps`; policy -rollout does not configure forecast bindings. See the -[policy guide](../policy-learning.md#python-and-inference) and -[inference example](../example_inputs/configs/infer-policy.yaml). +`model_type` selects the authored route before artifact resolution. `generative` +and `embedding` use forecast schemas; `policy` uses `PolicyInferenceConfig` +directly. Unknown and cross-route fields fail validation even when set to their +default values. CLI overrides and Python configuration loaders follow the same +selection rules. + +Policy inference requires one portable PT artifact and one unambiguous policy +interface. Supply raw CSV/Parquet observation rows; history alignment, vocabulary +and normalization come from that artifact. Policy supports the common model path, +interface, data path, formats, device, seed, `enforce_determinism`, batch size fields, plus: + +| Policy field | Default / meaning | +| --- | --- | +| `output_path` | null; write policy predictions here instead of the generated output path | +| `action_selection` | `deterministic`; choose the policy mode. `stochastic` samples. | +| `desired_return` | null; fixed return budget for a return-conditioned policy | +| `desired_return_column` | `desired_return`; per-row budget when no fixed budget is supplied | +| `previous_action_columns` | {}; input-column mapping for recorded previous actions | +| `sequence_id_column`, `item_position_column` | `sequenceId`, `itemPosition`; order raw history rows | +| `output_probabilities` | false; include policy distribution outputs | +| `rollout` | null; execute the policy in a version-3 online environment | + +Policy assertions may specify `metadata_config_path`, `input_columns`, +`static_columns`, `target_columns`, `column_data_types`, `target_column_types`, +`training_objective`, `context_length`, `target_offset`, `prediction_length`, and +`decoder_support`. Every supplied assertion must match the selected artifact. +Forecast fields `sample_from_distribution_columns`, `infer_with_dropout`, +`decode_categories`, dataset/part selection, image path overrides, +`autoregressive`, and `generation_steps` are rejected on policy requests. +Policy-only return/action/history options and the policy-only `output_path` +are rejected on forecast requests. Embedding requests also reject generative +sampling, category decoding, probability output, autoregression and rollout fields; +dropout remains supported for embeddings. + +```yaml +project_root: . +model_type: policy +model_path: models/policy/model.pt +model_interface: actor +data_path: observations.csv +read_format: csv +device: cpu +seed: 1010 +enforce_determinism: true +action_selection: stochastic +rollout: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + units: {} + max_steps: 8 + seed_index: 0 + trajectory_path: outputs/trajectory.json +``` + +Rollout schemas are selected by the inference route; do not declare `rollout.type`. +Forecast rollout requires `bindings` and a version-1 environment. Policy rollout +rejects bindings and requires the distinct version-3 environment protocol, +including implementation identity and matching units. Both use `max_steps` and +optional `max_episode_steps`. `seed_index` counts prepared windows in forecast +rollout and raw input rows in policy rollout, each zero-based in input order. + +`enforce_determinism` controls deterministic execution, independently of action +selection and seed. A seeded stochastic policy can reproduce its sampled sequence +under the same supported runtime, device and input ordering. Online evaluation +uses `action_selection` with the same meaning; its execution determinism is the +training run's `global_training.enforce_determinism`. See the +[environment protocols](../rollout.md) and [policy guide](../policy-learning.md). diff --git a/documentation/configs/train.md b/documentation/configs/train.md index d6d9b4f9..d451d4e7 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -5,7 +5,7 @@ IQL and online PPO through the same commands, see [policy learning](../policy-le `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated -adapter, and a decoder. Datasets own data and training policy; `model` +adapter, and a decoder. Phases own batching and optimization; datasets own data and losses; `model` owns architecture. ```console @@ -22,40 +22,60 @@ project_root: . model_name: event-model device: cuda seed: 1010 - global_training: read_format: parquet training_objective: causal context_length: 128 inference_batch_size: 256 - batch_size: 64 - learning_rate: 0.0001 - model: backbone: architecture: dim_model: 128 max_context_length: 512 num_layers: 6 - attention: {type: mha, n_heads: 8, n_kv_heads: 8, output_projection: true} - feed_forward: {dim: 512, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: rope, theta: 10000} + attention: + type: mha + n_heads: 8 + n_kv_heads: 8 + output_projection: true + feed_forward: + dim: 512 + activation: swiglu + normalization: + type: rmsnorm + norm_first: true + position_encoding: + type: rope + theta: 10000 dropout: 0.1 shared_layer_groups: [] interface: - input_columns: [event] - target_columns: [event] - ingestion: {type: embedding, output_dim: 128} - decoder: {type: linear, prediction_length: 1, support: 1} - + input_columns: + - event + target_columns: + - event + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 + support: 1 dataset: - part: {metadata_config_path: configs/metadata_configs/events.json} - criterion: {event: CrossEntropyLoss} - + part: + metadata_config_path: configs/metadata_configs/events.json + criterion: + event: CrossEntropyLoss training_plan: - epochs: 5 - + phases: + - epochs: 5 + mode: sequential + name: supervised + algorithm: supervised + batch_size: 64 + optimization: + model: + learning_rate: 0.0001 evaluation: true ``` @@ -109,7 +129,6 @@ project_root: . model_name: event-model device: cuda seed: 1010 - global_training: read_format: parquet training_objective: causal @@ -117,62 +136,108 @@ global_training: target_offset: 1 window_stride: 1 inference_batch_size: 256 - batch_size: 64 - accumulation_steps: 4 - learning_rate: 0.0001 - optimizer: {name: AdamW, weight_decay: 0.01} - scheduler: {name: StepLR, step_size: 1, gamma: 0.99} - scheduler_step_on: epoch - reset_optimization_on_phase: true - gradient_clip: 1.0 save_interval_epochs: 1 - model: backbone: architecture: dim_model: 128 max_context_length: 512 num_layers: 6 - attention: {type: mha, n_heads: 8, n_kv_heads: 8, output_projection: true} - feed_forward: {dim: 512, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: rope, theta: 10000} + attention: + type: mha + n_heads: 8 + n_kv_heads: 8 + output_projection: true + feed_forward: + dim: 512 + activation: swiglu + normalization: + type: rmsnorm + norm_first: true + position_encoding: + type: rope + theta: 10000 dropout: 0.1 shared_layer_groups: [] interfaces: event_prediction: - input_columns: [event] - target_columns: [event] - categorical_decoder_special_tokens: {event: [other]} - ingestion: {type: embedding, output_dim: 128} - decoder: {type: linear, prediction_length: 1, support: 1} - + input_columns: + - event + target_columns: + - event + categorical_decoder_special_tokens: + event: + - other + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 + support: 1 dataset_training: events: model_interface: event_prediction parts: - original: {metadata_config_path: configs/metadata_configs/events.json} - increment: {metadata_config_path: configs/metadata_configs/events-increment.json} - criterion: {event: CrossEntropyLoss} - loss_weights: {event: 1.0} + original: + metadata_config_path: configs/metadata_configs/events.json + increment: + metadata_config_path: configs/metadata_configs/events-increment.json + criterion: + event: CrossEntropyLoss + loss_weights: + event: 1.0 freeze: - backbone: {freeze: [attention.qkv]} - + backbone: + freeze: + - attention.qkv training_plan: phases: - - name: incremental_finetuning - epochs: 2 - mode: sequential - sources: [{source: events.increment}] - - name: complete_retraining - epochs: 5 - mode: interleaved - selection: round_robin - sources: [{source: events, batches_per_selection: 4}] - + - name: incremental_finetuning + epochs: 2 + mode: sequential + sources: + - source: events.increment + batch_size: 64 + accumulation_steps: 4 + optimization: + model: + learning_rate: 0.0001 + optimizer: + name: AdamW + weight_decay: 0.01 + scheduler: + name: StepLR + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip: 1.0 + optimizer_state: reset + - name: complete_retraining + epochs: 5 + mode: interleaved + selection: round_robin + sources: + - source: events + batches_per_selection: 4 + batch_size: 64 + accumulation_steps: 4 + optimization: + model: + learning_rate: 0.0001 + optimizer: + name: AdamW + weight_decay: 0.01 + scheduler: + name: StepLR + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip: 1.0 + optimizer_state: reset evaluation: - sources: [{source: events}] - + sources: + - source: events export_generative_model: true export_embedding_model: false export_onnx: true @@ -336,27 +401,38 @@ target in each dataset must have a positive weight. ## Optimizer plans -The single-optimizer form above continues to apply one optimizer to all -parameters and uses `global_training.learning_rate`. To select parameters or +The single-optimizer form applies one optimizer to all +parameters and uses the role’s `learning_rate`. To select parameters or use multiple optimizers, replace it with an ordered plan. Each group has its -own learning rate; omit the run-wide `learning_rate` in this form: +own learning rate; use each group’s `learning_rate` in this form: ```yaml -global_training: - optimizer: - groups: - - id: hidden_matrices - select: - semantic_groups: ["attention.*", "feed_forward.*"] - component: backbone - parameter_kind: weight - ndim: 2 - optimizer: {name: Muon, momentum: 0.95} - learning_rate: 0.02 - - id: remainder - select: otherwise - optimizer: {name: AdamW, weight_decay: 0.01} - learning_rate: 0.0003 +training_plan: + phases: + - name: supervised + algorithm: supervised + optimization: + model: + optimizer: + groups: + - id: hidden_matrices + select: + semantic_groups: + - attention.* + - feed_forward.* + component: backbone + parameter_kind: weight + ndim: 2 + optimizer: + name: Muon + momentum: 0.95 + learning_rate: 0.02 + - id: remainder + select: otherwise + optimizer: + name: AdamW + weight_decay: 0.01 + learning_rate: 0.0003 ``` Each supplied selector condition must match. `semantic_groups` matches the @@ -388,38 +464,83 @@ valid with `CyclicLR`. ## Optimization across phases -By default, every training phase starts with a new optimizer, scheduler, and -gradient scaler. Model weights and global progress counters continue across -phase boundaries, while optimizer moments and scheduler position do not. Set -`global_training.reset_optimization_on_phase: false` to keep the optimizer, -scheduler, and gradient scaler operating continuously across all phases. A -checkpoint restores the active optimization lifecycle exactly in either mode. - -For an epoch-stepped scheduler in reset mode, do not provide -`global_training.scheduler.total_steps` because Sequifier owns each phase's -scheduler duration. When a scheduler such as `OneCycleLR` accepts `total_steps`, -Sequifier sets it to the active phase's `epochs` in reset mode, or the sum of all -phase epochs in continuous mode. A continuous epoch-stepped scheduler may still -provide `total_steps` for compatibility, but it must equal the sum of all phase -epochs. Other scheduler arguments remain shared across phases. This example uses -a single optimizer: +Each phase owns `batch_size` (32 by default), `accumulation_steps` (1), and +`optimization.`. Supervised phases have the single role `model`, meaning +the trainable model parameters. Imitation, DPO and GRPO have `actor`; PPO also +has `value`; IQL has `actor`, `value`, and `critic`. Unknown roles fail validation. +Omitted roles use Adam, learning rate 0.0003 and gradient clip 1.0. Set +`gradient_clip: null` to disable clipping. Optimizer plans take their rates from +their groups. Supervised roles default to StepLR (`step_size: 1`, `gamma: 0.99`); +policy roles default to ConstantLR (`factor: 1`, `total_iters: 1`). ```yaml -global_training: - scheduler: - name: OneCycleLR - max_lr: 0.003 - pct_start: 0.2 - cycle_momentum: false - scheduler_step_on: epoch - reset_optimization_on_phase: false +training_plan: + phases: + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: [{source: observations.train}] + batch_size: 64 + accumulation_steps: 1 + optimizer_state: reset + optimization: + model: + optimizer: {name: Adam} + learning_rate: 0.001 + scheduler: {name: StepLR, arguments: {step_size: 1, gamma: 0.99}} + scheduler_step_on: epoch + gradient_clip: 1.0 ``` -Batch-stepped schedulers may configure `total_steps` explicitly because phase -update counts can depend on source interleaving, dataset transitions, and -gradient accumulation. In reset mode, that value applies independently to each -phase; configuration emits a warning to make that scope explicit. In continuous -mode, it applies once across the full run. +`optimizer_state: reset | continue` describes the transition **into** this phase; +its default is `reset`. Reset creates new optimizer, scheduler and scaler state. +Continue preserves these states from the immediately preceding phase and requires +that phase's `handoff: last`, identical algorithm, role configurations, parameter +ownership and freeze selections. The first phase cannot continue. Supervised +continuation requires the same dataset ownership; policy continuation requires +the same interface. Mixed plans reject continuation because their artifact +handoffs do not preserve optimizer state. Policy OneCycleLR continuation is +also rejected. Native checkpoints restore the active lifecycle in either mode. Supervised +`handoff: best` requires an evaluation monitor and selects the preceding phase’s +best weights before starting the next phase; its monitor state is phase-local. + +Supervised `scheduler_step_on: epoch` advances once after a completed phase epoch; +ReduceLROnPlateau receives the configured evaluation monitor at that boundary +and requires `evaluation.monitor`. `batch` advances once per applied optimizer +update, after accumulation, rather than once per microbatch. Policy roles only +accept `scheduler_step_on: optimizer_step`, their default. Policy plateau and +compound schedulers are unsupported. Constructor arguments are validated before +training; the shared optimizer shape does not imply shared scheduler capabilities. + +For epoch OneCycleLR, the resolved `total_steps` is the phase's epoch count, or +the sum of epochs in its contiguous continuation chain. An explicit value must +match. Batch OneCycleLR requires an explicit `total_steps`. Preflight counts loader +batches using the same sequential/interleaved source scheduler, phase seeds and +accumulation rules as training, including partial updates flushed at dataset +boundaries. The budget sums over a contiguous continuation chain. This calculation +does not consume samples or advance loader RNG. Early stopping and explicitly +skipped updates may consume less than the authored maximum. +Policy OneCycleLR requires the exact maximum authored optimizer-update budget: + +| Algorithm | Optimizer updates per role | +| --- | --- | +| Imitation | epochs × ceil(training examples / (batch_size × accumulation_steps)) | +| IQL / DPO | updates | +| PPO | optimization_epochs × sum(ceil(rollout size / (minibatch_size × accumulation_steps))) | +| GRPO | optimization_epochs × sum(ceil(group trajectories / (batch_size × accumulation_steps))) | + +PPO `rollout_length` counts collected transitions; `minibatch_size` counts +transitions in each optimization microbatch. They are distinct budgets. IQL +uses `batch_size: 64` by default and one update per sampled batch; it does not +accept accumulation. DPO counts preference pairs; GRPO counts trajectories. +PPO defaults to `minibatch_size: 64`; other batched algorithms default to 32. +Early stopping, skipped updates or KL limits can consume less than the maximum. + +The authored schema is separate from resolved training phases. Checkpoints and +plan fingerprints contain resolved roles, batching and scheduler budgets; +reproduction does not depend on global optimizer values. `global_training` +contains execution/distribution, resume, data/window and model contracts only. ## Training plans @@ -768,31 +889,15 @@ healthy active update. Momentum and weight decay cannot update parameters whose gradients are absent. -## Policy runs - -Policy training uses one `PolicyRunConfig` with top-level ordered `phases`, -`model`, `policy_interface`, `context_length`, optional named `sources` and -`preferences`, and run-wide output/resume settings. Supported algorithms are -`policy_imitation`, `iql`, `ppo`, `dpo`, and `grpo`. Epochs, offline updates, -PPO environment steps and GRPO groups are explicit per-phase budgets. - -Each phase selects `handoff: best | last` (default `best`). One frozen actor -representation and causal history contract spans the run; all optimizer/critic -state starts fresh at each handoff. Return conditioning supports imitation only. -Standalone legacy policy runs use their existing phase schema. Use -`run_type: unified` for supervised and policy phases in one plan, or -`initialize_from` plus `policy_binding.actions` to post-train an ordinary -supervised artifact. See the [unified learning guide](../supervised-policy-learning.md) -for typed actions, source bindings, prediction distributions, and recovery. - -See the [policy learning guide](../policy-learning.md) for full fields, objective -formulas, environment capabilities, preference manifests, checkpoint boundaries, -selection monitors and exact/non-exact continuation. Examples: - -- [BC](../example_inputs/configs/train-bc-toy.yaml) -- [Return-conditioned imitation](../example_inputs/configs/train-return-conditioned.yaml) -- [IQL](../example_inputs/configs/train-iql-from-table.yaml) -- [PPO](../../src/sequifier/templates/train-ppo.yaml) -- [DPO](../../src/sequifier/templates/train-dpo.yaml) -- [GRPO](../../src/sequifier/templates/train-grpo.yaml) -- [Imitation → IQL → DPO → GRPO → PPO](../example_inputs/configs/train-multiphase-control.yaml) +## Policy and mixed plans + +All algorithms use `training_plan.phases`. Declare policy sequence/preference +parts in `dataset_training`, and attach actions with `policy_binding` (one +interface) or `policy_bindings` (named interfaces). Run-wide settings remain in +`global_training`; no alternate run discriminator is needed. + +See the complete [supervised template](../../src/sequifier/templates/train-supervised.yaml), +[policy template](../../src/sequifier/templates/train-policy.yaml), and +[supervised-to-policy template](../../src/sequifier/templates/train-supervised-to-policy.yaml). +The [policy guide](../policy-learning.md) describes algorithm budgets and +[the mixed-plan guide](../supervised-policy-learning.md) describes artifact handoffs. diff --git a/documentation/example_inputs/configs/import-policy-sequences.yaml b/documentation/example_inputs/configs/import-policy-sequences.yaml index 77a35285..3df68100 100644 --- a/documentation/example_inputs/configs/import-policy-sequences.yaml +++ b/documentation/example_inputs/configs/import-policy-sequences.yaml @@ -1,11 +1,19 @@ operation: import_sequences project_root: . source: - paths: [data/demonstrations.csv] -policy: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - return_conditioned: false + paths: + - data/demonstrations.csv + columns: + observations: + position: position + velocity: velocity + static_inputs: {} output_path: data/imported-demonstrations episodes_per_shard: 64 +policy_binding: + actions: + action: + type: categorical + values: + - left + - right diff --git a/documentation/example_inputs/configs/infer-policy.yaml b/documentation/example_inputs/configs/infer-policy.yaml index ae708cb8..379b77c7 100644 --- a/documentation/example_inputs/configs/infer-policy.yaml +++ b/documentation/example_inputs/configs/infer-policy.yaml @@ -1,15 +1,10 @@ project_root: . -model_path: models/policy/actor.pt -# Raw observations/static inputs in original units; no forecast preprocessing required. +model_path: models/policy/model.pt data_path: data/policy-observations.csv -model_type: generative +model_type: policy read_format: csv write_format: csv device: cpu -deterministic: true -# Omit this for behavior cloning. Alternatively provide a desired_return column. desired_return: 10.0 -# For sequential input, include sequenceId and recorded action values, or: -# previous_action_column: previous_action -# output_probabilities: true -# output_path: outputs/policy-actions.csv +enforce_determinism: true +action_selection: deterministic diff --git a/documentation/example_inputs/configs/infer-ppo-toy.yaml b/documentation/example_inputs/configs/infer-ppo-toy.yaml index 3ac5a8f8..d25900cb 100644 --- a/documentation/example_inputs/configs/infer-ppo-toy.yaml +++ b/documentation/example_inputs/configs/infer-ppo-toy.yaml @@ -1,9 +1,10 @@ project_root: . -model_path: models/online-ppo/actor.pt +model_path: models/online-ppo/model.pt data_path: documentation/example_inputs/ppo-observations.csv -model_type: generative +model_type: policy read_format: csv write_format: csv device: cpu -deterministic: true output_path: outputs/ppo-actions.csv +enforce_determinism: true +action_selection: deterministic diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index 301efdde..d01473c9 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -1,18 +1,19 @@ project_root: . -model_path: models/online-ppo/actor.pt +model_path: models/online-ppo/model.pt data_path: documentation/example_inputs/ppo-observations.csv -model_type: generative +model_type: policy read_format: csv write_format: csv device: cpu -deterministic: true -autoregressive: true -generation_steps: 8 rollout: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 trajectory_path: outputs/ppo-rollout.json seed_index: 0 max_episode_steps: 8 + max_steps: 8 + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 +enforce_determinism: true +action_selection: deterministic diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml index 6fb71dd1..47535a44 100644 --- a/documentation/example_inputs/configs/train-bc-toy.yaml +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -8,32 +8,58 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} + attention: + n_heads: 4 + feed_forward: + dim: 128 positional_encoding_scope: global interfaces: actor: - input_columns: [position] - target_columns: [action] - policy: - observation_columns: [position] - action_column: action - action_values: [left, right] - return_conditioned: false + input_columns: + - position + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -context_length: 16 -sources: - demonstrations.recorded: - paths: [data/offline-transitions.csv] -validation_fraction: 0.2 -phases: -- name: behavior_cloning - algorithm: policy_imitation - sources: [demonstrations.recorded] - epochs: 30 - batch_size: 64 + feature_embedding_dims: + position: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 16 + validation_fraction: 0.2 +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right +dataset_training: + demonstrations: + parts: + recorded: + type: sequence + paths: + - data/offline-transitions.csv + model_interface: actor +training_plan: + phases: + - name: behavior_cloning + algorithm: policy_imitation + sources: + - source: demonstrations.recorded + epochs: 30 + batch_size: 64 diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index 52a936a7..342066e2 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -8,49 +8,80 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} + attention: + n_heads: 4 + feed_forward: + dim: 128 positional_encoding_scope: global interfaces: actor: - input_columns: [position, velocity] - target_columns: [action] - policy: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - return_conditioned: false + input_columns: + - position + - velocity + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 1, velocity: 1, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -context_length: 16 -sources: - demonstrations.recorded: - paths: [data/offline-transitions.parquet] - columns: - sequence_id: sequenceId - item_position: itemPosition - action: action - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {position: next_position, velocity: next_velocity} -validation_fraction: 0.2 -resume: {policy: if_exists} -phases: -- name: offline_improvement - algorithm: iql - sources: [demonstrations.recorded] - updates: 10000 - batch_size: 64 - gamma: 0.99 - expectile: 0.7 - inverse_temperature: 3.0 - max_weight: 100.0 - target_update_rate: 0.005 - evaluation_interval: 100 - checkpoint_interval: 100 + feature_embedding_dims: + position: 1 + velocity: 1 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 16 + validation_fraction: 0.2 + resume: + policy: if_exists +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right +dataset_training: + demonstrations: + parts: + recorded: + type: sequence + paths: + - data/offline-transitions.parquet + columns: + sequence_id: sequenceId + item_position: itemPosition + action: action + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: + position: next_position + velocity: next_velocity + model_interface: actor +training_plan: + phases: + - name: offline_improvement + algorithm: iql + sources: + - source: demonstrations.recorded + updates: 10000 + batch_size: 64 + gamma: 0.99 + expectile: 0.7 + inverse_temperature: 3.0 + max_weight: 100.0 + target_update_rate: 0.005 + evaluation_interval: 100 + checkpoint_interval: 100 diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml index 9c63cff9..61d1d138 100644 --- a/documentation/example_inputs/configs/train-iql-toy.yaml +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -9,41 +9,69 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} + attention: + n_heads: 4 + feed_forward: + dim: 128 positional_encoding_scope: global interfaces: actor: - input_columns: [position] - target_columns: [action] - policy: - observation_columns: [position] - action_column: action - action_values: [left, right] - return_conditioned: false + input_columns: + - position + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -context_length: 16 -sources: - demonstrations.recorded: - paths: [data/offline-transitions.csv] - columns: - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {position: next_position} -validation_fraction: 0.2 -resume: {policy: if_exists} -phases: -- name: offline_improvement - algorithm: iql - sources: [demonstrations.recorded] - updates: 10000 - batch_size: 64 - evaluation_interval: 100 - checkpoint_interval: 100 + feature_embedding_dims: + position: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 16 + validation_fraction: 0.2 + resume: + policy: if_exists +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right +dataset_training: + demonstrations: + parts: + recorded: + type: sequence + paths: + - data/offline-transitions.csv + columns: + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: + position: next_position + model_interface: actor +training_plan: + phases: + - name: offline_improvement + algorithm: iql + sources: + - source: demonstrations.recorded + updates: 10000 + batch_size: 64 + evaluation_interval: 100 + checkpoint_interval: 100 diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index 021911bb..fa4b4447 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -2,94 +2,147 @@ project_root: . model_name: multiphase-control device: cpu seed: 1010 -context_length: 4 model: backbone: architecture: dim_model: 32 num_layers: 1 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} + attention: + n_heads: 4 + feed_forward: + dim: 64 positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [cue] - target_columns: [action] - policy: - observation_columns: [cue] - action_values: [left, right] - return_conditioned: false + input_columns: + - cue + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -normalization: {source: identity} -phases: -- name: imitation - algorithm: policy_imitation - sources: [decisions] - epochs: 4 -- name: offline_improvement - algorithm: iql - sources: [decisions] - updates: 100 -- {name: preferences, algorithm: dpo, preferences: rankings, updates: 100, beta: 0.1} -- name: group_improvement - algorithm: grpo - total_groups: 32 - groups_per_update: 2 - group_size: 4 - max_steps: 8 - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - evaluation: + feature_embedding_dims: + cue: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 4 + normalization: + source: identity +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right +dataset_training: + decisions: + parts: + default: + type: sequence + paths: + - data/control-decisions.csv + columns: + episode_id: episodeId + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: + cue: next_cue + group: contextId + model_interface: actor + rankings: + parts: + default: + type: preferences + manifest: data/control-preferences.json +training_plan: + phases: + - name: imitation + algorithm: policy_imitation + sources: + - source: decisions.default + epochs: 4 + - name: offline_improvement + algorithm: iql + sources: + - source: decisions.default + updates: 100 + - name: preferences + algorithm: dpo + preferences: + source: rankings.default + updates: 100 + beta: 0.1 + - name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_update: 2 + group_size: 4 + max_steps: 8 environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 - metric: score_mean -- name: ppo_improvement - algorithm: ppo - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 + seeds: + - 17 + - 29 + - 43 + - 71 + max_episode_steps: 8 + timeout_seconds: 30 + monitor: + metric: score_mean + mode: max + - name: ppo_improvement + algorithm: ppo environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 -sources: - decisions: - paths: [data/control-decisions.csv] - columns: - episode_id: episodeId - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {cue: next_cue} - group: contextId -preferences: {rankings: data/control-preferences.json} + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 + seeds: + - 17 + - 29 + - 43 + - 71 + max_episode_steps: 8 + timeout_seconds: 30 diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml index 1bd6d7e1..96d5aace 100644 --- a/documentation/example_inputs/configs/train-policy-from-table.yaml +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -8,33 +8,63 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} + attention: + n_heads: 4 + feed_forward: + dim: 128 positional_encoding_scope: global interfaces: actor: - input_columns: [position, velocity] - target_columns: [action] - policy: - observation_columns: [position, velocity] - action_column: action - action_values: [left, right] - return_conditioned: false + input_columns: + - position + - velocity + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 1, velocity: 1, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -context_length: 16 -sources: - demonstrations.recorded: - paths: [data/demonstrations.parquet] -validation_fraction: 0.2 -phases: -- name: behavior_cloning - algorithm: policy_imitation - epochs: 80 - sources: [demonstrations.recorded] - batch_size: 64 - optimization: {actor: {learning_rate: 0.005}} + feature_embedding_dims: + position: 1 + velocity: 1 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 16 + validation_fraction: 0.2 +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right +dataset_training: + demonstrations: + parts: + recorded: + type: sequence + paths: + - data/demonstrations.parquet + model_interface: actor +training_plan: + phases: + - name: behavior_cloning + algorithm: policy_imitation + epochs: 80 + sources: + - source: demonstrations.recorded + batch_size: 64 + optimization: + actor: + learning_rate: 0.005 diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index fcad048b..44e45440 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -8,39 +8,73 @@ model: dim_model: 32 num_layers: 2 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} + attention: + n_heads: 4 + feed_forward: + dim: 128 + activation: swiglu positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [OBSERVATION_COLUMN] - target_columns: [action] - policy: - observation_columns: [OBSERVATION_COLUMN] - action_column: action - action_values: [0.0, 1.0] - return_conditioned: true - returns: {gamma: 1.0, scale: 1.0} + input_columns: + - OBSERVATION_COLUMN + - __policy_previous_action + - __policy_return + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} - decoder: {type: linear, prediction_length: 1, support: 1} + feature_embedding_dims: + OBSERVATION_COLUMN: 1 + __policy_previous_action: 6 + __policy_return: 1 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] model_path: models/policy/actor.pt report_path: outputs/policy/training-report.json -policy_interface: actor -context_length: 4 -sources: - demonstrations.recorded: - paths: [data/demonstrations.parquet] -validation_fraction: 0.2 -episode_cache_size: 8 -resume: {policy: never} -phases: -- name: offline_policy - algorithm: policy_imitation - epochs: 80 - sources: [demonstrations.recorded] - batch_size: 64 - optimization: {actor: {learning_rate: 0.005}} +global_training: + context_length: 4 + validation_fraction: 0.2 + episode_cache_size: 8 + resume: + policy: never +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: true + returns: + gamma: 1.0 + scale: 1.0 + actions: + action: + type: categorical + values: + - 0.0 + - 1.0 +dataset_training: + demonstrations: + parts: + recorded: + type: sequence + paths: + - data/demonstrations.parquet + model_interface: actor +training_plan: + phases: + - name: offline_policy + algorithm: policy_imitation + epochs: 80 + sources: + - source: demonstrations.recorded + batch_size: 64 + optimization: + actor: + learning_rate: 0.005 diff --git a/documentation/example_inputs/iql_toy_control.py b/documentation/example_inputs/iql_toy_control.py index 8a3a18e8..6592265e 100644 --- a/documentation/example_inputs/iql_toy_control.py +++ b/documentation/example_inputs/iql_toy_control.py @@ -12,37 +12,61 @@ import sys from pathlib import Path -from sequifier.config.rollout_config import RolloutConfig +from sequifier.config.policy_infer_config import PolicyRolloutConfig from sequifier.rl.checkpoint import load_policy +from sequifier.rl.environment import ( + EnvironmentCapabilities, + OnlineResetResult, + OnlineStepResult, +) from sequifier.rl.runtime import collect_policy_rollout -from sequifier.rollout.contracts import StepResult from sequifier.trajectory.schema import SeedHistory HORIZON = 12 class ToyControlAdapter: - protocol_version = 1 + protocol_version = 3 + implementation_id = "sequifier.toy-control.v1" + capabilities = EnvironmentCapabilities(seeded_reset=True) - def reset(self, request): - self.position = int(request.seed_history["position"][-1]) + def reset(self, seed): + initial = OnlineResetResult( + {"position": float(random.Random(seed).choice((-4, -3, 3, 4)))} + ) + return self.reset_from_observation(initial, seed) + + def reset_from_observation(self, initial, seed): + self.position = int(initial.observation["position"]) + self.steps = 0 + return initial - def step(self, request): - if int(request.observation["position"]) != self.position: - raise ValueError("control state and policy observation disagree") - action = request.predictions["action"] - self.position += -1 if action == "left" else 1 + def step(self, action): + self.position += -1 if action["action"] == "left" else 1 + self.steps += 1 done = self.position == 0 - return StepResult( - external_inputs={"position": float(self.position)}, + return OnlineStepResult( + observation={"position": float(self.position)}, reward=1.0 if done else -0.05, terminated=done, - truncated=request.step_id + 1 == HORIZON and not done, + truncated=self.steps == HORIZON and not done, ) def close(self): pass + def snapshot_state(self): + raise NotImplementedError("snapshot capability is not supported") + + def restore_state(self, state): + raise NotImplementedError("snapshot capability is not supported") + + def create_group_start(self, seed): + raise NotImplementedError("group-start capability is not supported") + + def reset_group(self, token, rollout_seed): + raise NotImplementedError("group-start capability is not supported") + def generate(path): rng = random.Random(1010) @@ -96,13 +120,14 @@ def generate(path): def evaluate(paths): - config = RolloutConfig.model_validate( + config = PolicyRolloutConfig.model_validate( { - "mode": "policy", - "adapter": {"factory": "iql_toy_control:ToyControlAdapter"}, - "bindings": [ - {"input": "position", "source": "environment", "key": "position"} - ], + "environment": { + "factory": "iql_toy_control:ToyControlAdapter", + "implementation_id": "sequifier.toy-control.v1", + "units": {}, + }, + "max_steps": HORIZON, "trajectory_path": "unused.json", "seed_index": 0, "max_episode_steps": HORIZON, @@ -113,8 +138,12 @@ def evaluate(paths): returns = [] for episode, position in enumerate((-4, -3, 3, 4) * 25): seed = SeedHistory( - observations={"position": [float(position)]}, - attention_valid_mask=[True], + observations={ + "position": [0.0] * (policy.config.context_length - 1) + + [float(position)] + }, + attention_valid_mask=[False] * (policy.config.context_length - 1) + + [True], static_inputs={}, ) trajectory = collect_policy_rollout( diff --git a/documentation/example_inputs/policy_control_data.py b/documentation/example_inputs/policy_control_data.py index 77e0c217..257dc582 100644 --- a/documentation/example_inputs/policy_control_data.py +++ b/documentation/example_inputs/policy_control_data.py @@ -63,7 +63,7 @@ def generate(directory): ) cue = -cue alternatives[kind] = { - "source": "decisions", + "source": "decisions.default", "episode_id": identity, "start": 0, "end": length, diff --git a/documentation/example_inputs/ppo_toy_control.py b/documentation/example_inputs/ppo_toy_control.py index 9f918257..a65d3e46 100644 --- a/documentation/example_inputs/ppo_toy_control.py +++ b/documentation/example_inputs/ppo_toy_control.py @@ -1,26 +1,20 @@ -"""Train the categorical control example with the canonical Python policy API.""" +"""Train and evaluate the shipped PPO example through the common Python API.""" from importlib.resources import files from sequifier.config.train_config import load_train_config -from sequifier.rl_api import ( - PolicyRunConfig, - evaluate_policy, - load_policy, - run_policy_training, -) +from sequifier.rl_api import evaluate_policy, load_policy +from sequifier.train import run_training def main(): config = load_train_config( str(files("sequifier").joinpath("templates", "train-ppo.yaml")), {}, False ) - if not isinstance(config, PolicyRunConfig): - raise TypeError("the example requires a policy run config") - result = run_policy_training(config) + result = run_training(config) actor = load_policy(result.model_path) print("Selected return:", result.phases[-1].selected_metric) - print(evaluate_policy(actor, config.phases[0].evaluation)) + print(evaluate_policy(actor, config.training_plan[0].evaluation)) if __name__ == "__main__": diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 9143c359..9d240409 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -5,113 +5,51 @@ input schemas, and mixed supervised/policy plans, see the [unified learning guide](supervised-policy-learning.md). The configuration below documents the compatible single-categorical authoring surface. -`sequifier train` and `sequifier.rl_api.run_policy_training` execute the same -`PolicyRunConfig`. Its ordered `phases` may contain `policy_imitation`, `iql`, -`ppo`, `dpo`, and `grpo` in any compatible order. Each phase takes ownership -of its predecessor's selected actor. Optimizers, schedulers, critics, samplers, -and environments start fresh at phase boundaries. - -This legacy surface supports one fixed categorical action, scalar or categorical -observations/static fields, causal fixed-length histories, FP32, and local -single-process CPU/CUDA/MPS execution. Return conditioning is available only -for imitation. A run containing another algorithm requires an unconditioned -actor throughout. Mixed phases use the unified run configuration linked above. -For this legacy surface, text tokenizers, continuous/joint actions, images/depth, distributed training, -vectorized environments, and reward models are outside this contract. +`sequifier train` accepts one `SequifierConfig` with ordered +`training_plan.phases`. Supervised, imitation, IQL, PPO, DPO and GRPO use the same +model/interface and artifact contracts. A policy binding selects named real or +categorical action heads. See the [training guide](configs/train.md) for phase +ownership, scheduling budgets, optimizer groups and transition validation. +Policy execution currently uses a local CPU/CUDA/MPS process; standalone +supervised training also supports distributed execution. ## Authoring a run -Start with the [multi-phase example](example_inputs/configs/train-multiphase-control.yaml), -[single-phase DPO](../src/sequifier/templates/train-dpo.yaml), or -[single-phase GRPO](../src/sequifier/templates/train-grpo.yaml). -Generate their curated data with: +Use the [policy template](../src/sequifier/templates/train-policy.yaml), +[PPO template](../src/sequifier/templates/train-ppo.yaml), or +[multiphase example](example_inputs/configs/train-multiphase-control.yaml). +The latter contains imitation → IQL → DPO → GRPO → PPO. Generate its data with: ```shell python documentation/example_inputs/policy_control_data.py data sequifier train --config-path documentation/example_inputs/configs/train-multiphase-control.yaml ``` -The example defines imitation → IQL → DPO → GRPO → PPO. Remove or reorder phase -entries to author smaller runs, such as imitation → IQL, imitation → PPO, -imitation → DPO, imitation → GRPO, or DPO → GRPO. The input preferences come from -an explicitly supplied sign-matching rule; Sequifier never derives preferences -from recorded reward. These are illustrative configurations, without a claim -that every algorithm order improves every task. - -```yaml -project_root: . -model_name: control -policy_interface: actor -context_length: 4 -seed: 1010 -device: cpu -# model uses the ordinary backbone/interfaces architecture schema. -model: { ... } -sources: - decisions: - paths: [data/control-decisions.csv] - columns: - episode_id: episodeId - group: contextId -preferences: - rankings: data/control-preferences.json -phases: - - name: imitate - algorithm: policy_imitation - sources: [decisions] - epochs: 4 - - name: rank - algorithm: dpo - preferences: rankings - updates: 100 - beta: 0.1 - reference: {source: phase_start} -resume: {policy: if_exists, mode: exact} -``` - -Run-wide fields: - -| Field | Meaning / default | -| --- | --- | -| `model`, `policy_interface`, `context_length`, `model_name`, `phases` | Required architecture, selected interface, history length, run name, ordered phases | -| `project_root`, `device`, `seed`, `enforce_determinism` | `.`, `cpu`, 1010, true; seed in `[0, 2**32)` | -| `sources` | Named `SequenceSourceConfig` objects; paths/manifest, format, explicit column mappings, prepared metadata | -| `preferences` | Named paths to version-1 preference-pair JSON manifests | -| `units` | Frozen observation/static units; `{}` means unitless; online training/evaluation adapters must match | -| `initialize_from` | Optional portable actor with matching full representation and architecture | -| `normalization` | Optional explicit frozen `source: identity` or `source: statistics` with exact real-field `statistics: {column: {mean: ..., std: ...}}` | -| `validation_fraction`, `episode_cache_size` | 0.2 and 8; split independent components before fitting codecs/statistics | -| `resume` | `{policy: never, mode: reset_environments}`; policy also `if_exists`/`required`, mode also `exact` | -| `model_path` | `models//actor.pt` | -| `report_path` | `outputs/training//report.json` | -| `checkpoint_path` | `checkpoints//run.pt` | -| `phase_directory` | `outputs/training//phases`; subdirectories `000-`, etc. | - -The model must contain exactly `policy_interface`. Its input columns list the -policy observations followed by static fields; its target is the action column. -Use a scalar categorical decoder with `support: 1`, `prediction_length: 1`. -Declare `return_conditioned: false` for BC or any improvement algorithm. Standard -ingestion/backbone/decoder configuration remains available. Previous executable -actions enter the input as `__policy_previous_action`; its start token is -input-only. Return-conditioned imitation additionally uses `__policy_return`. - -The first offline phase fits missing categorical vocabularies and normalization -on its training episodes only. A declared normalization takes precedence over -fitting statistics. An online-first run must supply frozen normalization and -categorical counts/codecs, or a compatible `initialize_from` actor. Later phases -reuse the complete frozen representation. They may optionally declare -`representation` and online `normalization` as assertions; any disagreement -fails with a field-specific diagnostic before training. An online phase after -an offline phase normally omits `normalization` to reuse the fitted values. - -All input files, preference manifests, metadata, initial actors and references -are protected from output collisions. Source bytes, partitions, codecs, model, -units, reference weights, and adapter identities are checked on resume. +All sources live in `dataset_training..parts.`. Use `type: sequence` +for raw tables or exported sequence manifests, prepared metadata parts for +supervised data, and `type: preferences` with `manifest` for preference pairs. +Phase source references are objects such as `{source: decisions.default}`; +DPO `preferences` uses the same reference shape. Every data-bearing dataset +selects its model interface explicitly when more than one is available. + +Run-wide `context_length`, normalization, validation fraction (0.2), episode +cache size (8), determinism and resume settings live in `global_training`. +`resume` defaults to `{policy: never, mode: reset_environments}`; use `if_exists` +or `required` with a checkpoint path, and `exact` for supported online adapters. +`policy_binding` applies to a single model interface; use `policy_bindings` keyed +by interface name for multiple interfaces. Environment units must match the binding. +No optimizer defaults live in this run-wide block. + +Default outputs are `models//model.pt`, +`outputs/training//report.json`, and +`outputs/training//phases`. The resolved configuration records +role defaults and execution contracts. Fingerprints and checkpoints include +phase settings and validate source, reference and environment identities on resume. ## Selection and optimization Every phase has a unique identifier `name` and `handoff: best | last` (default -`best`). Best selection retains the earliest finite best on ties. The final +`last`). Best selection retains the earliest finite best on ties. The final update always evaluates. Missing finite evaluation for `best` fails explicitly. `last` selects the most recently completed optimizer transaction. The final export uses the last phase's selected actor, with metric, transaction and weight @@ -337,7 +275,7 @@ independent seeds and a whole-evaluation POSIX/main-thread deadline. It evaluate the live actor synchronously and restores every module's training mode and the training RNG state afterward. It requires `environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; -`selection: deterministic | stochastic` defaults to deterministic. Default +`action_selection: deterministic | stochastic` defaults to deterministic. Default `metric: mean_completed_return`, `direction: max` selects true completed returns. `mean_completed_length` and `success_rate` are also supported. Censored episodes and unavailable metrics are explicit; best selection fails when no finite @@ -452,38 +390,47 @@ source/reference provenance, selections, clocks and continuation modes. ## Python and inference +The Python API starts from the same authored schema as YAML: + ```python -from sequifier.config.train_config import load_train_config -from sequifier.rl_api import PolicyRunConfig, run_policy_training, load_policy - -config = load_train_config("configs/train-policy.yaml", {}, False) -if not isinstance(config, PolicyRunConfig): - raise TypeError("expected a policy run") -result = run_policy_training(config) -actor = load_policy(result.model_path) -for phase in result.phases: - print(phase.name, phase.selected_metric, phase.clocks) +from sequifier.config.train_config import SequifierConfig +from sequifier.config.training_resolution import resolve_training_run +from sequifier.train import run_training +import yaml + +with open("configs/train-policy.yaml") as stream: + authored = SequifierConfig.model_validate(yaml.safe_load(stream)) +resolved = resolve_training_run(authored) +result = run_training(resolved) ``` -`PolicyRunConfig.model_validate` accepts the same mapping directly. The single -`PolicyRunResult` exposes `policy`, ordered `phases`, `representation` and output -paths. `rl_api` also exports the phase schemas, preference records, online -capabilities, `PolicyWindowView`, and model loading/execution helpers. - -Use `sequifier infer` with the final artifact for categorical sequence inference -or rollout. `collect_policy_rollout` uses the same causal `PolicyHistory` and -`PolicyCursor` decision/step path as online training and evaluation. -Return-conditioned actors require -an explicit desired return. `PolicyRolloutConfig` declares `environment`, -`trajectory_path`, `seed_index` and optional `max_episode_steps`; the environment -must support seeded reset. Successors always come from the environment, so no -prediction bindings are configured. A true terminal may omit its successor; -truncation and collection cutoffs must retain it. -See [policy inference](configs/infer.md), [rollout](rollout.md), and the +`run_training` selects the supervised engine, native policy runner, or mixed-plan +coordinator from the resolved phase algorithms. `load_train_config(path, {}, False)` +combines YAML loading and resolution. `run_policy_training` takes an internal +resolved policy run, rather than a separate authored configuration. Direct Python +inference can validate `PolicyInferenceConfig`, resolve it with +`policy_inference_config(request.model_dump(exclude_unset=True))`, and call +`sequifier.rl.inference.infer_policy`; it uses the same schema as the YAML loader. +`rl_api` exports phase schemas, policy bindings, action declarations, environment +capabilities, sequence records and `load_policy`. + +Use `model_type: policy` with a selected portable PT artifact for raw CSV/Parquet +observation rows. `enforce_determinism` requests deterministic execution; +`action_selection: deterministic | stochastic` selects the policy mode or sampling. +The seed is independent of both. Seeded sampling can reproduce a sequence under +the same supported runtime, device and input order. Online evaluation uses +`action_selection`, defaulting to deterministic, with the run's execution settings. +Evaluation reports record that choice. + +Policy `rollout.environment` declares the version-3 environment, `max_steps` +sets the collection budget, and `seed_index` selects a zero-based raw input row. +`trajectory_path` is required; `max_episode_steps` is optional. Policy rollout +rejects forecast bindings and generation settings. It requires seeded observation +reset; successors come from actual environment transitions. True termination may +omit a successor; truncation and collection cutoffs retain it. Forecast rollout +instead selects a prepared model window and follows the separate version-1 +protocol. See [inference](configs/infer.md), [rollout](rollout.md), and the [Python PPO workflow](example_inputs/ppo_toy_control.py). -CPU determinism is the reference execution contract. CUDA and MPS reproducibility -is limited by the selected PyTorch backend/operators and matching device/RNG -topology. This implementation was statically checked; no tests or training runs -were written/executed for this change, so numerical and end-to-end acceptance -verification remains outstanding. +CPU tests cover the reference execution contract. CUDA and MPS reproducibility +also depends on backend/operators and matching device/RNG topology. diff --git a/documentation/public-config-surface-implementation-plan.md b/documentation/public-config-surface-implementation-plan.md new file mode 100644 index 00000000..5a65b18f --- /dev/null +++ b/documentation/public-config-surface-implementation-plan.md @@ -0,0 +1,368 @@ +# Public configuration surface implementation plan + +## Design goal and reference + +Build an ergonomic policy-training and rollout extension of the public configuration at commit `2d01d407cd2134c8197b1a18f0034d01486f18a9`. Use that commit's ownership rules, singleton conveniences, source references, component specifications, and authored/resolved separation as the foundation. + +Use commit `082225833e1d744e662d094008bcc44268ab868a` as the capability baseline. Every supported operation and meaningful control at that commit must remain expressible and executable through the new design. The earlier reference commit guides layout, not the set of features to retain. Record and assess any subsequent capability additions before implementation finishes. + +Extend `SequifierConfig` and `ResolvedSequifierConfig` instead of introducing a parallel public run abstraction. Keep `training_plan` as the home of training phases, `dataset_training` as the home of offline data and dataset training policy, and `model` as the home of architecture. Keep the explicit inference selector `model_type`, extending its values to include `policy`. + +This is one clean public contract. The implementation removes the competing policy/mixed-run schemas and routing mechanisms. No configuration-version switch, conversion tool, deprecation period, alternate-format loader, import wrapper, or checkpoint-format upgrade is part of the work. + +The examples below describe the proposed design, not the schemas currently implemented. + +## 1. Reuse the reference commit's layout and abstractions + +The reference commit's train, infer, and hyperparameter-search guides and their corresponding configuration models establish these patterns: + +| Reference pattern | Apply it to the new surface | +| --- | --- | +| One shared backbone and named model interfaces | Policies select ordinary interfaces and existing target heads. No separate policy architecture declaration. | +| `dataset_training..parts.` | Put offline policy data in typed dataset parts. Reuse dataset/interface ownership and metadata resolution. | +| `sources: [{source: dataset.part}]` | Use the same reference object in supervised and offline policy phases and dataset evaluation. | +| `training_plan.phases` and direct single-phase fields | Add algorithm-specific phase variants in the same plan; support the same singleton convenience for policy phases. | +| Unique routes inferred; named routes required when ambiguous | Apply the rule to policy interfaces, sources, and bindings. Never select the first eligible mapping entry. | +| `model.interface`, `dataset.part`, and `evaluation: true` | Keep these intentional authoring conveniences and their deterministic expansion into named structures. | +| `ComponentSpec` and `OptimizerPlan` | Reuse flattened component arguments, parameter selectors, optimizer groups, and scheduler validation for policy optimization roles. | +| `global_training` owns runtime, optimization defaults, and recovery | Extend that owner rather than adding policy-only top-level runtime/recovery conventions. | +| Preprocessing metadata owns types, vocabularies, statistics, and storage facts | Resolve policy execution metadata from the same source of truth. | +| `SequifierConfig` / `ResolvedSequifierConfig`, `InferenceConfig` / `ResolvedInferenceConfig` | Validate authored choices first; perform resource loading and execution resolution separately. | +| Explicit required `model_type`, with `generative` and `embedding` operations | Add the `policy` operation; remove `execution_mode` and artifact-triggered operation selection. | +| Composed YAML and HPO trees use ordinary config paths | Extend the same composition, singleton expansion, and trial validation mechanisms. | + +The distinction between singleton and named authoring is useful ergonomics. Both forms expand to one schema and one execution meaning. Neither selects a different run class or runtime path. + +## 2. Training: extend `SequifierConfig` and `training_plan` + +### Run ownership + +Use the reference layout with a small number of deliberate extensions: + +| Location | Ownership | +| --- | --- | +| Top-level `project_root`, `model_name`, `device`, `seed`, export fields | Run identity, execution target, and outputs, as in the reference. | +| `model.backbone`, `model.interfaces` | Architecture, input/target columns, static/image/depth components, parameter initialization. | +| `initialize_from` | Optional full-model artifact initialization; distinct from resuming a run or reusing only a backbone repository. | +| `global_training` | Shared runtime settings, window defaults, supervised optimization defaults, determinism, precision, checkpointing, and resume. | +| Top-level `model_path`, `report_path`, `phase_directory` | Optional destinations for the final model, run report, and phase artifacts; retain configurable outputs alongside existing export fields. | +| `dataset_training` | Named offline datasets, interface references, typed parts, supervised loss settings, and dataset freezing. | +| `policy_bindings` | Interface-keyed task declarations: named actions, action distributions, and explicit policy input bindings. | +| `training_plan` | Ordered phases, phase budgets, and supervised curriculum policy. | +| `evaluation` | Shared dataset evaluation specification, with phase-local evaluation where the algorithm needs a different contract. | + +Do not add top-level `phases`, `run_type`, `policy_interface`, `execution_model`, `sources`, `preferences`, or a second `resume` block. Their responsibilities belong to the owners above. + +Extend `TrainingPlanModel.phases` to a discriminated union keyed by `algorithm`: `supervised`, `policy_imitation`, `iql`, `ppo`, `dpo`, and `grpo`. An omitted algorithm means `supervised`, keeping a simple supervised phase as concise as the reference. Insert that default before discriminated validation. + +Keep algorithm-specific budgets: supervised/imitation `epochs`, IQL/DPO `updates`, PPO `total_env_steps`, and GRPO `total_groups`. Do not make them fit an artificial universal duration field. Keep supervised `mode`, `selection`, weights, and batches per selection specific to supervised phases. + +Direct single-phase fields under `training_plan` work for every algorithm. Expand them into one phase named `train`; fill `mode: sequential` only for supervised phases. Reject direct phase fields combined with `training_plan.phases`. Policy phases never receive supervised-only defaults. + +### Global settings and phase optimization + +Retain `global_training` as one shared owner, including in policy-only runs. Extend its validation so users do not have to invent supervised settings for an online policy phase: + +- Require supervised objective/loss/data-loader settings only when selected supervised phases use them. +- Resolve window/model-contract facts from declared model metadata or `initialize_from`, checking explicit values as assertions. +- Keep policy-specific learning dynamics in their phase schema, including batch sizes that have algorithm-specific meanings. +- Reuse `ComponentSpec`, `OptimizerPlan`, parameter catalogs, and freezing selectors for each policy `optimization` role. A policy role uses its phase settings and documented algorithm defaults; do not silently inherit a supervised epoch scheduler as a policy update scheduler. +- Phase `optimizer_state: reset | continue` replaces the global reset switch. Continuing state requires matching algorithms, roles, parameter ownership, optimizer/scheduler definitions and a last handoff; mixed coordinator transitions are rejected. +- Keep `global_training.resume` using `ResumeConfig.policy` and `checkpoint_path`. Extend that block with an environment recovery `mode: exact | reset_environments`, defaulting to `reset_environments` when relevant. Validate snapshot capabilities for exact online recovery. +- Put automatic offline holdout sizing at `global_training.validation_fraction` and bounded episode caching at `global_training.episode_cache_size`. Keep both configurable; neither requires an offline preprocessing step. +- Retain phase-local `freeze`, `evaluation_interval`, and `checkpoint_interval`, alongside dataset-specific freezing. Resolve their combined parameter ownership with the existing catalogs/selectors and validate which roles may update each parameter. + +Retain every applicable algorithm-specific control, including PPO partial-rollout handling, normalization, reward scaling, clipping/KL/entropy/value settings, optimization epochs, minibatching and accumulation; IQL expectile, temperature/weight limits, target updates and discounting; DPO beta and reference selection; and GRPO grouping, reference/old-policy KL settings and collection limits. Keep `reference.source: phase_start | artifact` and `reference.path` for DPO/GRPO. Keep `scoring_chunk_size`, `max_buffer_bytes`, `max_env_steps`, `archive_directory`, and PPO `archive_scores` on the applicable phase schemas. Their units and effects are algorithm-specific and must be documented rather than dropped during consolidation. + +Use top-level `model_path`, `report_path`, and `phase_directory` overrides for run outputs and `global_training.resume.checkpoint_path` for the rolling run checkpoint destination, including when resume policy is `never`. Preserve timed/batch/epoch/latest checkpoint controls and supported export/dropout/embedding options. Resolve paths under `project_root` and validate input/output collisions. Document derived defaults once for the unified run; no output location should become unconfigurable merely because its former run class is removed. + +Keep supervised phase transitions on continuing model weights. Add `handoff: last | best` to phases, defaulting to `last`; `best` requires a usable evaluation monitor and restores the complete selected model/distribution state. Handoff and optimizer-state continuation are separate decisions and must be validated together. + +### One offline source system + +Extend dataset parts with a discriminated `type`, following the component pattern: + +| Part type | Authored data contract | Eligible uses | +| --- | --- | --- | +| `prepared` (default) | The reference `metadata_config_path`, media paths, and file ordering. | Supervised training; policy learning only when explicit action/episode mappings supply the needed task semantics. | +| `sequence` | Direct `paths`, a source `manifest`, or a metadata-selected split; optional schema/normalization metadata and typed column mappings. | Imitation and transition-based policy learning from tables, trajectories, or prepared data. | +| `preferences` | Preference manifest referencing sequence parts through the shared dataset catalog; metadata supplied by those parts or the resolved interface as needed. | DPO preference scoring. | + +Reuse `SequenceSourceConfig`, `PolicyTableColumns`, trajectory readers, and preference readers as implementation building blocks. Refactor their public placement into dataset parts instead of exposing parallel top-level source registries. Keep raw/model encoding declarations with the part's data contract. + +Sequence parts retain the current source controls: `paths`, `manifest`, `format`, `columns`, `encoding`, `metadata_config_path`, `split_index`, `static_data_path`, `episode_end`, `episode_namespace`, `units`, and `action_semantics`. Select exactly one data location: direct paths, a manifest, or `metadata_config_path` with `split_index`. Metadata may also accompany direct paths/manifests as a schema assertion; its presence alone must not replace their data location. Retain automatic format detection and explicit table/trajectory/prepared reading where currently supported. + +Raw table and trajectory inputs do not require a generated preprocessing metadata file when the model/artifact, source declarations, and frozen normalization supply the necessary contract. Model-encoded/prepared inputs still require the metadata needed to interpret their values. Missing necessary information is a precise resolution error, not a requirement to preprocess every raw source. Preference manifests resolve their sequence references against the same dataset/part catalog, with no independent top-level registry. + +Keep explicit `action_semantics: executed_predictions` for learning from forecast trajectories, separate from policy-action records. Retain episode identity namespaces, end-of-file completion declarations, previous-action/successor mappings, and bootstrap flags. These affect task meaning and leakage checks, not merely data-loading syntax. + +`dataset_training..model_interface` still owns the dataset's interface. A reference to a whole dataset selects all its parts in declaration order; `dataset.part` selects one. Parts in one dataset must agree on the selected interface's codecs, normalization, and input semantics. Phase validation rejects incompatible part types or missing task fields. + +Make `criterion` and supervised loss settings required when that dataset is selected by a supervised phase. Policy-only datasets do not need dummy criteria. Rewards, terminal/truncation flags, successors, action labels, and episode boundaries require explicit data semantics; a supervised window is not automatically an RL transition. + +Use `TrainingSourceSpecModel` references throughout: `sources: [{source: demonstrations.recorded}]`, not a different list-of-strings syntax for policy phases. DPO's `preferences` field uses the same reference object and requires a preference part. Validate whole-dataset references for that algorithm rather than silently skipping incompatible parts. + +Retain explicit offline `validation_sources` and DPO `validation_preferences` on phases, using the same source reference objects as their training counterparts. Resolve holdouts in this order: explicit phase validation references, declared metadata validation partitions where available, then a seeded automatic split using `global_training.validation_fraction`. Apply the automatic split only when no validation partition was supplied. Partition sequence data by episode and shared demonstration/group identity; split preference data by connected underlying episodes/spans rather than individual pairs. Detect overlap even when the same content is exposed through different source names. Keep explicit separate validation datasets and automatic splitting as first-class workflows. + +Dataset evaluation uses the same source references as training. Its selected sources/metrics must agree with the effective validation partitions; an evaluation declaration must not silently re-split or replace phase validation data. Extend `EvaluationMonitorSpecModel` with registered algorithm metrics while retaining the reference spelling `mode: min | max`. Online evaluation retains its environment, seeds, episode budget, timeout, and deterministic/stochastic `selection`; expose its monitor with the same `metric`/`mode` vocabulary. A phase-local evaluation specification replaces the shared evaluation for that phase. `evaluation: true` expands only when one dataset evaluation source can be inferred; it never invents online evaluation seeds or an environment. Each phase must have a compatible effective evaluation specification when its algorithm or handoff requires one. + +For example, a raw demonstration source and a separate holdout need no source metadata file when an initialization artifact supplies their model contract: + +```yaml +dataset_training: + demonstrations: + model_interface: control + parts: + train: {type: sequence, paths: [data/train.csv], format: table} + holdout: {type: sequence, paths: [data/holdout.csv], format: table} +training_plan: + algorithm: policy_imitation + sources: [{source: demonstrations.train}] + validation_sources: [{source: demonstrations.holdout}] + epochs: 3 +``` + +This fragment assumes matching column names and disjoint episode/group identities. Omitting `validation_sources` permits the configured automatic holdout when the source has no declared validation partition. + +### Minimal policy authoring + +This proposed complete run declaration initializes from an ordinary model artifact, infers its unique interface, and reuses the reference singleton conventions: + +```yaml +project_root: . +model_name: controller-imitation +device: cpu +initialize_from: models/controller.pt + +global_training: + resume: {policy: never} + +policy_binding: + actions: + action: {type: categorical, values: [left, right]} + +dataset: + part: + type: sequence + metadata_config_path: configs/metadata_configs/demonstrations.json + +training_plan: + algorithm: policy_imitation + epochs: 3 + +evaluation: true +export_pt: true +``` + +`policy_binding` is the singleton spelling of `policy_bindings.`, following `model.interface`; it has exactly the same schema and meaning. Reject singular and plural declarations together, even if their contents match. A binding never lives under `model.interfaces..policy` in authored config. + +For mixed training, the surrounding model, global settings, and datasets keep their locations. The phase list can contain: + +```yaml +training_plan: + phases: + - name: supervised_warmup + epochs: 5 + mode: sequential + sources: [{source: control_data.train}] + - name: imitate + algorithm: policy_imitation + sources: [{source: demonstrations.recorded}] + epochs: 3 + - name: online_improvement + algorithm: ppo + model_interface: control + total_env_steps: 4096 + rollout_length: 128 + environment: + factory: my_control:make_environment + implementation_id: control-v1 + evaluation: + environment: + factory: my_control:make_environment + implementation_id: control-v1 + seeds: [17, 29] + max_episode_steps: 100 + timeout_seconds: 30 + monitor: {metric: mean_completed_return, mode: max} +``` + +This second example is a fragment; it assumes the referenced datasets and binding for interface `control` are declared. Offline phases derive their interface from their dataset references. An explicit phase `model_interface` is an assertion when a source already fixes it, and selects the route for online phases. Infer it only when one eligible interface exists. A policy phase operates on one interface; reject offline references spanning several interfaces. + +## 3. Policy bindings: task choices separated from execution facts + +Introduce a small authored `PolicyBinding` and a separate `ResolvedPolicyBinding`. Use named `actions` for a single action and for several actions alike. Remove the `action_column`/`action_values` alternative and the runtime distinction selected by `is_joint`. + +An authored binding contains: + +- Named action distributions/domains, using `ActionFieldConfig` and existing categorical/real type vocabulary. +- `decision_position` when selecting a particular decoder output is necessary. +- `previous_action_inputs` and `initial_inputs` when ordinary declared model inputs require them. +- Optional return conditioning, defaulting to disabled, and its return definition/initial desired return when enabled. +- Task `units`, checked against source and environment declarations and retained in execution/trajectory identities. + +Resolve action ordering from the selected interface's target/decoder contract. Validate that actions are a compatible subset of model targets, categorical support matches executable decoder IDs, real scales are finite and explicit, and conditional activity respects decoder dependencies. Leave unrelated prediction heads available to supervised phases. + +The model and preprocessing metadata supply observation/static membership, types, class counts, ID maps, normalization, context geometry, and decoder semantics. Do not expose `alignment`, `input_schema`, `input_statistics`, class maps, `execution_model`, or phase `representation` in authored config. Do not insert previous-action/return inputs or replace target heads implicitly. + +A fresh policy model uses the same `ModelSpecModel`, component initialization, and metadata-driven interface resolution as a fresh supervised model. For online-only models, add optional `global_training.metadata_config_path`: a string when the model interface is unique, or an interface-to-path map when named routes need different metadata. This reuses the reference's scalar-or-named-map pattern for media paths and supplies input metadata without a dummy offline dataset. If a selected dataset or initialization artifact already supplies the interface metadata, this declaration is unnecessary; simultaneous declarations must agree. Keep statistics/vocabularies in the metadata file. Reuse the explicit frozen-normalization contract for online data instead of deriving normalization from a changing rollout stream. + +Full-model artifact initialization resolves architecture, codecs, and existing weights once. Bindings attach to existing targets; explicit architecture or metadata assertions must agree. Reuse parameter initialization/catalog abstractions for declared new distribution parameters. Do not overload backbone repository settings with full-policy initialization. + +Artifact serialization retains resolved policy metadata for portable execution. Artifact readers and checkpoint validation consume the resolved type directly; they do not pass internal dictionaries through authored config validation. Use one ordinary model-input execution path for all named action configurations. + +Removing the separate scalar-action branch must retain single-categorical behavior cloning and return-conditioned training. Provide explicit model inputs and binding declarations for previous-action history, initial/start values, and desired-return features; the resolver must support their encoding and update rules without depending on that branch. Cover string/numeric category labels, class IDs, first-step history, sequence fragments, padded windows, return-to-go/remaining-return calculations, sampled/deterministic action selection, likelihood, gradients, and collection/replay. Existing mixed/real distributions, learned scales, bounds, conditional activity, inactive values, and point-mass atoms remain available through `ActionFieldConfig`. Branch deletion is gated on capability coverage in the unified execution path. + +## 4. Inference: extend the existing `model_type` + +Keep `model_type` required and explicit, just as in the reference commit. Extend its allowed values; remove `execution_mode` and artifact-based auto-selection. + +| `model_type` | Operation | +| --- | --- | +| `generative` | Ordinary prediction on the selected model interface, even when the artifact also has policy metadata. | +| `embedding` | Extract configured representations using the existing embedding path. | +| `policy` | Execute the selected interface's resolved policy binding. Fail if that capability is absent. | + +There is no operation default to discover from an artifact. Missing `model_type` is a normal required-field error. Interface inference still follows the unique-route rule; unsupported formats and ambiguous routes produce explicit errors. Adding policy metadata cannot change what `model_type: generative` does. + +Keep the reference field locations and defaults for `project_root`, `model_path`, `data_path`, `training_config_path`, `dataset`, `part`, `model_interface`, read/write formats, device, batching, sampling, and media inputs. Share common inference fields rather than maintaining independent policy-only defaults. Retain top-level `output_path` for an explicit policy prediction destination, along with output-format and probability controls and default derived paths. + +For `model_type: policy`, keep `desired_return`, `desired_return_column`, named `previous_action_columns`, `sequence_id_column`, and `item_position_column`. A single previous action uses a one-entry named map. Retain independent-row and ordered-sequence inference, fragment-start previous actions, and recorded action history. Validate missing fragment context and incompatible return conditioning rather than silently resetting history or discarding the desired return. + +Use policy `action_selection: deterministic | stochastic` for action choice, consistently with online evaluation; use `enforce_determinism` for runtime algorithm determinism. Preserve seed-controlled stochastic sampling, categorical probability outputs, and joint-action probability/log-probability outputs with their existing meanings. Generative sampling/dropout and embedding output options remain available. Consolidating defaults must not remove either action-selection mode or make stochastic sampling depend on disabling runtime determinism. + +Refactor `load_inferer_config` around the reference resolution order: + +1. Compose YAML and apply supported CLI overrides. +2. Validate the explicit operation and authored route/field shapes. +3. Load the selected training route and/or portable artifact through a shared cache keyed by normalized path, independent of the chosen operation. +4. Resolve interface and metadata; explicit values are assertions against those contracts. +5. Validate operation capabilities and build the resolved inference specification. + +Fix the PT cache issue as part of this loader: prediction must not depend on a policy-routing helper having populated `artifacts[model_path]`. Require the declared policy operation to resolve a usable policy interface; no fallback to generative prediction. Extend the existing inference models with a typed operation-specific specification and share resolution services instead of dispatching between unrelated public constructors. + +## 5. Rollout: explicit type and local collection budget + +Use a discriminated `rollout.type: forecast | policy`, following the reference's `type` convention for components. Both variants share `adapter`, `trajectory_path`, `seed_index`, positive `max_steps`, and optional positive `max_episode_steps`. + +```yaml +project_root: . +model_path: models/controller.pt +model_type: policy +data_path: data/initial_observations.csv +read_format: csv +device: cpu +rollout: + type: policy + adapter: + factory: my_control:make_environment + implementation_id: control-v1 + config: {} + seed_index: 0 + max_steps: 200 + trajectory_path: outputs/control-trajectory.json +``` + +`model_type` selects the inference operation; `rollout.type` declares the interaction contract. Validate `forecast` with `generative` and `policy` with `policy`; reject embeddings with a rollout. Do not make either discriminator silently override the other. + +Reuse a shared adapter-factory base with `factory` and `config`, plus distinct forecast/online schemas. Forecast adds input `bindings`; online policy adds implementation identity, units, and capabilities. The type fixes the protocol: forecast uses protocol 1, online policy uses protocol 3. Keep protocol validation internal; users need not repeat a fixed `protocol_version` in YAML. Use the same online adapter schema in rollout, training `environment`, and online evaluation. + +Keep the two runtime protocols distinct: forecast reset receives prepared history and steps return external inputs; online policy reset/step supplies observations and exposes snapshot/group-start capabilities. Share loading, resource cleanup, limit validation, and meaningful common vocabulary. + +Define the collection controls precisely: + +- `max_steps` limits transitions collected by this invocation. Reaching it alone is a collection cutoff. +- `max_episode_steps` is an optional environment time limit with truncation semantics. True termination remains distinct; preserve correct bootstrapping when limits coincide. +- Forecast `seed_index` selects a prepared window in dataset/chunk order. Policy `seed_index` selects an initial observation row. Document the seed contract under each explicit rollout type. + +An environment rollout requires no `autoregressive` or top-level `generation_steps`. Reject those fields when `rollout` is present, so there is only one collection budget. Standalone autoregressive prediction keeps the reference `autoregressive`/`generation_steps` controls. + +Pass the local budget through forecast/policy inference, collectors, trajectory metadata, and public rollout APIs. Keep the collected data formats faithful to their forecast or policy semantics. + +## 6. Resolution, execution, and public APIs + +Reuse the existing pipeline and extend its typed results: + +```text +compose YAML + supported CLI overrides + -> expand singleton conveniences + -> validate authored SequifierConfig / InferenceConfig + -> resolve model, metadata, sources, bindings, and capabilities + -> ResolvedSequifierConfig / ResolvedInferenceConfig + -> phase coordinator and native algorithm runners +``` + +Singleton expansion is deterministic, non-mutating, and structural. It infers only unique routes, rejects singular/plural mixtures, and never performs artifact reads or chooses an execution implementation. Resolve an artifact-backed unique interface after resource loading, with the same ambiguity rule. + +Retain `additional_config_paths`, project-root-relative paths, duplicate-field rejection for complementary fragments, list replacement for overlays, and explicit assertions against artifact/metadata contracts. Extend discriminator replacement to phase `algorithm`, dataset-part `type`, rollout `type`, and operation-specific inference fields, so stale fields cannot survive a variant change. + +Extend HPO's same-path parameter tree and singleton expansion using the actual authored model. Validate each materialized trial; do not create another mirrored schema. Make algorithm/search capability limits explicit before trials start. + +Use `LoadedTrainConfig`, `ResolvedTrainingPhase`, `SelectedInterfaceConfig`, dataset build views, and `ExecutionSchema` as foundations. Extract neutral shared definitions where necessary to avoid circular imports; do not add a new public run-envelope class merely to carry internal runner arguments. + +Refactor the phase coordinator to consume the resolved `training_plan`. Preflight all source/interface/environment and execution-mode requirements before any optimizer update or output publication. Reuse native supervised and policy runners behind internal typed phase specifications; remove public `PolicyRunConfig`/`LearningRunConfig` dispatch. Retain supported supervised distributed execution, component initialization, exports, integrations, and optimizer plans as product capabilities. Unsupported mixed execution combinations fail with the relevant phase/capability named. + +Preflight restrictions may describe combinations the capability baseline does not support; they cannot be used to remove a combination that already works. Retain direct Python operations for model/policy loading, attachment, training, evaluation, collection, trajectory import/export, preference scoring, and replay/inspection through the new shared types. Configuration cleanup must not make these operations CLI-only. Keep the sequence-import preprocessing operation, sharding controls, source readers, and forecast/policy trajectory save/load/validation facilities; refactor their authored contracts to reuse the same source and binding types. + +Use one recovery contract owned by `global_training.resume`. Current-design checkpoints record phase position, committed optimizer clocks, RNG, role optimizers/schedulers/scalers, source/model/binding identities, selected handoff state, and necessary environment state. Validate exact versus reset-environment recovery and prevent duplicate phase execution. Test recovery of runs created by this design; no checkpoint conversion or older-format support is required. + +Update CLI and Python entry points together. Expose the same authored and resolved types through the existing public API modules. Remove superseded config classes, alternate selectors, scalar-action branches, and source registries rather than wrapping them. + +## 7. Implementation sequence and acceptance gates + +Before implementation, create a field-by-field capability inventory against `082225833e1d744e662d094008bcc44268ab868a`. Inspect public config models, CLI commands/overrides, Python API exports, readers, runners, and examples. For each functional option, record its new authored location or derived execution owner, supported combinations, and an executable acceptance test. An internal fact may become resolved state; a user-controllable behavior needs a new authored home. Retired duplicate spellings are not separate capabilities. The matrix below establishes minimum coverage and must be expanded if the inventory finds additional functions. + +| Current capability family | New owner / required expression | Completion evidence | +| --- | --- | --- | +| Raw tables, trajectories, source manifests, prepared splits, media/static inputs | Typed dataset parts with direct locations and optional/required metadata according to encoding. | Raw-source training without a generated metadata file; manifest and prepared-split execution; static/media resolution. | +| Episode/transition semantics and forecast-derived actions | Part column mappings, end declarations, namespaces, units, encoding, and `action_semantics`. | Correct history, terminal/bootstrap semantics, episode identities, and forecast-trajectory import. | +| Explicit and automatic validation | Phase `validation_sources` / `validation_preferences`, metadata partitions, `global_training.validation_fraction`. | Separate holdout execution; deterministic episode/component splits; overlap rejection. | +| Bounded episode loading | `global_training.episode_cache_size` and existing data-loader/memory controls. | Cache limit respected without changing episode content or partitions. | +| All supervised objectives/components and training policies | Existing model/global/dataset fields, phase selection/interleaving, curriculum, losses, hashing, initialization/freezing. | Supported model/objective fixtures, loss/gradient checks, and multi-dataset execution. | +| Imitation, IQL, PPO, DPO, GRPO and their optimization roles | Typed phases retaining every applicable budget, objective parameter, optimizer/scheduler and freeze control. | Each algorithm executes a meaningful update with representative non-default controls. | +| References, scoring, archives, and collection memory/step caps | Phase reference/scoring/archival/limit fields. | Phase-start/artifact references, chunked score agreement, archive score round trip, enforced limits. | +| Single categorical, return-conditioned, joint and real policies | Named bindings, explicit model inputs, existing action distribution fields. | History/return encoding, sampling, likelihood/gradients, update, and replay tests. | +| Policy inference controls | Explicit `model_type`, `selection`, desired-return/history/sequence columns, `output_path`, probabilities. | Independent rows and fragments, deterministic/stochastic actions, custom output and probability checks. | +| Prediction, embeddings, dropout and supported artifact/device routes | Existing inference/export fields and explicit operation. | PT/ONNX and supported device/mode tests; prediction independent of policy metadata. | +| Forecast and policy environment interaction | Typed rollouts/adapters, units/capabilities, local budgets, seed and episode controls. | Both protocols, snapshot/group-start capabilities, truncation/cutoff/cleanup checks. | +| Output and checkpoint control | Run output paths/export settings; global resume/checkpoint destinations and intervals; phase intervals/archives. | Custom destinations used, snapshot timing/rolling checkpoint behavior, report/phase artifact checks. | +| Recovery and mixed-phase handoff | Global resume mode, full phase state and identities, phase handoff/optimizer policy. | Same-design stop/resume, exact/reset environments, supervised → policy → supervised with no duplicate updates. | +| Public CLI/Python workflows and sequence import | Shared authored/resolved models and existing operation entry points. | Equivalent CLI/Python execution; import/sharding, direct evaluation/collection and trajectory operations. | +| Composition, HPO, integrations and supported distribution | Reference composition/search trees, runtime services, integration APIs. | Supported overrides/trials/integrations/distribution remain executable through the unified contract. | + +Each inventory entry must link its acceptance test before its implementation slice is complete. The final release gate requires no uncovered or removed baseline capability. New field names, defaults, explicit selectors, and derived metadata ownership may differ; functional expressiveness and supported operations must remain. + +| Slice | Implementation and principal locations | Acceptance gate | +| --- | --- | --- | +| 1. Reference contract and authored models | Complete the capability inventory; extend `SequifierConfig`, `TrainingPlanModel`, inference models, singleton expansion, and neutral shared definitions in `config/`. Write schema/examples first. | Every inventoried control has an owner; supervised/policy singleton and named configs use one model; public schemas contain no execution metadata or competing selectors. | +| 2. Dataset parts and shared references | Extend dataset parts/resolution, source and preference readers, validation partitioning, cache controls, source references and evaluation models. | Shared references resolve each data type; direct raw paths/manifests work without unnecessary metadata; explicit/automatic holdouts and bounded caching execute. | +| 3. Binding and execution resolution | Add authored/resolved bindings; update attachment, execution schemas, artifact config/export and shared materialization. Remove scalar-action execution selection only after coverage. | Single-categorical/return-conditioned and joint/real policies retain input, likelihood, update and replay capabilities; attachment and new artifact round trips work. | +| 4. Training lifecycle | Update training/coordinator/algorithm runners, all phase controls, optimizer/freezing services, output/checkpoint controls, global resume and API exports. | Each algorithm and references/archives/limits work; policy-only and mixed plans execute; custom outputs, handoff, role ownership and recovery pass their inventory tests. | +| 5. Inference and rollout | Extend explicit `model_type`, shared artifact caching, complete policy inference options, typed rollout/adapter schemas and public APIs. | Custom outputs, probabilities, return/fragment history and both action-selection modes work; generative routing is stable; both rollout contracts execute. | +| 6. Tooling, docs, and cleanup | Update composition, HPO, imports/CLI/Python operations, templates, examples/guides and CI; delete superseded public/runtime branches after coverage. | Every inventory entry has a passing acceptance test; all baseline capabilities remain expressible/executable; examples and shared CLI/Python contracts validate. | + +Dependencies: establish the authored shape in slice 1 before implementing resolution. Dataset resolution and bindings converge before complete training integration. Inference/rollout can build on binding resolution while training lifecycle work proceeds. Include recovery identities and fixtures during slice 4, not after public APIs are finished. + +## 8. Validation and completion criteria + +Keep tests in the adjacent `../sequifier-test` repository: schema and validation tests under `local/unit`, execution tests under `local/integration`, and shared fixtures in `local/conftest.py`. Reuse its existing conventions and validate the intended behavior of the new design. + +| Area | Required checks | +| --- | --- | +| Ergonomics and structure | Supervised singleton example from the reference; singleton policy run; named mixed run; independent singleton conveniences; unique-route inference; singular/plural conflicts; no extra routing/version flags. | +| Authored/resolved boundary | JSON schemas exclude internal fields; authored structural validation performs no file/environment loading; unknown fields and invalid algorithm-specific budgets fail at useful paths. | +| Dataset resolution | Raw table/trajectory paths without preprocessing metadata when resolvable; manifests/prepared splits; part eligibility; encoding/static/media/units; explicit versus automatic validation and configurable fractions; episode/preference leakage; episode-cache bounds; evaluation source/monitor validation. | +| Policy execution | Single-categorical behavior cloning and return conditioning; categorical IDs/start values; fragment/padded history; real/joint distributions and atoms; decoder order; fresh/initialized models; sampling, likelihood, gradients, update and replay after artifact reload. | +| Training | Each algorithm's full control inventory, budgets and roles; reference selection; chunked scoring; partial rollouts; archives/scores and buffer/step limits; dataset/phase freezing; interleaving/curriculum; handoff and optimization lifecycle. | +| Runtime capabilities | Supervised distribution/integrations and image/depth/static inputs; clear preflight errors for unsupported combinations; no source-shape-based runner selection. | +| Inference | Explicit operations and shared PT cache; independent-row/sequence/fragment inputs; desired-return/history column overrides; deterministic/stochastic selection independent of runtime determinism; custom output paths, probability outputs, dropout/embeddings and supported artifact/device routes. | +| Rollout | Forecast/online protocol validation; seed preparation for each type; budget 1 and invalid limits; termination/truncation/cutoff/bootstrap; adapter close on failure; no redundant generation controls. | +| Recovery | Uninterrupted versus resumed updates; phase-boundary commits; exact/reset-environment recovery; changed source/binding/adapter identities rejected; no repeated completed phase. | +| Outputs and checkpoints | Custom model/report/phase destinations; checkpoint destination with resume enabled/disabled; phase and timed/batch/epoch/latest intervals; archive paths; export/dropout settings and input/output collision checks. | +| Tooling | Composition/variant replacement; singleton/named HPO paths; CLI/Python operation coverage; sequence import/sharding; direct collection/evaluation and trajectory/preference inspection; all inventory entries linked to tests. | + +Use deterministic tiny CPU models and toy adapters for integration tests. Include actual artifact-backed inference, one mixed phase run, and stop/resume checks. Test encoding, gradients/log probabilities, and observable outputs rather than relying on schema parsing alone. + +Run focused pytest suites, relevant training/inference/rollout checks, Ruff and type checks. Update `README.md`, `documentation/configs/{train,infer,hyperparameter-search}.md`, policy/supervised-policy/rollout guides, templates and example configs; regenerate `src/sequifier/consolidated-docs.md` with `hooks/consolidate_docs.py`. + +The design is complete when a user familiar with the reference commit can add policy data and phases using the same ownership, naming, metadata, source-reference, and singleton rules, and every capability inventoried from `082225833e1d744e662d094008bcc44268ab868a` remains expressible and passes its acceptance test. The additional concepts should express the task itself: action distributions, interaction contracts, and algorithm-specific budgets. They should not expose implementation paths or introduce another way to describe a run. diff --git a/documentation/rollout.md b/documentation/rollout.md index 93af3d0a..c5d53801 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -17,15 +17,12 @@ Add this to an otherwise valid inference configuration for a model with inputs ```yaml device: cpu model_type: generative -autoregressive: true -generation_steps: 3 rollout: - mode: forecast + max_steps: 3 seed_index: 0 trajectory_path: outputs/trajectory.json - adapter: + environment: factory: your_application.adapters:make_environment - protocol_version: 1 config: {} # keyword arguments accepted by your factory bindings: - {input: x, source: prediction, key: x} @@ -153,3 +150,25 @@ save_trajectory(trajectory, config.trajectory_path) Multimodal and categorical observations, reset-only initialization, vectorized environments, remote transport, learning algorithms, and new export support are outside this slice. + +## Policy environments and reproducibility + +Policy training, online evaluation and policy rollout use `environment` with +`factory`, `config`, `implementation_id`, and `units` (empty means unitless). +This is a version-3 online environment, distinct from the forecast version-1 +protocol above. Policy rollout requires seeded observation reset; training +requires ordinary reset. Exact resume requires snapshot/restore capabilities. +GRPO also requires reproducible group starts. See the +[policy protocol reference](policy-learning.md#online-adapters). + +The inference route selects the rollout schema. Policy rejects forecast +`bindings`; forecast rejects policy implementation/units declarations. Neither +schema accepts a rollout discriminator. Both use `max_steps`, `trajectory_path`, +`seed_index`, and optional `max_episode_steps`. Forecast `seed_index` indexes +prepared windows; policy `seed_index` indexes raw input rows before sequence +history preparation. Both indices are zero-based and out-of-range values fail +before environment creation. + +Use `enforce_determinism` for execution and policy `action_selection` for mode +versus sampling. A fixed seed can make stochastic actions reproducible under +the same supported runtime and ordering. diff --git a/documentation/supervised-policy-learning.md b/documentation/supervised-policy-learning.md index 769f0dc1..312af8ce 100644 --- a/documentation/supervised-policy-learning.md +++ b/documentation/supervised-policy-learning.md @@ -14,21 +14,23 @@ initialize a policy run. Its interface does not need existing policy metadata. project_root: . model_name: posttrained initialize_from: models/supervised/model.pt -policy_interface: main policy_binding: actions: mode: {type: categorical, values: ["off", "on"]} - price: {type: real, scale: 0.1} - quantity: {type: real, scale: 0.5} -sources: + price: {type: real, scale: 0.5} +dataset_training: demonstrations: - paths: [data/demonstrations.parquet] - format: table -phases: - - name: imitation - algorithm: policy_imitation - sources: [demonstrations] - epochs: 5 + parts: + recorded: {type: sequence, paths: [data/demonstrations.parquet], format: table} +training_plan: + phases: + - name: imitate + algorithm: policy_imitation + sources: [{source: demonstrations.recorded}] + epochs: 5 + batch_size: 64 + optimization: + actor: {learning_rate: 0.0003} ``` Use the actual interface and target names from the source artifact. Quote YAML @@ -128,70 +130,55 @@ input. Offline conditioning needs a known reward tail. Online conditioning needs ## Mixed training plans -`run_type: unified` accepts ordinary supervised settings and policy sources in -one ordered plan: +Supervised and policy phases use the same `training_plan.phases` structure. +See the full [supervised-to-policy template](../src/sequifier/templates/train-supervised-to-policy.yaml). +Supervised prepared parts and policy sequence parts both belong to +`dataset_training`; each phase references the appropriate parts. No additional +run discriminator is needed. `global_training` owns execution and model/data +contracts. Each phase owns batching and `optimization.`: ```yaml -run_type: unified -project_root: . -model_name: mixed_run -initialize_from: models/supervised/model.pt -policy_interface: main -policy_bindings: - main: - actions: - quantity: {type: real, scale: 0.5} -global_training: - training_objective: causal - context_length: 32 - batch_size: 32 - inference_batch_size: 32 - learning_rate: 0.0003 -dataset_training: - supervised: - model_interface: main - parts: - train: {metadata_config_path: configs/metadata.json} - criterion: {quantity: MSELoss} -sources: - demonstrations: {paths: [data/demonstrations.parquet], format: table} -phases: - - name: supervised_before - algorithm: supervised - mode: sequential - sources: [{source: supervised.train}] - epochs: 2 - - name: policy - algorithm: policy_imitation - sources: [demonstrations] - epochs: 2 - - name: supervised_after - algorithm: supervised - mode: sequential - sources: [{source: supervised.train}] - epochs: 1 -resume: {policy: if_exists, mode: reset_environments} +training_plan: + phases: + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: [{source: observations.train}] + batch_size: 64 + accumulation_steps: 1 + optimization: + model: {optimizer: {name: Adam}, learning_rate: 0.001} + - name: imitation + algorithm: policy_imitation + epochs: 5 + sources: [{source: demonstrations.recorded}] + batch_size: 64 + accumulation_steps: 1 + optimization: + actor: {optimizer: {name: Adam}, learning_rate: 0.0003} ``` -Replace the middle phase with IQL, PPO, DPO, or GRPO using that algorithm's -ordinary configuration. A policy phase may select `model_interface`; otherwise -it uses `policy_interface`. All interfaces remain in the model artifact. -Updates to a shared backbone can change other interfaces' predictions. - -The coordinator preflights source and environment contracts, commits incoming -artifact identities, and hands each phase to its native runner. Optimizers, -critics, references, and rollout state are phase-local. `handoff: best` or `last` -selects the complete actor, including exploration parameters. Native checkpoints -recover in-phase progress; atomic phase receipts recover completed handoffs. -Mixed plans currently execute locally. Existing standalone supervised distributed -execution remains available. Exact online resume still requires the adapter's -snapshot capabilities. - -The Python API exposes `LearningRunConfig` and `run_learning_plan` from -`sequifier.training_api`, and `Policy`, `PolicyBinding`, `PolicyConfig`, -`ActionFieldConfig`, and `load_policy` -from `sequifier.rl_api`. `load_policy(..., binding=...)` attaches a task directly -to an ordinary supervised artifact. +The coordinator preflights model, source and environment contracts and commits +incoming artifacts before native phase execution. Phase-local checkpoints recover +active work; atomic receipts recover completed handoffs. Policy `model_interface` +selects an interface when source references are insufficient to identify it. +All interfaces remain in the model artifact; a shared backbone update can change +other interfaces' predictions. + +`optimizer_state` defaults to `reset`. Mixed-plan continuation is rejected because +artifact handoffs do not preserve optimization state. Native supervised-only and +compatible policy-only plans support explicit continuation under the +[training transition rules](configs/train.md#optimization-across-phases). +`handoff: last | best` selects phase weights; continuation requires the predecessor's +`last` handoff. Mixed plans execute locally. Exact online recovery requires the +adapter's snapshot capabilities. + +The Python API exposes `SequifierConfig`, `ResolvedSequifierConfig`, and +`run_learning_plan` from `sequifier.training_api`. The common entry point +`sequifier.train.run_training` dispatches a resolved plan to its appropriate +runner. `load_policy(..., binding=...)` attaches a task directly to a supervised +artifact; policy bindings never author a second model architecture. ## Distribution-aware supervised losses and inference diff --git a/hooks/consolidate_docs.py b/hooks/consolidate_docs.py index ff918528..da497ceb 100644 --- a/hooks/consolidate_docs.py +++ b/hooks/consolidate_docs.py @@ -11,6 +11,8 @@ "documentation/configs/hyperparameter-search.md", "documentation/training/multi-gpu-training.md", "documentation/policy-learning.md", + "documentation/supervised-policy-learning.md", + "documentation/rollout.md", ] OUTPUT_FILE = "src/sequifier/consolidated-docs.md" diff --git a/release-notes/unreleased-phase-ownership.md b/release-notes/unreleased-phase-ownership.md new file mode 100644 index 00000000..d851bc6e --- /dev/null +++ b/release-notes/unreleased-phase-ownership.md @@ -0,0 +1,24 @@ +# Phase-owned training and explicit inference routes + +- Move batching and role-keyed optimization into each training phase. Supervised + uses `optimization.model`; policy roles remain algorithm-specific. Remove global + optimization fields and the global reset switch. +- Add phase `optimizer_state: reset | continue`, with reset as the default and + validated compatible native continuation. Reject mixed-plan continuation and + policy OneCycle continuation. Preserve supervised epoch, update and metric + scheduling; resolve epoch OneCycle chain budgets and validate batch budgets, including interleaving and dataset-boundary flushes. +- Store resolved phase settings in checkpoint/config fingerprints and native + artifacts needed for recovery. +- Validate policy inference directly against its own schema and reject fields + belonging to another route. CLI overrides use the same validation. +- Rename inference execution determinism to `enforce_determinism` and policy + inference/evaluation selection to `action_selection`. Seeded sampling and + deterministic execution remain separate controls. +- Use `rollout.environment` and select rollout schemas from `model_type`. + Forecast version-1 bindings and policy version-3 environments remain distinct. + Document prepared-window versus raw-row `seed_index` units. +- Update shipped examples, scaffolding, Python examples and consolidated docs. + CPU smoke tests cover native supervised and all policy algorithms, mixed + handoff/recovery, inference, rollout and configuration rejection. + +This is a breaking authoring change; obsolete placements and names are rejected. diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index 06cadb66..4c7f9793 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -1,9 +1,10 @@ # Multi-phase categorical policy training (unreleased) -- One `PolicyRunConfig`, CLI dispatch and Python `run_policy_training` entry point +- One `SequifierConfig` authoring schema with `training_plan.phases`, CLI dispatch and Python `run_training` entry point support ordered imitation, IQL, PPO, DPO and GRPO phases with natural budgets. - A frozen architecture/interface/codec/normalization/context/units contract spans - phases. Selected actors are copied at handoff; optimizer/critic state is local. + phases. Selected actors are copied at handoff; compatible native phases may explicitly + continue optimizer/critic state. Mixed plans reset optimization at handoffs. Best/last selection is explicit, ties retain the earliest best, and final export uses the final phase's selection. - DPO adds versioned reward-free preference spans, connected-component-safe @@ -24,12 +25,13 @@ phase configures optimizer roles under `optimization`; gradient validation and clipping precede all optimizer steps in a transaction. - Actor-only portable artifacts remain usable for inference, rollout and cross-run - initialization. Default output is `models//actor.pt`; phase reports and + initialization. Default output is `models//model.pt`; phase reports and selected actors use indexed directories. - Templates, examples, public exports and guides use the canonical schema. The pre-existing supervised training runtime remains independent. -Scope: one categorical action, scalar/categorical inputs, causal fixed histories, -local single-process FP32. Return conditioning supports imitation only. Text/LLM, -continuous/joint actions, images/depth, vector/distributed execution, reward models, -optimizer/critic inheritance and mid-group checkpoints remain deferred. +Policy bindings support categorical and real action heads, including joint actions, +through the shared model execution schema. Policy algorithms currently execute +locally; distributed supervised execution remains available. See the current +[phase ownership notes](unreleased-phase-ownership.md) and +[policy guide](../documentation/policy-learning.md) for the supported surface. diff --git a/release-notes/unreleased-supervised-policy-unification.md b/release-notes/unreleased-supervised-policy-unification.md index 4b46a7f7..c598bb96 100644 --- a/release-notes/unreleased-supervised-policy-unification.md +++ b/release-notes/unreleased-supervised-policy-unification.md @@ -17,7 +17,7 @@ - Write model artifact version 3 and typed policy trajectory/collection version 2; retain readers for earlier model artifacts and single-categorical interfaces. -Tests and training experiments are intentionally deferred. See -[the usage guide](../documentation/supervised-policy-learning.md) and -[the implementation plan](../documentation/supervised-policy-unification-plan.md) -for the acceptance criteria that remain to be verified. +Focused CPU tests now cover phase-owned optimization, all policy algorithms, +checkpoint recovery, mixed handoffs and forecast/policy inference and rollout. +See [the usage guide](../documentation/supervised-policy-learning.md) and +[phase ownership notes](unreleased-phase-ownership.md) for the implemented contracts. diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index fbbf70b7..5117d87e 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -103,12 +103,18 @@ def build_model_artifact( f"{interface_name}:{site.name}" for site in network.trace_catalog_for(interface_name) ) + resolved_provenance = dict(provenance or {}) + phases = getattr(config, "training_plan", None) + if phases: + resolved_provenance["training_plan"] = [ + phase.model_dump(mode="json") for phase in phases + ] artifact = ModelArtifact( format_version=MODEL_ARTIFACT_FORMAT_VERSION, model_config=ModelExecutionConfig.from_training_config(config), model_state_dict=state, metadata=ModelArtifactMetadata( - trace_sites=tuple(trace_sites), provenance=dict(provenance or {}) + trace_sites=tuple(trace_sites), provenance=resolved_provenance ), ) artifact.validate() diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 4d26c639..99c86bf3 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.train_config import ( BackboneComponentConfig, DatasetFreezingSpecModel, @@ -107,8 +107,6 @@ def resolved_config_from_model_config( context_length=context_length, target_offset=target_offset, inference_batch_size=1, - batch_size=1, - learning_rate=1e-3, layer_type_dtypes=values.get("layer_type_dtypes"), next_occurrence_config=values.get("next_occurrence_config"), torch_compile="none", @@ -143,14 +141,6 @@ def resolved_config_from_model_config( if key in ModelInterfaceSpecModel.model_fields } policy_spec = interface.get("policy") - if policy_spec is not None: - authored["policy"] = PolicySchemaConfig.model_validate( - { - key: value - for key, value in policy_spec.items() - if key in PolicySchemaConfig.model_fields - } - ) authored_interfaces[name] = ModelInterfaceSpecModel.model_validate(authored) target_decoder_ids = interface.get("target_decoder_ids", {}) target_n_classes = interface.get( @@ -165,7 +155,7 @@ def resolved_config_from_model_config( if key in ResolvedModelInterface.model_fields }, "name": name, - "policy": CategoricalPolicyConfig.model_validate(policy_spec) + "policy": ResolvedPolicyBinding.model_validate(policy_spec) if policy_spec is not None else None, "n_classes": interface.get("n_classes", target_n_classes), diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index ae023aec..56ee833d 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -38,12 +38,6 @@ _MISSING = object() _DISTRIBUTION_KEYS = frozenset({"low", "high", "step", "log", "type"}) _PRIMITIVE_CATEGORICAL_TYPES = (str, int, float, bool, type(None)) -_NAME_DISCRIMINATOR_PATHS = frozenset( - { - ("global_training", "optimizer"), - ("global_training", "scheduler"), - } -) @beartype @@ -293,7 +287,11 @@ def spaces(self) -> list[_SearchSpace]: def _component_discriminator(path: ConfigPath) -> str | None: """Return the discriminator for a typed component mapping, if any.""" - if path in _NAME_DISCRIMINATOR_PATHS: + if ( + len(path) >= 3 + and path[-1] in {"optimizer", "scheduler"} + and "optimization" in path + ): return "name" if path and path[-1] in {"weight", "bias"} and "initialization" in path: return "method" @@ -872,7 +870,9 @@ def compile_canonical_hyperparameter_search_config( f"referenced by {config_path!r}: {error}" ) from error - if not isinstance(base_config, SequifierConfig): + if not isinstance(base_config, SequifierConfig) or any( + phase.algorithm != "supervised" for phase in base_config.training_plan.phases + ): raise ValueError( "hyperparameter search for trajectory training is not yet supported" ) diff --git a/src/sequifier/config/components.py b/src/sequifier/config/components.py index 0ed26d3b..8b025d21 100644 --- a/src/sequifier/config/components.py +++ b/src/sequifier/config/components.py @@ -1096,6 +1096,7 @@ class ResumeConfig(BaseModel): policy: Literal["never", "if_exists", "required"] = "never" checkpoint_path: Optional[str] = None + mode: Literal["exact", "reset_environments"] = "reset_environments" @model_validator(mode="after") @beartype diff --git a/src/sequifier/config/composition.py b/src/sequifier/config/composition.py index 026b4c89..ee713d8b 100644 --- a/src/sequifier/config/composition.py +++ b/src/sequifier/config/composition.py @@ -21,8 +21,6 @@ DEFAULT_ATOMIC_PATHS: frozenset[ConfigPath] = frozenset( { ("model", "backbone"), - ("global_training", "optimizer"), - ("global_training", "scheduler"), ("global_training", "bert_spec", "span_masking"), } ) @@ -327,7 +325,7 @@ def _deep_merge_dicts( def _changes_discriminator( base: Mapping[str, Any], override: Mapping[str, Any] ) -> bool: - for discriminator in ("type", "name"): + for discriminator in ("type", "name", "algorithm", "model_type"): if ( discriminator in base and discriminator in override diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 6eeb4086..cb59f9ab 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -1,30 +1,25 @@ import json import os import re -from typing import Any, Generic, Literal, Optional, TypeVar, Union +from typing import Any, Generic, Optional, TypeVar, Union import numpy as np -from pydantic import ( - BaseModel, - ConfigDict, - Field, - PrivateAttr, - field_validator, - model_validator, -) +from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator +from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.config.composition import ( load_composed_yaml_config, merge_config_fragments, ) from sequifier.config.image_config import MediaPath, bind_image_paths, image_views +from sequifier.config.inference_common import InferenceExecutionConfig from sequifier.config.metadata import ( DatasetMetadata, extract_inline_metadata, load_dataset_metadata, ) from sequifier.config.policy_infer_config import ( - PolicyInferenceConfig, + ResolvedPolicyInferenceConfig, policy_inference_config, ) from sequifier.config.rollout_config import RolloutConfig @@ -393,7 +388,7 @@ def _assert_folder_metadata_matches( @beartype def load_inferer_config( config_path: str, args_config: dict, skip_metadata: bool -) -> "ResolvedInferenceConfig | PolicyInferenceConfig": +) -> "ResolvedInferenceConfig | ResolvedPolicyInferenceConfig": """Compose inference YAML, load model defaults, and resolve metadata.""" config_values = load_composed_yaml_config(config_path) @@ -402,7 +397,26 @@ def load_inferer_config( key: value for key, value in args_config.items() if key != "skip_metadata" } authored_values = merge_config_fragments((config_values, cli_values)) + if authored_values.get("model_type") == "policy": + from sequifier.config.policy_infer_config import PolicyInferenceConfig + + PolicyInferenceConfig.model_validate(authored_values) + if skip_metadata: + raise ValueError("skip_metadata is not supported for policy inference") + else: + forecast_values, _ = extract_inline_metadata(authored_values) + InferenceConfig.model_validate(forecast_values) artifacts = {} + artifact_paths = authored_values.get("model_path", []) + if isinstance(artifact_paths, str): + artifact_paths = [artifact_paths] + by_path = {} + for path in artifact_paths: + if path.lower().endswith(".pt"): + normalized = normalize_path(path, authored_values.get("project_root", ".")) + if normalized not in by_path: + by_path[normalized] = read_execution_artifact(normalized) + artifacts[path] = by_path[normalized] policy_config = policy_inference_config(authored_values, artifacts) if policy_config is not None: return policy_config @@ -411,7 +425,7 @@ def load_inferer_config( ) inline_metadata_values = extracted_metadata_values if skip_metadata else None - project_root = authored_values["project_root"] + project_root = authored_values.get("project_root", ".") selected_metadata = None selected_metadata_path = None artifact_metadata_source = None @@ -440,12 +454,6 @@ def load_inferer_config( from sequifier.config.train_config import load_train_config training = load_train_config(training_config_path, {}, False) - from sequifier.config.train_config import PolicyRunConfig - - if isinstance(training, PolicyRunConfig): - raise ValueError( - "policy inference requires model_path pointing to its portable artifact" - ) datasets = training.dataset_training if dataset_name is not None: if dataset_name not in datasets: @@ -661,6 +669,9 @@ def resolve_inference_config( ) -> "ResolvedInferenceConfig": """Return an inference config with all metadata-derived values populated.""" + if config.context_length is None or config.training_objective is None: + raise ValueError("Inference requires context_length and training_objective") + if config.prediction_alignment_steps != metadata.prediction_alignment_steps: raise ValueError( "Inference config prediction_alignment_steps does not match dataset " @@ -893,30 +904,27 @@ def resolve_inference_config( _DataPathT = TypeVar("_DataPathT") _InputColumnsT = TypeVar("_InputColumnsT") _ColumnTypesT = TypeVar("_ColumnTypesT") +_ObjectiveT = TypeVar("_ObjectiveT", bound=str | None) +_ContextLengthT = TypeVar("_ContextLengthT", bound=int | None) class _InferenceConfigBase( - BaseModel, Generic[_DataPathT, _InputColumnsT, _ColumnTypesT] + InferenceExecutionConfig, + Generic[_DataPathT, _InputColumnsT, _ColumnTypesT, _ObjectiveT, _ContextLengthT], ): """Shared fields and validation for authored and resolved inference config.""" - execution_mode: Literal["prediction", "policy"] | None = None - model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - project_root: str preprocessing_data_path: Optional[str] = None metadata_config_path: Optional[str] = None model_path: Union[str, list[str]] model_type: str - training_objective: str + training_objective: _ObjectiveT data_path: _DataPathT training_config_path: Optional[str] = None dataset: Optional[str] = None part: Optional[str] = None - model_interface: Optional[str] = None - read_format: str = Field(default="parquet") - write_format: str = Field(default="csv") input_columns: _InputColumnsT static_columns: list[str] = Field(default_factory=list) @@ -927,22 +935,18 @@ class _InferenceConfigBase( column_data_types: _ColumnTypesT target_column_types: _ColumnTypesT - deterministic: bool = Field(default=False) output_probabilities: bool = Field(default=False) decode_categories: bool = Field(default=True) - seed: int = 1010 - device: str - context_length: int = Field(gt=0) + context_length: _ContextLengthT = Field(gt=0) target_offset: int = Field(default=1, ge=0) window_stride: Optional[int] = Field(default=None, gt=0) prediction_length: Optional[int] = None decoder_support: int = Field(default=1, gt=0) prediction_alignment_steps: int = Field(default=1, gt=0) - inference_batch_size: int = Field(default=1, gt=0) sample_from_distribution_columns: Optional[list[str]] = Field(default=None) infer_with_dropout: bool = Field(default=False) - rollout: Optional[RolloutConfig] = None + rollout: RolloutConfig | None = None autoregressive: bool = Field(default=False) generation_steps: Optional[int] = Field(default=None) @@ -971,8 +975,8 @@ def normalize_single_input_column(cls, value): @field_validator("training_objective") @classmethod @beartype - def validate_authored_training_objective(cls, value: str) -> str: - if value not in ALLOWED_OBJECTIVE_NAMES: + def validate_authored_training_objective(cls, value: str | None) -> str | None: + if value is not None and value not in ALLOWED_OBJECTIVE_NAMES: raise ValueError( f"Only {OBJECTIVE_NAME_MESSAGE} are allowed, found {value}" ) @@ -983,12 +987,14 @@ def validate_authored_training_objective(cls, value: str) -> str: @beartype def validate_authored_model_type(cls, value: str) -> str: if value not in {"embedding", "generative"}: - raise ValueError("model_type must be either embedding or generative") + raise ValueError("forecast model_type must be generative or embedding") return value @model_validator(mode="after") @beartype def validate_authored_paths(self): + if self.context_length is None or self.training_objective is None: + return self ModelWindowView( context_length=self.context_length, objective=self.training_objective, @@ -1000,16 +1006,44 @@ def validate_authored_paths(self): class InferenceConfig( - _InferenceConfigBase[Optional[str], Optional[list[str]], Optional[dict[str, str]]] + _InferenceConfigBase[ + Optional[str], + Optional[list[str]], + Optional[dict[str, str]], + str | None, + int | None, + ] ): """User-authored configuration for one inference run.""" + @model_validator(mode="after") + def validate_embedding_options(self): + generative_fields = { + "sample_from_distribution_columns", + "decode_categories", + "output_probabilities", + "autoregressive", + "generation_steps", + "rollout", + } + if self.model_type == "embedding" and ( + invalid := self.model_fields_set & generative_fields + ): + raise ValueError(f"embedding inference does not support {sorted(invalid)}") + return self + + training_objective: str | None = None + context_length: int | None = Field(default=None, gt=0) + input_columns: list[str] | None = None + target_columns: list[str] = Field(default_factory=list) data_path: Optional[str] = None column_data_types: Optional[dict[str, str]] = None target_column_types: Optional[dict[str, str]] = None -class ResolvedInferenceConfig(_InferenceConfigBase[str, list[str], dict[str, str]]): +class ResolvedInferenceConfig( + _InferenceConfigBase[str, list[str], dict[str, str], str, int] +): """Internal inference config after dataset metadata has been resolved.""" categorical_columns: list[str] @@ -1025,9 +1059,9 @@ class ResolvedInferenceConfig(_InferenceConfigBase[str, list[str], dict[str, str def validate_numeric_rollout(self): if self.rollout is None: return self - if not self.autoregressive or self.generation_steps is None: + if self.autoregressive or self.generation_steps is not None: raise ValueError( - "rollout requires autoregressive: true and generation_steps" + "rollout uses max_steps, not autoregressive/generation_steps" ) if self.device != "cpu" or self.model_type != "generative": raise ValueError("numeric rollout requires a CPU generative model") diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py new file mode 100644 index 00000000..c516e62b --- /dev/null +++ b/src/sequifier/config/inference_common.py @@ -0,0 +1,16 @@ +"""Execution options consumed by every inference route.""" + +from pydantic import BaseModel, ConfigDict, Field + + +class InferenceExecutionConfig(BaseModel): + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + project_root: str = "." + model_interface: str | None = None + read_format: str = "parquet" + write_format: str = "csv" + device: str = "cpu" + seed: int = 1010 + enforce_determinism: bool = False + inference_batch_size: int = Field(default=1, gt=0) diff --git a/src/sequifier/config/learning_run.py b/src/sequifier/config/learning_run.py deleted file mode 100644 index b0c529b3..00000000 --- a/src/sequifier/config/learning_run.py +++ /dev/null @@ -1,97 +0,0 @@ -"""A common authored run plan with supervised and policy phases.""" - -from pathlib import Path -from typing import Annotated, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator - -from sequifier.config.policy_config import PolicySchemaConfig -from sequifier.config.policy_plan import ( - DPOPhaseConfig, - GRPOPhaseConfig, - IQLPhaseConfig, - PolicyImitationPhaseConfig, - PolicyResumeConfig, - PPOPhaseConfig, -) -from sequifier.config.policy_train_config import SequenceSourceConfig -from sequifier.config.train_config import ( - DatasetTrainingSpecModel, - EvaluationSpecModel, - GlobalTrainingSpecModel, - ModelSpecModel, - TrainingPhaseSpecModel, -) - - -class SupervisedLearningPhase(TrainingPhaseSpecModel): - handoff: Literal["best", "last"] = "best" - - -LearningPhase = Annotated[ - SupervisedLearningPhase - | PolicyImitationPhaseConfig - | IQLPhaseConfig - | PPOPhaseConfig - | DPOPhaseConfig - | GRPOPhaseConfig, - Field(discriminator="algorithm"), -] - - -class LearningRunConfig(BaseModel): - """One model with typed sources and independently budgeted phase runners.""" - - model_config = ConfigDict(extra="forbid", allow_inf_nan=False) - run_type: Literal["unified"] = "unified" - project_root: str = "." - model_name: str = Field(min_length=1, pattern=r"^[^/\\]+$") - device: str = "cpu" - seed: int = Field(default=1010, ge=0, lt=2**32) - model: ModelSpecModel | None = None - initialize_from: str | None = None - global_training: GlobalTrainingSpecModel | None = None - dataset_training: dict[str, DatasetTrainingSpecModel] = Field(default_factory=dict) - evaluation: EvaluationSpecModel | None = None - policy_interface: str | None = None - policy_bindings: dict[str, PolicySchemaConfig] = Field(default_factory=dict) - sources: dict[str, SequenceSourceConfig] = Field(default_factory=dict) - preferences: dict[str, str] = Field(default_factory=dict) - units: dict[str, str] = Field(default_factory=dict) - phases: list[LearningPhase] = Field(min_length=1) - resume: PolicyResumeConfig = Field(default_factory=PolicyResumeConfig) - validation_fraction: float = Field(default=0.2, gt=0, lt=1) - episode_cache_size: int = Field(default=8, gt=0) - checkpoint_path: str = "" - model_path: str = "" - report_path: str = "" - phase_directory: str = "" - export_onnx: bool = False - - @model_validator(mode="after") - def validate_plan(self): - root = Path(self.project_root).resolve() - self.project_root = str(root) - if self.model_name in {".", ".."}: - raise ValueError("model_name must name one directory") - for key, default in ( - ("checkpoint_path", f"checkpoints/{self.model_name}/learning-run.pt"), - ("model_path", f"models/{self.model_name}/model.pt"), - ("report_path", f"outputs/training/{self.model_name}/report.json"), - ("phase_directory", f"outputs/training/{self.model_name}/phases"), - ): - setattr(self, key, str((root / (getattr(self, key) or default)).resolve())) - if self.initialize_from: - self.initialize_from = str((root / self.initialize_from).resolve()) - if self.model is None and self.initialize_from is None: - raise ValueError("a run requires a model or initialize_from") - if len({phase.name for phase in self.phases}) != len(self.phases): - raise ValueError("phase names must be unique") - if any(phase.algorithm == "supervised" for phase in self.phases): - if self.global_training is None or not self.dataset_training: - raise ValueError( - "supervised phases require global_training and dataset_training" - ) - if self.phases[0].algorithm != "supervised" and self.initialize_from is None: - raise ValueError("policy-first unified runs require initialize_from") - return self diff --git a/src/sequifier/config/optimizer_config.py b/src/sequifier/config/optimizer_config.py index 92110343..04f8d610 100644 --- a/src/sequifier/config/optimizer_config.py +++ b/src/sequifier/config/optimizer_config.py @@ -88,7 +88,7 @@ def resolve_plan_scheduler_arguments( if scheduler.name == "CyclicLR": if "base_lr" in arguments or "max_lr" in arguments: raise ValueError( - "global_training.scheduler.base_lr and max_lr must be omitted " + "optimization..scheduler.base_lr and max_lr must be omitted " "with an optimizer plan; set rates in each optimizer group." ) missing = [ @@ -116,7 +116,7 @@ def resolve_plan_scheduler_arguments( if scheduler.name == "OneCycleLR": if "max_lr" in arguments: raise ValueError( - "global_training.scheduler.max_lr must be omitted with an " + "optimization..scheduler.max_lr must be omitted with an " "optimizer plan; each group's learning_rate is its peak rate." ) arguments["max_lr"] = [group.learning_rate for group in plan.groups] diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 304b453f..be5554d8 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -80,115 +80,50 @@ def validate_distribution(self): return self -class PolicySchemaConfig(PolicyConfigBase): - alignment: Literal["current_observation_previous_action", "model_inputs"] = ( - "current_observation_previous_action" - ) - observation_columns: list[str] = Field(default_factory=list) - static_columns: list[str] = Field(default_factory=list) - categorical_columns: list[str] = Field(default_factory=list) - # Numeric trajectory fields may also carry already encoded categorical IDs. - observation_n_classes: dict[str, Annotated[int, Field(strict=True, ge=2)]] = Field( - default_factory=dict - ) - static_n_classes: dict[str, Annotated[int, Field(strict=True, ge=2)]] = Field( - default_factory=dict - ) - # The same label -> global ID convention as preprocessing metadata. - id_maps: dict[str, dict[str | int, int]] = Field(default_factory=dict) - categorical_encoding: Literal["global"] = "global" - action_column: str = Field(default="action", min_length=1) - action_values: list[DomainValue] = Field( - default_factory=lambda: [0.0, 1.0], min_length=2 - ) - actions: dict[str, ActionFieldConfig] = Field(default_factory=dict) - action_order: list[str] = Field(default_factory=list) +class PolicyBinding(PolicyConfigBase): + """A task attached to named targets of an ordinary model interface.""" + + actions: dict[str, ActionFieldConfig] = Field(min_length=1) decision_position: int = Field(default=-1, le=-1) - # Existing input -> lagged action; no implicit features in model_inputs mode. previous_action_inputs: dict[str, str] = Field(default_factory=dict) initial_inputs: dict[str, Any] = Field(default_factory=dict) - input_schema: dict[str, Any] | None = None - input_statistics: dict[str, dict[str, float]] = Field(default_factory=dict) - input_encoding: Literal["raw", "model"] = "raw" - return_conditioned: bool = True + return_conditioned: bool = False initial_desired_return: float | None = Field(default=None, allow_inf_nan=False) returns: ReturnDefinition = Field(default_factory=ReturnDefinition) + units: dict[str, str] = Field(default_factory=dict) @model_validator(mode="after") - def validate_names(self): - if self.action_order and ( - set(self.action_order) != set(self.actions) - or len(self.action_order) != len(self.actions) - ): - raise ValueError("action_order must name every action field exactly once") - names = self.observation_columns + self.static_columns - if any(not name for name in names) or len(names) != len(set(names)): - raise ValueError( - "policy observation/static columns must be nonempty and disjoint" - ) - reserved = {"__policy_previous_action", "__policy_return", "__policy_action"} - if self.alignment != "model_inputs" and reserved.intersection( - names + [self.action_column] - ): - raise ValueError("policy feature names use reserved internal columns") - for counts, columns in ( - (self.observation_n_classes, self.observation_columns), - (self.static_n_classes, self.static_columns), - ): - if counts.keys() - set(columns) - set(self.previous_action_inputs): - raise ValueError( - "class counts must name declared observation/static columns" - ) - counts = {**self.observation_n_classes, **self.static_n_classes} - if len(set(self.categorical_columns)) != len(self.categorical_columns) or set( - self.categorical_columns - ) - set(names) - set(self.previous_action_inputs): - raise ValueError( - "categorical_columns must name unique observation/static features" - ) - if set(self.id_maps) - set(counts): - raise ValueError("id_maps must name declared categorical inputs") - for column, mapping in self.id_maps.items(): - if ( - not mapping - or len(set(mapping.values())) != len(mapping) - or any( - value < 0 or value >= counts[column] for value in mapping.values() - ) - ): - raise ValueError(f"{column}: invalid categorical ID map") - if len(self.action_values) != len(set(self.action_values)): - raise ValueError("policy action_values must be unique") - if len({isinstance(value, str) for value in self.action_values}) != 1: - raise ValueError( - "action_values must use consistently numeric or string labels" - ) + def validate_input_bindings(self): + if any(not name for name in self.actions): + raise ValueError("actions require nonempty target names") + if set(self.previous_action_inputs.values()) - set(self.actions): + raise ValueError("previous_action_inputs must reference declared actions") + if set(self.initial_inputs) - set(self.previous_action_inputs): + raise ValueError("initial_inputs must name previous_action_inputs") return self - @property - def action_columns(self) -> list[str]: - return ( - (self.action_order or list(self.actions)) - if self.actions - else [self.action_column] - ) - - @property - def is_joint(self) -> bool: - return bool(self.actions) - @property - def categorical_n_classes(self) -> dict[str, int]: - return {**self.observation_n_classes, **self.static_n_classes} - - -class CategoricalPolicyConfig(PolicySchemaConfig): - """Resolved alignment metadata; architecture belongs to the ordinary model.""" +class ResolvedPolicyBinding(PolicyBinding): + """Fitted task binding persisted with a model's execution contract.""" context_length: int = Field(gt=0) interface_name: NonemptyStr + observation_columns: list[str] = Field(default_factory=list) + static_columns: list[str] = Field(default_factory=list) + categorical_columns: list[str] = Field(default_factory=list) + observation_n_classes: dict[str, int] = Field(default_factory=dict) + static_n_classes: dict[str, int] = Field(default_factory=dict) + id_maps: dict[str, dict[str | int, int]] = Field(default_factory=dict) + categorical_encoding: Literal["global"] = "global" + action_order: list[str] = Field(default_factory=list) + input_schema: dict[str, Any] + input_statistics: dict[str, dict[str, float]] = Field(default_factory=dict) + input_encoding: Literal["raw", "model"] = "raw" + @property + def action_columns(self): + return self.action_order or list(self.actions) -# General names for the shared contract; retain the original public names. -PolicyBinding = PolicySchemaConfig -PolicyConfig = CategoricalPolicyConfig + @property + def categorical_n_classes(self): + return {**self.observation_n_classes, **self.static_n_classes} diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 1a268cdf..a52eba3d 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -5,6 +5,7 @@ from pydantic import Field, PrivateAttr, model_validator from sequifier.artifacts.model_artifact import ModelArtifact, read_execution_artifact +from sequifier.config.inference_common import InferenceExecutionConfig from sequifier.config.policy_config import PolicyConfigBase from sequifier.config.policy_plan import OnlineEnvironmentConfig from sequifier.helpers import normalize_path @@ -12,60 +13,72 @@ class PolicyRolloutConfig(PolicyConfigBase): environment: OnlineEnvironmentConfig + max_steps: int = Field(gt=0) trajectory_path: str = Field(min_length=1) seed_index: int = Field(ge=0) max_episode_steps: int | None = Field(default=None, gt=0) -class PolicyInferenceConfig(PolicyConfigBase): - _artifacts: dict[str, ModelArtifact] = PrivateAttr(default_factory=dict) - - project_root: str = "." +class PolicyInferenceConfig(InferenceExecutionConfig): model_path: str - execution_mode: Literal["prediction", "policy"] | None = None - model_type: Literal["policy", "generative"] = "generative" - model_interface: str | None = None + model_type: Literal["policy"] data_path: str - read_format: Literal["csv", "parquet"] = "csv" - write_format: Literal["csv", "parquet"] = "csv" output_path: str | None = None output_probabilities: bool = False - inference_batch_size: int = Field(default=64, gt=0) - device: str = "cpu" - seed: int = 1010 - deterministic: bool = True + action_selection: Literal["deterministic", "stochastic"] = "deterministic" desired_return: float | None = Field(default=None, allow_inf_nan=False) desired_return_column: str = "desired_return" - previous_action_column: str | None = None previous_action_columns: dict[str, str] = Field(default_factory=dict) sequence_id_column: str = "sequenceId" item_position_column: str = "itemPosition" rollout: PolicyRolloutConfig | None = None - autoregressive: bool = False - generation_steps: int | None = Field(default=None, gt=0) + # Optional assertions against the portable artifact contract. + metadata_config_path: str | None = None + input_columns: list[str] | None = None + static_columns: list[str] | None = None + target_columns: list[str] | None = None + column_data_types: dict[str, str] | None = None + target_column_types: dict[str, str] | None = None + training_objective: str | None = None + context_length: int | None = Field(default=None, gt=0) + target_offset: int | None = Field(default=None, ge=0) + prediction_length: int | None = Field(default=None, gt=0) + decoder_support: int | None = Field(default=None, gt=0) @model_validator(mode="after") def validate_execution(self): - if self.rollout is not None: - if not self.autoregressive or self.generation_steps is None: - raise ValueError( - "policy rollout requires autoregressive: true and generation_steps" - ) - if self.output_probabilities: - raise ValueError( - "rollout writes a trajectory including action log probabilities" - ) - elif self.autoregressive or self.generation_steps is not None: - raise ValueError("policy generation requires an environment rollout") + if self.read_format not in {"csv", "parquet"} or self.write_format not in { + "csv", + "parquet", + }: + raise ValueError("policy inference formats must be csv or parquet") + if self.rollout is not None and self.output_probabilities: + raise ValueError( + "rollout writes a trajectory including action log probabilities" + ) return self +class ResolvedPolicyInferenceConfig(PolicyInferenceConfig): + """Validated policy request with selected artifact/interface and assertions.""" + + _artifacts: dict[str, ModelArtifact] = PrivateAttr(default_factory=dict) + + def policy_inference_config( values: dict, artifacts: dict[str, ModelArtifact] | None = None -) -> PolicyInferenceConfig | None: +) -> ResolvedPolicyInferenceConfig | None: """Dispatch from the persisted interface contract, without training YAML.""" - if values.get("execution_mode") == "prediction": + if values.get("model_type") != "policy": return None + from sequifier.artifacts.model_config import resolved_config_from_model_config + from sequifier.config.infer_config import ( + _assert_metadata_matches_source, + _execution_source, + ) + from sequifier.config.metadata import load_dataset_metadata + + request = PolicyInferenceConfig.model_validate(values) artifacts = {} if artifacts is None else artifacts model_path = values.get("model_path") paths = [model_path] if isinstance(model_path, str) else model_path or [] @@ -84,15 +97,46 @@ def policy_inference_config( selected is None or selected == name ): policy_routes.append(name) - if not policy_routes and values.get("model_type") != "policy": - return None if len(paths) != 1 or len(policy_routes) != 1: raise ValueError( "policy inference requires one artifact and one selected policy interface" ) - authored = dict(values) + selected = policy_routes[0] + execution, _ = resolved_config_from_model_config( + artifacts[paths[0]].model_config.values, + device=request.device, + interface_name=selected, + ) + policy = execution.interface(selected).policy + if policy is None: + raise ValueError("selected interface has no policy") + if ( + request.rollout is not None + and request.rollout.environment.units != policy.units + ): + raise ValueError("rollout environment units differ from policy artifact") + contract, metadata = _execution_source(execution, execution.dataset(selected)) + for key, actual in contract.items(): + if ( + key in request.model_fields_set + and getattr(request, key) is not None + and getattr(request, key) != actual + ): + raise ValueError(f"{key} differs from policy artifact") + if request.metadata_config_path: + explicit = load_dataset_metadata( + normalize_path(request.metadata_config_path, request.project_root) + ) + _assert_metadata_matches_source( + explicit, + metadata, + "policy metadata", + "storage_layout" + in artifacts[paths[0]].model_config.values["interfaces"][selected], + ) + authored = request.model_dump() authored["model_path"] = paths[0] - authored["model_interface"] = policy_routes[0] - config = PolicyInferenceConfig.model_validate(authored) + authored["model_interface"] = selected + config = ResolvedPolicyInferenceConfig.model_validate(authored) config._artifacts = artifacts return config diff --git a/src/sequifier/config/policy_optimization.py b/src/sequifier/config/policy_optimization.py index 6a048004..1532e925 100644 --- a/src/sequifier/config/policy_optimization.py +++ b/src/sequifier/config/policy_optimization.py @@ -1,4 +1,7 @@ -"""Concrete optimizer ownership and scheduler budgets for policy phases.""" +"""Shared role optimization specs with explicit algorithm scheduling constraints.""" + +import inspect +from typing import Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -18,7 +21,7 @@ def default_scheduler(): return ComponentSpec(name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1}) -class OptimizationConfig(BaseModel): +class RoleOptimizationConfig(BaseModel): """One authored optimizer/scheduler configuration per trainable role.""" model_config = ConfigDict(extra="forbid", allow_inf_nan=False) @@ -26,11 +29,23 @@ class OptimizationConfig(BaseModel): default_factory=default_optimizer, union_mode="left_to_right" ) scheduler: ComponentSpec = Field(default_factory=default_scheduler) - learning_rate: float = Field(default=3e-4, gt=0) - gradient_clip: float = Field(default=1.0, gt=0) + learning_rate: float | None = Field(default=None, gt=0) + gradient_clip: float | None = Field(default=1.0, gt=0) @model_validator(mode="after") def validate_components(self): + if isinstance(self.optimizer, OptimizerPlan): + if self.learning_rate is not None: + raise ValueError("optimizer plans set learning_rate in each group") + elif self.learning_rate is None: + self.learning_rate = 3e-4 + if self.scheduler.name == "OneCycleLR" and { + "epochs", + "steps_per_epoch", + }.intersection(self.scheduler.arguments): + raise ValueError( + "OneCycleLR uses the phase budget; configure total_steps instead of scheduler epochs/steps_per_epoch" + ) specs = ( [group.optimizer for group in self.optimizer.groups] if isinstance(self.optimizer, OptimizerPlan) @@ -42,26 +57,23 @@ def validate_components(self): raise ValueError( "optimizer ownership and learning rates must be configured explicitly" ) - if ( - self.scheduler.name - in {"ReduceLROnPlateau", "SequentialLR", "ChainedScheduler"} - or "optimizer" in self.scheduler.arguments - ): - raise ValueError( - "policy schedulers must advance independently on optimizer steps" + arguments = ( + resolve_plan_scheduler_arguments(self.optimizer, self.scheduler) + if isinstance(self.optimizer, OptimizerPlan) + else self.scheduler.arguments + ) + try: + inspect.signature(get_scheduler_class(self.scheduler.name)).bind( + object(), **arguments ) + for spec in specs: + inspect.signature(get_optimizer_class(spec.name)).bind( + object(), lr=1.0, **spec.arguments + ) + except TypeError as error: + raise ValueError(str(error)) from error return self - def validate_budget(self, step_budget): - if ( - step_budget is not None - and self.scheduler.name == "OneCycleLR" - and self.scheduler.arguments.get("total_steps") != step_budget - ): - raise ValueError( - f"OneCycleLR.total_steps must equal the maximum authored optimizer-step budget ({step_budget})" - ) - def build(self, module): from sequifier.model.parameter_catalog import ParameterCatalog from sequifier.training.optimization import create_optimizer @@ -82,3 +94,61 @@ def build(self, module): return optimizer, get_scheduler_class(self.scheduler.name)( optimizer, **arguments ) + + +class OptimizationConfig(RoleOptimizationConfig): + """Policy roles advance their schedulers once per applied optimizer update.""" + + scheduler_step_on: Literal["optimizer_step"] = "optimizer_step" + + def validate_budget(self, step_budget): + if ( + step_budget is not None + and self.scheduler.name == "OneCycleLR" + and self.scheduler.arguments.get("total_steps") != step_budget + ): + raise ValueError( + f"OneCycleLR.total_steps must equal the maximum authored optimizer-step budget ({step_budget})" + ) + + @model_validator(mode="after") + def validate_policy_scheduler(self): + if ( + self.scheduler.name + in {"ReduceLROnPlateau", "SequentialLR", "ChainedScheduler"} + or "optimizer" in self.scheduler.arguments + ): + raise ValueError( + "policy schedulers must advance independently on optimizer steps" + ) + return self + + +class SupervisedOptimizationConfig(RoleOptimizationConfig): + """Epoch scheduling includes metric-driven ReduceLROnPlateau.""" + + scheduler_step_on: Literal["epoch", "batch"] = "epoch" + scheduler: ComponentSpec = Field( + default_factory=lambda: ComponentSpec( + name="StepLR", arguments={"step_size": 1, "gamma": 0.99} + ) + ) + + @model_validator(mode="after") + def validate_supervised_scheduler(self): + if ( + self.scheduler.name == "ReduceLROnPlateau" + and self.scheduler_step_on != "epoch" + ): + raise ValueError("ReduceLROnPlateau requires scheduler_step_on: epoch") + if self.scheduler.name in {"SequentialLR", "ChainedScheduler"}: + raise ValueError("nested schedulers are not supported") + if ( + self.scheduler.name == "OneCycleLR" + and self.scheduler_step_on == "batch" + and "total_steps" not in self.scheduler.arguments + ): + raise ValueError( + "batch OneCycleLR requires an explicit total_steps optimizer-update budget" + ) + return self diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 2ba2672f..75d28252 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -6,17 +6,17 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from sequifier.config.identifiers import identifier -from sequifier.config.policy_config import CategoricalPolicyConfig from sequifier.config.policy_optimization import OptimizationConfig +from sequifier.config.sources import TrainingSourceSpecModel class PhaseConfig(BaseModel): model_config = ConfigDict(extra="forbid", allow_inf_nan=False) name: str - handoff: Literal["best", "last"] = "best" + optimizer_state: Literal["reset", "continue"] = "reset" + handoff: Literal["best", "last"] = "last" model_interface: str | None = None freeze: list[str] = Field(default_factory=list) - representation: CategoricalPolicyConfig | None = None optimization_roles: ClassVar[tuple[str, ...]] = ("actor",) optimization: dict[str, OptimizationConfig] = Field(default_factory=dict) evaluation_interval: int = Field(default=1, gt=0) @@ -52,8 +52,10 @@ class BatchedPhaseConfig(PhaseConfig): class OfflinePhaseConfig(PhaseConfig): - sources: list[str] = Field(min_length=1) - validation_sources: list[str] | None = Field(default=None, min_length=1) + sources: list[TrainingSourceSpecModel] = Field(min_length=1) + validation_sources: list[TrainingSourceSpecModel] | None = Field( + default=None, min_length=1 + ) class PolicyImitationPhaseConfig(BatchedPhaseConfig, OfflinePhaseConfig): @@ -93,7 +95,11 @@ class OnlineEnvironmentConfig(BaseModel): model_config = ConfigDict(extra="forbid") factory: str config: dict = Field(default_factory=dict) - protocol_version: Literal[3] = 3 + + @property + def protocol_version(self): + return 3 + implementation_id: str = Field(min_length=1) # Empty means unitless; adapters must report this exact declaration. units: dict[str, str] = Field(default_factory=dict) @@ -134,7 +140,7 @@ class EvaluationEpisodesConfig(BaseModel): seeds: list[int] = Field(min_length=1) max_episode_steps: int = Field(gt=0) timeout_seconds: float = Field(gt=0, allow_inf_nan=False) - selection: Literal["deterministic", "stochastic"] = "deterministic" + action_selection: Literal["deterministic", "stochastic"] = "deterministic" @field_validator("seeds") @classmethod @@ -144,17 +150,23 @@ def validate_seeds(cls, values): return values -class OnlineEvaluationConfig(EvaluationEpisodesConfig): +class OnlineEvaluationMonitor(BaseModel): metric: Literal[ "mean_completed_return", "success_rate", "mean_completed_length", "score_mean" ] = "mean_completed_return" - direction: Literal["min", "max"] = "max" + mode: Literal["min", "max"] = "max" -class PolicyResumeConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - policy: Literal["never", "if_exists", "required"] = "never" - mode: Literal["exact", "reset_environments"] = "reset_environments" +class OnlineEvaluationConfig(EvaluationEpisodesConfig): + monitor: OnlineEvaluationMonitor = Field(default_factory=OnlineEvaluationMonitor) + + @property + def metric(self): + return self.monitor.metric + + @property + def direction(self): + return self.monitor.mode class PPOPhaseConfig(PhaseConfig): @@ -218,8 +230,8 @@ def location(self): class DPOPhaseConfig(BatchedPhaseConfig): algorithm: Literal["dpo"] - preferences: str - validation_preferences: str | None = None + preferences: TrainingSourceSpecModel + validation_preferences: TrainingSourceSpecModel | None = None updates: int = Field(gt=0) beta: float = Field(default=0.1, gt=0) scoring_chunk_size: int = Field(default=64, gt=0) @@ -231,9 +243,10 @@ def optimizer_step_budget(self, training_size=None): return self.updates -class GRPOEvaluationConfig(EvaluationEpisodesConfig): - metric: Literal["score_mean"] = "score_mean" - direction: Literal["max"] = "max" +class GRPOEvaluationConfig(OnlineEvaluationConfig): + monitor: OnlineEvaluationMonitor = Field( + default_factory=lambda: OnlineEvaluationMonitor(metric="score_mean") + ) class GRPOPhaseConfig(BatchedPhaseConfig): diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index 9d9941ca..f5c651bc 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -18,22 +18,27 @@ class AdapterConfig(BaseModel): factory: str config: dict[str, Any] = Field(default_factory=dict) - protocol_version: Literal[1] = 1 + + @property + def protocol_version(self): + return 1 @field_validator("factory") @classmethod def validate_factory(cls, value: str) -> str: module, separator, attribute = value.partition(":") if not separator or not module or not attribute.isidentifier(): - raise ValueError("adapter.factory must be a 'module:factory' import path") + raise ValueError( + "environment.factory must be a 'module:factory' import path" + ) return value class RolloutConfig(BaseModel): model_config = ConfigDict(extra="forbid") - mode: Literal["forecast"] = "forecast" - adapter: AdapterConfig + max_steps: int = Field(gt=0) + environment: AdapterConfig bindings: list[InputBinding] trajectory_path: str = Field(min_length=1) # Index over prepared windows in dataset/chunk order, before legacy seed filtering. diff --git a/src/sequifier/config/sequence_import_config.py b/src/sequifier/config/sequence_import_config.py index ec6c0b49..299578a4 100644 --- a/src/sequifier/config/sequence_import_config.py +++ b/src/sequifier/config/sequence_import_config.py @@ -1,11 +1,15 @@ -"""The explicit sequence import operation of sequifier preprocess.""" +"""Explicit sequence import with the same source and authored task contracts.""" from pathlib import Path from typing import Literal from pydantic import Field -from sequifier.config.policy_config import PolicyConfigBase, PolicySchemaConfig +from sequifier.config.policy_config import ( + PolicyBinding, + PolicyConfigBase, + ResolvedPolicyBinding, +) from sequifier.config.policy_train_config import SequenceSourceConfig @@ -13,7 +17,8 @@ class SequenceImportConfig(PolicyConfigBase): operation: Literal["import_sequences"] project_root: str = "." source: SequenceSourceConfig - policy: PolicySchemaConfig + policy_binding: PolicyBinding + initialize_from: str | None = None output_path: str episodes_per_shard: int = Field(default=64, gt=0) @@ -21,3 +26,39 @@ def resolved_source(self) -> SequenceSourceConfig: from sequifier.config.policy_train_config import resolve_sequence_source return resolve_sequence_source(self.source, Path(self.project_root).resolve()) + + def resolved_policy(self) -> ResolvedPolicyBinding: + if self.initialize_from: + from sequifier.artifacts.model_artifact import read_execution_artifact + from sequifier.artifacts.model_config import ( + resolved_config_from_model_config, + ) + from sequifier.rl.attachment import resolve_binding + + execution, name = resolved_config_from_model_config( + read_execution_artifact( + Path(self.project_root) / self.initialize_from + ).model_config.values, + device="cpu", + ) + if len(execution.interface_names) != 1: + raise ValueError( + "sequence import initialize_from requires one model interface" + ) + return resolve_binding(execution, name, self.policy_binding) + observations = list(self.source.columns.observations) + static = list(self.source.columns.static_inputs) + if not observations: + raise ValueError( + "sequence import requires source.columns.observations or initialize_from" + ) + # Import validates and stores domain episodes, without constructing a model. + return ResolvedPolicyBinding( + **self.policy_binding.model_dump(), + context_length=1, + interface_name="default", + observation_columns=observations, + static_columns=static, + action_order=list(self.policy_binding.actions), + input_schema={}, + ) diff --git a/src/sequifier/config/sources.py b/src/sequifier/config/sources.py new file mode 100644 index 00000000..df168c85 --- /dev/null +++ b/src/sequifier/config/sources.py @@ -0,0 +1,27 @@ +"""Named dataset and part references shared by training algorithms.""" + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from sequifier.config.identifiers import identifier as _identifier +from sequifier.typechecking import beartype + + +class TrainingSourceSpecModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + source: str + weight: Optional[float] = Field(default=None, gt=0) + batches_per_selection: Optional[int] = Field(default=None, gt=0) + + @field_validator("source") + @classmethod + @beartype + def validate_source(cls, value): + parts = value.split(".") + if len(parts) > 2: + raise ValueError("Source refs use only dataset or dataset.part") + for part in parts: + _identifier(part, "Source reference component") + return value diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index dc914a2a..ee2129ee 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from pathlib import Path from types import SimpleNamespace -from typing import Any, Literal, Optional +from typing import Annotated, Any, ClassVar, Literal, Optional import torch from pydantic import ( @@ -33,7 +33,6 @@ BackboneComponentConfig, BERTSpecModel, CategoricalHashingConfig, - ComponentSpec, DecoderComponentConfig, FeatureLayoutRegistryModel, ImageEncoderConfig, @@ -55,27 +54,23 @@ image_views, ) from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata -from sequifier.config.optimizer_config import ( - OptimizerPlan, - resolve_plan_scheduler_arguments, -) from sequifier.config.policy_config import ( ActionFieldConfig, - CategoricalPolicyConfig, - NonemptyStr, - PolicySchemaConfig, + PolicyBinding, + ResolvedPolicyBinding, ) +from sequifier.config.policy_optimization import SupervisedOptimizationConfig from sequifier.config.policy_plan import ( + DPOPhaseConfig, FrozenNormalizationConfig, GRPOPhaseConfig, + IQLPhaseConfig, + PolicyImitationPhaseConfig, PolicyPhase, - PolicyResumeConfig, PPOPhaseConfig, ) -from sequifier.config.policy_train_config import ( - SequenceSourceConfig, - resolve_sequence_source, -) +from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.config.sources import TrainingSourceSpecModel from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -93,7 +88,6 @@ get_objective_class, target_offset_for_objective, ) -from sequifier.optimizers.optimizers import get_optimizer_class, get_scheduler_class from sequifier.special_tokens import ( SPECIAL_TOKEN_IDS, SPECIAL_TOKEN_NAMES, @@ -132,7 +126,17 @@ def _unique_columns(value: list[str], usage: str) -> list[str]: _SINGLETON_CONFIG_NAME = "default" _SINGLE_PHASE_NAME = "train" _PHASE_FIELD_NAMES = frozenset( - {"algorithm", "name", "epochs", "mode", "selection", "sources"} + { + "algorithm", + "name", + "epochs", + "mode", + "selection", + "sources", + "updates", + "total_env_steps", + "total_groups", + } ) _TRAINING_PLAN_POLICY_FIELDS = frozenset({"curriculum_training", "curriculum_column"}) @@ -202,6 +206,12 @@ def normalize_train_config_surface(values: Any) -> Any: datasets = normalized.get("dataset_training") _normalize_dataset_part_surface(datasets) + if isinstance(datasets, dict): + for dataset in datasets.values(): + if isinstance(dataset, dict): + for part in dataset.get("parts", {}).values(): + if isinstance(part, dict): + part.setdefault("type", "prepared") interfaces = model.get("interfaces") if isinstance(model, dict) else None if ( @@ -214,6 +224,17 @@ def normalize_train_config_surface(values: Any) -> Any: if isinstance(dataset, dict): dataset.setdefault("model_interface", interface_name) + if "policy_binding" in normalized and "policy_bindings" in normalized: + raise ValueError("Use policy_binding or policy_bindings, not both") + if ( + "policy_binding" in normalized + and isinstance(interfaces, dict) + and len(interfaces) == 1 + ): + normalized["policy_bindings"] = { + next(iter(interfaces)): normalized.pop("policy_binding") + } + implicit_source = _implicit_source(normalized) training_plan = normalized.get("training_plan") if isinstance(training_plan, dict): @@ -233,15 +254,22 @@ def normalize_train_config_surface(values: Any) -> Any: if key not in _TRAINING_PLAN_POLICY_FIELDS } phase.setdefault("name", _SINGLE_PHASE_NAME) - phase.setdefault("mode", "sequential") + if phase.get("algorithm", "supervised") == "supervised": + phase.setdefault("mode", "sequential") normalized["training_plan"] = {**policy, "phases": [phase]} training_plan = normalized["training_plan"] phases = training_plan.get("phases") - if isinstance(phases, list) and implicit_source is not None: + if isinstance(phases, list): for phase in phases: if isinstance(phase, dict): - phase.setdefault("sources", [{"source": implicit_source}]) + phase.setdefault("algorithm", "supervised") + if implicit_source is not None and phase["algorithm"] in { + "supervised", + "policy_imitation", + "iql", + }: + phase.setdefault("sources", [{"source": implicit_source}]) evaluation = normalized.get("evaluation") if isinstance(evaluation, bool): @@ -359,34 +387,21 @@ def normalize_train_config_parameter_surface( class GlobalTrainingSpecModel(BaseModel): - """Run-wide data, optimization, precision, and distribution settings.""" + """Run-wide data, precision, execution and distribution settings.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + validation_fraction: float = Field(default=0.2, gt=0, lt=1) + episode_cache_size: int = Field(default=8, gt=0) + metadata_config_path: str | dict[str, str] | None = None + normalization: FrozenNormalizationConfig | None = None read_format: Literal["csv", "parquet", "pt"] = "parquet" - training_objective: str + training_objective: str | None = None forward_attention: Literal["causal", "bidirectional"] | None = None - context_length: int = Field(gt=0) + context_length: int | None = Field(default=None, gt=0) target_offset: int = Field(default=1, ge=0) window_stride: Optional[int] = Field(default=None, gt=0) - inference_batch_size: int = Field(gt=0) - batch_size: int = Field(gt=0) - accumulation_steps: Optional[int] = Field(default=None, gt=0) - learning_rate: Optional[float] = Field(default=None, gt=0) - # Validate the plan branch first: ComponentSpec rejects non-mapping model - # instances with TypeError, which would otherwise abort union validation. - optimizer: OptimizerPlan | ComponentSpec = Field( - default_factory=lambda: ComponentSpec(name="Adam"), - union_mode="left_to_right", - ) - scheduler: ComponentSpec = Field( - default_factory=lambda: ComponentSpec( - name="StepLR", arguments={"step_size": 1, "gamma": 0.99} - ) - ) - scheduler_step_on: Literal["epoch", "batch"] = "epoch" - reset_optimization_on_phase: bool = True - gradient_clip: Optional[float] = Field(default=None, gt=0) + inference_batch_size: int = Field(default=1, gt=0) bert_spec: Optional[BERTSpecModel] = None next_occurrence_config: Optional[NextOccurrenceConfigModel] = None @@ -424,83 +439,6 @@ def validate_objective(cls, value: str) -> str: ) return value - @field_validator("optimizer", mode="before") - @classmethod - @beartype - def validate_optimizer(cls, value: Any) -> OptimizerPlan | ComponentSpec: - if isinstance(value, OptimizerPlan) or ( - isinstance(value, dict) and "groups" in value - ): - plan = OptimizerPlan.model_validate(value) - for group in plan.groups: - cls._validate_optimizer_spec( - group.optimizer, - group.learning_rate, - location=f"optimizer group {group.id!r}", - ) - return plan - spec = ComponentSpec.model_validate(value) - cls._validate_optimizer_spec(spec, 1.0, location="global_training") - return spec - - @classmethod - def _validate_optimizer_spec( - cls, spec: ComponentSpec, learning_rate: float, *, location: str - ) -> None: - name = spec.name - try: - optimizer_class = get_optimizer_class(name) - except ValueError: - raise ValueError(f"{name} not in the configured optimizer registries") - kwargs = dict(spec.arguments) - if "lr" in kwargs: - raise ValueError( - f"{location} must configure learning rate through its " - "learning_rate field, not optimizer.lr" - ) - kwargs["lr"] = learning_rate - _validate_constructor_arguments( - "optimizer", - name, - optimizer_class, - (object(),), - kwargs, - ) - - @model_validator(mode="after") - def validate_optimization_config(self) -> "GlobalTrainingSpecModel": - if isinstance(self.optimizer, OptimizerPlan): - if self.learning_rate is not None: - raise ValueError( - "global_training.learning_rate is only used with a single optimizer; " - "set learning_rate in each optimizer group instead." - ) - elif self.learning_rate is None: - raise ValueError( - "global_training.learning_rate is required with a single optimizer." - ) - - scheduler_class = get_scheduler_class(self.scheduler.name) - scheduler_arguments = dict(self.scheduler.arguments) - if isinstance(self.optimizer, OptimizerPlan): - scheduler_arguments = resolve_plan_scheduler_arguments( - self.optimizer, self.scheduler - ) - _validate_constructor_arguments( - "scheduler", - self.scheduler.name, - scheduler_class, - (object(),), - scheduler_arguments, - ) - return self - - @field_validator("scheduler", mode="before") - @classmethod - @beartype - def validate_scheduler(cls, value: Any) -> ComponentSpec: - return ComponentSpec.model_validate(value) - @field_validator("layer_type_dtypes") @classmethod @beartype @@ -525,21 +463,6 @@ def validate_layer_dtypes(cls, value: Optional[dict[str, str]]): @model_validator(mode="after") @beartype def validate_distribution(self): - if ( - self.scheduler.name == "ReduceLROnPlateau" - and self.scheduler_step_on != "epoch" - ): - raise ValueError("ReduceLROnPlateau requires scheduler_step_on=epoch") - scheduler_total_steps = self.scheduler.arguments.get("total_steps") - if ( - scheduler_total_steps is not None - and self.reset_optimization_on_phase - and self.scheduler_step_on == "epoch" - ): - raise ValueError( - "global_training.scheduler.total_steps is managed per phase by " - "Sequifier for epoch-stepped schedulers and must not be provided." - ) if self.distributed and self.data_parallelism is None: raise ValueError("distributed=true requires data_parallelism") if self.data_parallelism != "fsdp" and self.fsdp_cpu_offload is not None: @@ -607,7 +530,6 @@ class ModelInterfaceSpecModel(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") input_columns: list[str] = Field(default_factory=list) - policy: PolicySchemaConfig | None = None prediction_distributions: dict[str, ActionFieldConfig] = Field(default_factory=dict) static_conditioning: StaticConditioningConfig | None = None preconditioning_image_encoder: ImageEncoderConfig | None = None @@ -740,280 +662,9 @@ def validate_interface_names(cls, value): return value -class PolicyRunConfig(BaseModel): - """Canonical local policy run, shared by the CLI and Python API.""" - - model_config = ConfigDict(extra="forbid", allow_inf_nan=False) - project_root: str = "." - model_name: NonemptyStr = Field(pattern=r"^[^/\\]+$") - device: str = "cpu" - seed: int = Field(default=1010, ge=0, lt=2**32) - enforce_determinism: bool = True - model: ModelSpecModel | None = None - policy_binding: PolicySchemaConfig | None = None - execution_model: dict[str, Any] | None = None - policy_interface: str | None = None - context_length: int | None = Field(default=None, gt=0) - units: dict[str, str] = Field(default_factory=dict) - initialize_from: str | None = None - normalization: FrozenNormalizationConfig | None = None - sources: dict[str, SequenceSourceConfig] = Field(default_factory=dict) - preferences: dict[str, str] = Field(default_factory=dict) - phases: list[PolicyPhase] = Field(min_length=1) - validation_fraction: float = Field(default=0.2, gt=0, lt=1) - episode_cache_size: int = Field(default=8, gt=0) - resume: PolicyResumeConfig = Field(default_factory=PolicyResumeConfig) - model_path: str = "" - report_path: str = "" - checkpoint_path: str = "" - phase_directory: str = "" - - @model_validator(mode="after") - def resolve(self): - root = Path(self.project_root).resolve() - self.project_root = str(root) - if self.model_name in {".", ".."}: - raise ValueError("model_name must name one directory") - for field, default in ( - ("model_path", f"models/{self.model_name}/actor.pt"), - ("report_path", f"outputs/training/{self.model_name}/report.json"), - ("checkpoint_path", f"checkpoints/{self.model_name}/run.pt"), - ("phase_directory", f"outputs/training/{self.model_name}/phases"), - ): - setattr( - self, field, str((root / (getattr(self, field) or default)).resolve()) - ) - if self.initialize_from: - self.initialize_from = str((root / self.initialize_from).resolve()) - self.sources = { - name: resolve_sequence_source(source, root) - for name, source in self.sources.items() - } - self.preferences = { - name: str((root / path).resolve()) - for name, path in self.preferences.items() - } - if self.initialize_from and self.execution_model is None: - from sequifier.artifacts.model_artifact import read_execution_artifact - - if Path(self.initialize_from).is_file(): - self.execution_model = read_execution_artifact( - self.initialize_from - ).model_config.values - elif self.resume.policy != "never" and Path(self.checkpoint_path).is_file(): - import json - - saved = torch.load( - self.checkpoint_path, map_location="cpu", weights_only=False - ) - representation = json.loads(saved["representation"]) - self.execution_model = representation.get("execution") - else: - raise FileNotFoundError(self.initialize_from) - if self.execution_model is not None: - from sequifier.artifacts.model_config import ( - resolved_config_from_model_config, - ) - from sequifier.rl.attachment import apply_binding, resolve_binding - - execution, selected = resolved_config_from_model_config( - self.execution_model, - device=self.device, - interface_name=self.policy_interface, - ) - self.policy_interface = selected - if ( - self.context_length is not None - and self.context_length != execution.global_training.context_length - ): - raise ValueError("attachment preserves the source context length") - self.context_length = execution.global_training.context_length - source_policy = execution.interface(selected).policy - binding = self.policy_binding or source_policy - if binding is None: - raise ValueError( - "supervised initialization requires policy_binding with named actions" - ) - if binding.is_joint: - if self.model is not None: - if ( - self.model.backbone.architecture - != execution.model.backbone.architecture - ): - raise ValueError( - "declared backbone differs from the initialization artifact" - ) - for name, declared in self.model.interfaces.items(): - actual = execution.model.interfaces.get(name) - if actual is None or any( - getattr(declared, key) != getattr(actual, key) - for key in ( - "input_columns", - "target_columns", - "ingestion", - "decoder", - ) - ): - raise ValueError( - f"declared interface {name!r} differs from the initialization artifact" - ) - fitted = resolve_binding(execution, selected, binding) - apply_binding(execution, selected, fitted) - self.model = execution.model - elif self.model is None: - model = execution.model.model_copy(deep=True) - model.interfaces[selected].input_columns = ( - binding.observation_columns + binding.static_columns - ) - model.interfaces[selected].target_columns = [binding.action_column] - self.model = model - if ( - self.model is None - or self.policy_interface is None - or self.context_length is None - ): - raise ValueError( - "model, policy_interface and context_length are required without an artifact" - ) - joint = self.policy.is_joint - if not joint and set(self.model.interfaces) != {self.policy_interface}: - raise ValueError( - "policy runs require exactly the selected policy_interface" - ) - spec = self.policy - if set(self.units) - set( - spec.observation_columns + spec.static_columns + spec.action_columns - ): - raise ValueError( - "units must name policy observation, static or action fields" - ) - interface = self.model.interfaces[self.policy_interface] - if ( - not joint - and interface.input_columns - != spec.observation_columns + spec.static_columns - ): - raise ValueError( - "model.interfaces inputs must match policy observations and static fields in order" - ) - if not joint and interface.target_columns != [spec.action_column]: - raise ValueError("model.interfaces targets must contain the policy action") - if not joint and ( - interface.decoder.support != 1 or interface.decoder.prediction_length != 1 - ): - raise ValueError( - "policy decoder requires support=1 and prediction_length=1" - ) - if not joint and any( - ( - interface.categorical_hashing, - interface.static_conditioning, - interface.preconditioning_image_encoder, - interface.feature_layout, - ) - ): - raise ValueError( - "policy runs support scalar observation/static fields only" - ) - if not joint and self.model.backbone.repository is not None: - raise ValueError("policy initialization uses initialize_from") - if self.context_length > self.model.backbone.architecture.max_context_length: - raise ValueError( - "context_length exceeds model.backbone.architecture.max_context_length" - ) - if torch.device(self.device).type not in {"cpu", "cuda", "mps"}: - raise ValueError("policy runs require a local CPU, CUDA or MPS device") - for name, source in self.sources.items(): - if source.units is not None and source.units != self.units: - raise ValueError(f"sources.{name}.units differs from run units") - names = [phase.name for phase in self.phases] - if len(names) != len(set(names)): - raise ValueError("policy phase names must be unique") - for phase in self.phases: - if ( - phase.model_interface is not None - and phase.model_interface != self.policy_interface - ): - raise ValueError( - "use a unified plan to switch policy interfaces between phases" - ) - if phase.algorithm != "policy_imitation" and spec.return_conditioned: - if not joint: - raise ValueError( - f"phases.{phase.name}: legacy conditioned policies support imitation only" - ) - if ( - isinstance(phase, (PPOPhaseConfig, GRPOPhaseConfig)) - and spec.initial_desired_return is None - ): - raise ValueError( - "online return conditioning requires initial_desired_return" - ) - refs = getattr(phase, "sources", []) + ( - getattr(phase, "validation_sources", None) or [] - ) - if len(refs) != len(set(refs)) or set(refs) - self.sources.keys(): - raise ValueError( - f"phases.{phase.name}.sources: unknown, repeated or overlapping source names" - ) - if phase.algorithm == "dpo": - for name in [phase.preferences, phase.validation_preferences]: - if name is not None and name not in self.preferences: - raise ValueError( - f"phases.{phase.name}.preferences: unknown manifest {name!r}" - ) - reference = getattr(phase, "reference", None) - if reference is not None and reference.path is not None: - reference.path = str((root / reference.path).resolve()) - if isinstance(phase, (PPOPhaseConfig, GRPOPhaseConfig)): - for environment in (phase.environment, phase.evaluation.environment): - if environment.units != self.units: - raise ValueError( - f"phases.{phase.name}.environment.units differs from run units" - ) - archive = getattr(phase, "archive_directory", None) - if ( - isinstance(phase, (PPOPhaseConfig, GRPOPhaseConfig)) - and archive is not None - ): - archive_path = (root / archive).resolve() - phase.archive_directory = str(archive_path) - return self - - @property - def policy(self) -> CategoricalPolicyConfig: - if ( - self.model is None - or self.policy_interface is None - or self.context_length is None - ): - raise ValueError( - "policy model, interface and context must be resolved first" - ) - schema = self.model.interfaces[self.policy_interface].policy - if schema is None: - raise ValueError("policy_interface requires a categorical policy schema") - if schema.is_joint and self.execution_model is not None: - from sequifier.artifacts.model_config import ( - resolved_config_from_model_config, - ) - from sequifier.rl.attachment import resolve_binding - - execution, _ = resolved_config_from_model_config( - self.execution_model, - device=self.device, - interface_name=self.policy_interface, - ) - return resolve_binding(execution, self.policy_interface, schema) - return CategoricalPolicyConfig( - **schema.model_dump(), - interface_name=self.policy_interface, - context_length=self.context_length, - ) - - class DatasetPartSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") + type: Literal["prepared"] = "prepared" metadata_config_path: str image_path: MediaPath = None @@ -1021,6 +672,22 @@ class DatasetPartSpecModel(BaseModel): file_order: Literal["shuffled", "name"] = "shuffled" +class SequencePartSpecModel(SequenceSourceConfig): + type: Literal["sequence"] + + +class PreferencePartSpecModel(BaseModel): + model_config = ConfigDict(extra="forbid") + type: Literal["preferences"] + manifest: str = Field(min_length=1) + + +DatasetPart = Annotated[ + DatasetPartSpecModel | SequencePartSpecModel | PreferencePartSpecModel, + Field(discriminator="type"), +] + + class DatasetFreezingSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -1045,9 +712,9 @@ def active(self) -> bool: class DatasetTrainingSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") - model_interface: str - parts: dict[str, DatasetPartSpecModel] = Field(..., min_length=1) - criterion: dict[str, str] = Field(..., min_length=1) + model_interface: str | None = None + parts: dict[str, DatasetPart] = Field(..., min_length=1) + criterion: dict[str, str] = Field(default_factory=dict) class_weights: Optional[dict[str, list[float]]] = None loss_weights: Optional[dict[str, float]] = None class_share_log_columns: list[str] = Field(default_factory=list) @@ -1057,7 +724,11 @@ class DatasetTrainingSpecModel(BaseModel): @classmethod @beartype def validate_interface_name(cls, value): - return _identifier(value, "Model interface reference") + return ( + _identifier(value, "Model interface reference") + if value is not None + else None + ) @field_validator("parts") @classmethod @@ -1095,29 +766,22 @@ def validate_loss_weights(cls, value): return value -class TrainingSourceSpecModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - source: str - weight: Optional[float] = Field(default=None, gt=0) - batches_per_selection: Optional[int] = Field(default=None, gt=0) - - @field_validator("source") - @classmethod - @beartype - def validate_source(cls, value): - parts = value.split(".") - if len(parts) > 2: - raise ValueError("Source refs use only dataset or dataset.part") - for part in parts: - _identifier(part, "Source reference component") - return value - - class TrainingPhaseSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") algorithm: Literal["supervised"] = "supervised" + handoff: Literal["last", "best"] = "last" + + optimization_roles: ClassVar[tuple[str, ...]] = ("model",) + optimizer_state: Literal["reset", "continue"] = "reset" + batch_size: int = Field(default=32, gt=0) + accumulation_steps: int = Field(default=1, gt=0) + optimization: dict[Literal["model"], SupervisedOptimizationConfig] = Field( + default_factory=lambda: {"model": SupervisedOptimizationConfig()} + ) + + def optimization_configs(self): + return {"model": self.optimization.get("model", SupervisedOptimizationConfig())} name: str epochs: int = Field(gt=0) @@ -1152,12 +816,23 @@ def validate_mode_fields(self): return self +TrainingPhase = Annotated[ + TrainingPhaseSpecModel + | PolicyImitationPhaseConfig + | IQLPhaseConfig + | PPOPhaseConfig + | DPOPhaseConfig + | GRPOPhaseConfig, + Field(discriminator="algorithm"), +] + + class TrainingPlanModel(BaseModel): model_config = ConfigDict(extra="forbid") curriculum_training: bool = False curriculum_column: Optional[str] = Field(default=None, min_length=1) - phases: list[TrainingPhaseSpecModel] = Field(..., min_length=1) + phases: list[TrainingPhase] = Field(..., min_length=1) @model_validator(mode="after") @beartype @@ -1173,6 +848,43 @@ def validate_unique_names(self): names = [phase.name for phase in self.phases] if len(names) != len(set(names)): raise ValueError("Training phase names must be unique") + mixed = len({phase.algorithm == "supervised" for phase in self.phases}) > 1 + for index, phase in enumerate(self.phases): + if phase.optimizer_state != "continue": + continue + if index == 0: + raise ValueError("the first phase cannot continue optimizer state") + previous = self.phases[index - 1] + if mixed: + raise ValueError( + "mixed-plan coordination cannot preserve optimizer state" + ) + if previous.algorithm != phase.algorithm or previous.handoff != "last": + raise ValueError( + "optimizer continuation requires the same algorithm and a last handoff" + ) + if phase.algorithm == "supervised" and ( + {ref.source.partition(".")[0] for ref in previous.sources} + != {ref.source.partition(".")[0] for ref in phase.sources} + ): + raise ValueError( + "supervised optimizer continuation requires identical dataset ownership" + ) + if getattr(previous, "freeze", None) != getattr(phase, "freeze", None): + raise ValueError( + "optimizer continuation requires identical parameter selection" + ) + if previous.optimization_configs() != phase.optimization_configs(): + raise ValueError( + "optimizer continuation requires identical role optimization settings" + ) + if phase.algorithm != "supervised" and any( + item.scheduler.name == "OneCycleLR" + for item in phase.optimization_configs().values() + ): + raise ValueError( + "policy OneCycleLR continuation is unsupported; use optimizer_state: reset" + ) return self @@ -1180,7 +892,7 @@ class EvaluationMonitorSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") source: str - metric: Literal["loss"] = "loss" + metric: str = "loss" mode: Literal["min", "max"] = "min" @@ -1219,9 +931,17 @@ class SequifierConfig(BaseModel): model_name: str device: str seed: int = 1010 - global_training: GlobalTrainingSpecModel - model: ModelSpecModel - dataset_training: dict[str, DatasetTrainingSpecModel] = Field(..., min_length=1) + global_training: GlobalTrainingSpecModel = Field( + default_factory=GlobalTrainingSpecModel + ) + model: ModelSpecModel | None = None + initialize_from: str | None = None + policy_bindings: dict[str, PolicyBinding] = Field(default_factory=dict) + policy_binding: PolicyBinding | None = Field(default=None, exclude=True) + model_path: str = "" + report_path: str = "" + phase_directory: str = "" + dataset_training: dict[str, DatasetTrainingSpecModel] = Field(default_factory=dict) training_plan: TrainingPlanModel evaluation: Optional[EvaluationSpecModel] = None @@ -1259,67 +979,79 @@ def validate_model_name(cls, value): @model_validator(mode="after") @beartype def validate_relationships(self): - if self.global_training.training_objective == "policy": + if self.model is None and self.initialize_from is None: + raise ValueError("a run requires model or initialize_from") + for phase in self.training_plan.phases: + if ( + phase.algorithm == "supervised" + and phase.handoff == "best" + and (self.evaluation is None or self.evaluation.monitor is None) + ): + raise ValueError("supervised best handoff requires evaluation.monitor") + for phase in self.training_plan.phases: + refs = list(getattr(phase, "sources", [])) + list( + getattr(phase, "validation_sources", None) or [] + ) + for key in ("preferences", "validation_preferences"): + ref = getattr(phase, key, None) + if ref is not None: + refs.append(ref) + for ref in refs: + self._validate_source(ref.source, f"phase {phase.name!r}") + if phase.algorithm != "supervised" and ( + ref.weight is not None or ref.batches_per_selection is not None + ): + raise ValueError( + "policy source references cannot configure supervised interleaving" + ) + for phase in self.training_plan.phases: + if ( + phase.algorithm == "supervised" + and phase.optimization_configs()["model"].scheduler.name + == "ReduceLROnPlateau" + and (self.evaluation is None or self.evaluation.monitor is None) + ): + raise ValueError("ReduceLROnPlateau requires evaluation.monitor") + if any(phase.algorithm != "supervised" for phase in self.training_plan.phases): + if self.global_training.distributed: + raise ValueError("policy phases currently require local execution") + return self + if self.initialize_from is not None: + return ( + self # Model-contract assertions are checked after artifact resolution. + ) + model = self.model + assert model is not None + if ( + self.global_training.training_objective is None + or self.global_training.context_length is None + ): raise ValueError( - "policy training requires PolicyRunConfig; supervised and policy phases cannot be mixed" + "supervised training requires training_objective and context_length" ) if self.global_training.early_stopping_epochs is not None and ( self.evaluation is None or self.evaluation.monitor is None ): raise ValueError("early stopping requires evaluation.monitor") - if self.global_training.scheduler.name == "ReduceLROnPlateau" and ( - self.evaluation is None or self.evaluation.monitor is None - ): - raise ValueError("ReduceLROnPlateau requires evaluation.monitor") - - scheduler_total_steps = self.global_training.scheduler.arguments.get( - "total_steps" - ) - if scheduler_total_steps is not None: - total_epochs = sum(phase.epochs for phase in self.training_plan.phases) - reset_on_phase = self.global_training.reset_optimization_on_phase - if reset_on_phase: - if self.global_training.scheduler_step_on == "batch": - warnings.warn( - "Batch-stepped scheduler " - f"total_steps={scheduler_total_steps} is applied " - "independently to every training phase.", - stacklevel=2, - ) - elif self.global_training.scheduler_step_on == "epoch": - if scheduler_total_steps != total_epochs: - raise ValueError( - "scheduler total steps: " - f"{scheduler_total_steps} != {total_epochs}: total epochs" - ) - else: - warnings.warn( - f"{scheduler_total_steps} scheduler steps at {total_epochs} " - "epochs implies " - f"{scheduler_total_steps / total_epochs:.2f} batches. " - "Does this seem correct?", - stacklevel=2, - ) - - for interface in self.model.interfaces.values(): + for interface in model.interfaces.values(): validate_embedding_layer_names( self.embedding_layer_names, SimpleNamespace( - backbone=self.model.backbone, + backbone=model.backbone, decoder=interface.decoder, ), ) referenced_interfaces = set() for dataset_name, dataset in self.dataset_training.items(): - if dataset.model_interface not in self.model.interfaces: + if dataset.model_interface not in model.interfaces: raise ValueError( f"Dataset {dataset_name!r} references unknown model interface " f"{dataset.model_interface!r}." ) referenced_interfaces.add(dataset.model_interface) - interface = self.model.interfaces[dataset.model_interface] + interface = model.interfaces[dataset.model_interface] if set(dataset.criterion) != set(interface.target_columns): raise ValueError( f"Dataset {dataset_name!r} criterion keys must equal interface " @@ -1345,7 +1077,7 @@ def validate_relationships(self): raise ValueError( f"Dataset {dataset_name!r} class_weights references unknown targets." ) - unreferenced = set(self.model.interfaces) - referenced_interfaces + unreferenced = set(model.interfaces) - referenced_interfaces if unreferenced: warnings.warn( f"Unreferenced model interfaces: {sorted(unreferenced)}", @@ -1389,17 +1121,17 @@ def validate_relationships(self): any( route.static_conditioning is not None or route.preconditioning_image_encoder is not None - for route in self.model.interfaces.values() + for route in model.interfaces.values() ) ) if ( context_length + prefix_positions - > self.model.backbone.architecture.max_context_length + > model.backbone.architecture.max_context_length ): raise ValueError( "context_length plus static prefix exceeds backbone max_context_length" ) - for name, interface in self.model.interfaces.items(): + for name, interface in model.interfaces.items(): if interface.decoder.support > context_length: raise ValueError( f"Interface {name!r} decoder support exceeds context_length" @@ -1447,7 +1179,7 @@ class ResolvedDatasetPart(BaseModel): class ResolvedModelInterface(BaseModel): - policy: CategoricalPolicyConfig | None = None + policy: ResolvedPolicyBinding | None = None prediction_distributions: dict[str, ActionFieldConfig] = Field(default_factory=dict) model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") @@ -1526,6 +1258,12 @@ class ResolvedTrainingPhase(BaseModel): algorithm: Literal["supervised"] = "supervised" + handoff: Literal["last", "best"] = "last" + optimizer_state: Literal["reset", "continue"] = "reset" + batch_size: int = Field(default=32, gt=0) + accumulation_steps: int = Field(default=1, gt=0) + optimization: dict[str, SupervisedOptimizationConfig] + name: str epochs: int mode: Literal["sequential", "interleaved"] @@ -1533,6 +1271,60 @@ class ResolvedTrainingPhase(BaseModel): sources: list[ResolvedTrainingSource] +def resolve_supervised_phases(authored_phases): + """Resolve role defaults and epoch budgets without accessing datasets.""" + phases = [ + ResolvedTrainingPhase( + name=phase.name, + epochs=phase.epochs, + handoff=phase.handoff, + optimizer_state=phase.optimizer_state, + batch_size=phase.batch_size, + accumulation_steps=phase.accumulation_steps, + optimization={ + role: value.model_copy(deep=True) + for role, value in phase.optimization_configs().items() + }, + mode=phase.mode, + selection=phase.selection or "round_robin", + sources=[ + _source( + source.source, + phase.accumulation_steps, + weight=source.weight, + batches_per_selection=( + None + if phase.mode == "sequential" + else source.batches_per_selection + ), + ) + for source in phase.sources + ], + ) + for phase in authored_phases + ] + # A continued epoch scheduler spans the entire contiguous ownership chain. + for index, phase in enumerate(phases): + role = phase.optimization["model"] + if role.scheduler.name != "OneCycleLR" or role.scheduler_step_on != "epoch": + continue + start = index + while start > 0 and phases[start].optimizer_state == "continue": + start -= 1 + end = index + 1 + while end < len(phases) and phases[end].optimizer_state == "continue": + end += 1 + budget = sum(item.epochs for item in phases[start:end]) + explicit = role.scheduler.arguments.get("total_steps") + if explicit is not None and explicit != budget: + raise ValueError( + f"{phase.name}: OneCycleLR.total_steps must equal epoch budget {budget}" + ) + arguments = {**role.scheduler.arguments, "total_steps": budget} + role.scheduler = role.scheduler.model_copy(update={"arguments": arguments}) + return phases + + class ResolvedSequifierConfig(BaseModel): """Runtime configuration after all dataset parts and interfaces resolve.""" @@ -1545,7 +1337,15 @@ class ResolvedSequifierConfig(BaseModel): global_training: GlobalTrainingSpecModel model: ModelSpecModel dataset_training: dict[str, ResolvedDatasetTrainingSpec] - training_plan: list[ResolvedTrainingPhase] + authored: SequifierConfig | None = Field(default=None, exclude=True) + initialize_from: str | None = None + model_path: str = "" + report_path: str = "" + phase_directory: str = "" + sequence_sources: dict[str, SequenceSourceConfig] = Field(default_factory=dict) + preference_sources: dict[str, str] = Field(default_factory=dict) + policy_bindings: dict[str, ResolvedPolicyBinding] = Field(default_factory=dict) + training_plan: list[ResolvedTrainingPhase | PolicyPhase] curriculum_training: bool = False curriculum_column: Optional[str] = None evaluation_sources: list[ResolvedTrainingSource] @@ -1657,8 +1457,8 @@ def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: @dataclass(frozen=True) class LoadedTrainConfig: - config: Any - resolved: Any + config: SequifierConfig + resolved: ResolvedSequifierConfig metadata: dict[str, DatasetMetadata] @@ -1874,6 +1674,10 @@ def _resolve_interface( global_spec: GlobalTrainingSpecModel, storage_layout: StoredWindowLayout, ) -> ResolvedModelInterface: + if global_spec.training_objective is None or global_spec.context_length is None: + raise ValueError( + "resolved training requires training_objective and context_length" + ) if any(metadata.depth_layouts.is_deep_column(c) for c in spec.target_columns): raise ValueError("Depth targets are not supported") signature = _part_signature(metadata, spec) @@ -1984,15 +1788,7 @@ def _resolve_interface( resolve_window_view(storage_layout, window_view) return ResolvedModelInterface( name=name, - policy=( - CategoricalPolicyConfig( - **spec.policy.model_dump(), - interface_name=name, - context_length=global_spec.context_length, - ) - if spec.policy is not None - else None - ), + policy=None, input_columns=spec.input_columns, prediction_distributions=spec.prediction_distributions, static_conditioning=spec.static_conditioning, @@ -2049,7 +1845,7 @@ def _resolve_interface( if target_types[target] == "real" ] ) - if spec.policy is None or column in metadata.selected_columns_statistics + if column in metadata.selected_columns_statistics }, normalize_real_columns=metadata.normalize_real_columns, target_decoder_ids=target_decoder_ids, @@ -2113,6 +1909,16 @@ def resolve_sequifier_config( ) -> ResolvedSequifierConfig: """Resolve every dataset part, compatibility contract, and source.""" + if config.model is None: + raise ValueError("resolved training requires a model") + if ( + config.global_training.context_length is None + or config.global_training.training_objective is None + ): + raise ValueError( + "resolved training requires training_objective and context_length" + ) + part_refs = [ f"{dataset_name}.{part_name}" for dataset_name, dataset in config.dataset_training.items() @@ -2136,6 +1942,8 @@ def resolve_sequifier_config( interface_semantics: dict[str, dict[str, Any]] = {} for dataset_name, dataset_spec in config.dataset_training.items(): + if dataset_spec.model_interface is None: + raise ValueError(f"Dataset {dataset_name!r} requires a model interface") interface_spec = config.model.interfaces[dataset_spec.model_interface] resolved_parts = {} expected_signature = None @@ -2348,31 +2156,9 @@ def resolve_sequifier_config( freeze=dataset_spec.freeze, ) - accumulation = config.global_training.accumulation_steps - phases = [ - ResolvedTrainingPhase( - name=phase.name, - epochs=phase.epochs, - mode=phase.mode, - selection=phase.selection or "round_robin", - sources=[ - _source( - source.source, - accumulation, - weight=source.weight, - batches_per_selection=( - None - if phase.mode == "sequential" - else source.batches_per_selection - ), - ) - for source in phase.sources - ], - ) - for phase in config.training_plan.phases - ] + phases = resolve_supervised_phases(config.training_plan.phases) evaluation_sources = ( - [_source(source.source, accumulation) for source in config.evaluation.sources] + [_source(source.source, 1) for source in config.evaluation.sources] if config.evaluation is not None else [] ) @@ -2483,6 +2269,8 @@ def _inline_metadata( max_target_offset = values.get("max_target_offset", 1) layout_version = values.get("stored_window_layout_version", 2) if window_length is None: + if global_spec.context_length is None: + raise ValueError("inline metadata requires context_length or window_length") window_length = global_spec.context_length + max(1, global_spec.target_offset) return DatasetMetadata.model_validate( { @@ -2542,39 +2330,22 @@ def load_train_config_with_source( if args: raise ValueError(f"Unsupported training CLI overrides: {sorted(args)}") - if "phases" in raw and ( - raw.get("run_type") == "unified" - or "policy_bindings" in raw - or "dataset_training" in raw - or "global_training" in raw + config = try_catch_excess_keys(config_path, SequifierConfig, raw) + if ( + config.initialize_from is not None + or config.policy_bindings + or config.policy_binding is not None + or any(phase.algorithm != "supervised" for phase in config.training_plan.phases) ): - from sequifier.config.learning_run import LearningRunConfig - if sensitive or metadata_inline_values or skip_metadata: - raise ValueError("mixed learning plans require explicit source metadata") - authored = LearningRunConfig.model_validate(raw) - return LoadedTrainConfig(config=authored, resolved=authored, metadata={}) - if "phases" in raw: - if ( - "training_plan" in raw - or "dataset_training" in raw - or "global_training" in raw - ): raise ValueError( - "policy phases cannot be mixed with supervised training configuration" + "policy/mixed plans require source declarations instead of inline metadata overrides" ) - if sensitive or metadata_inline_values or skip_metadata: - raise ValueError("policy runs do not accept supervised metadata overrides") - authored = PolicyRunConfig.model_validate(raw) - return LoadedTrainConfig(config=authored, resolved=authored, metadata={}) - plan = raw.get("training_plan", {}) - phases = plan.get("phases", [plan]) if isinstance(plan, dict) else [] - if any(phase.get("algorithm", "supervised") != "supervised" for phase in phases): - raise ValueError( - "policy training requires PolicyRunConfig with top-level phases" - ) + from sequifier.config.training_resolution import resolve_training_run - config = try_catch_excess_keys(config_path, SequifierConfig, raw) + return LoadedTrainConfig( + config=config, resolved=resolve_training_run(config), metadata={} + ) part_overrides: dict[str, dict[str, str]] = {} if sensitive: selected_part = _override_part(config) @@ -2636,7 +2407,7 @@ def load_train_config_with_source( @beartype def load_train_config( config_path: str, args_config: dict[str, Any], skip_metadata: bool -) -> Any: +) -> ResolvedSequifierConfig: return load_train_config_with_source( config_path, args_config, skip_metadata ).resolved @@ -2691,6 +2462,7 @@ class SelectedInterfaceConfig: class SelectedDatasetPartConfig(SelectedInterfaceConfig): """Explicit dataset-part configuration used by data loaders.""" + batch_size: int dataset_training: ResolvedDatasetTrainingSpec part: ResolvedDatasetPart curriculum_training: bool @@ -2766,6 +2538,9 @@ def dataset_part_view( ) return SelectedDatasetPartConfig( + batch_size=config.training_plan[0].batch_size + if config.training_plan + else config.global_training.inference_batch_size, **interface_values, dataset_training=dataset, part=part, diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py new file mode 100644 index 00000000..68e7da37 --- /dev/null +++ b/src/sequifier/config/training_resolution.py @@ -0,0 +1,653 @@ +"""Resolve the common authored plan into model, dataset and task contracts.""" + +from pathlib import Path + +from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata +from sequifier.config.policy_train_config import ( + SequenceSourceConfig, + resolve_sequence_source, +) +from sequifier.config.train_config import ( + DatasetPartSpecModel, + GlobalTrainingSpecModel, + PreferencePartSpecModel, + SequencePartSpecModel, + SequifierConfig, + _resolve_interface, + resolve_sequifier_config, + resolve_supervised_phases, +) +from sequifier.special_tokens import SPECIAL_TOKEN_IDS + + +def source_parts(config, reference): + """Expand a reference without silently filtering out incompatible parts.""" + dataset, _, part = reference.partition(".") + spec = config.dataset_training[dataset] + return [ + (f"{dataset}.{name}", value) + for name, value in spec.parts.items() + if not part or name == part + ] + + +def phase_interface(config, phase): + names = set() + for ref in list(getattr(phase, "sources", [])) + list( + getattr(phase, "validation_sources", None) or [] + ): + names.add(config.dataset_training[ref.source.partition(".")[0]].model_interface) + explicit = getattr(phase, "model_interface", None) + if explicit is not None: + names.add(explicit) + if not names: + names = set(config.policy_bindings) or set(config.model.interfaces) + if len(names) != 1 or None in names: + raise ValueError( + f"training_plan.phases.{phase.name}: select one model_interface" + ) + name = next(iter(names)) + if name not in config.model.interfaces: + raise ValueError(f"unknown model interface {name!r}") + return name + + +def _inferred_metadata(config, name, binding): + """Identity numeric inputs and explicitly declared categorical action codecs.""" + interface = config.model.interfaces[name] + columns = set(interface.input_columns + interface.target_columns) + if interface.static_conditioning is not None: + columns.update(interface.static_conditioning.columns) + types = dict.fromkeys(columns, "Float32") + counts, maps = {}, {} + for column, action in binding.actions.items(): + if action.type == "categorical": + if not action.values: + raise ValueError( + f"{column}: fresh categorical targets need values or metadata" + ) + types[column] = "Int64" + counts[column] = SPECIAL_TOKEN_IDS.user_start + len(action.values) + maps[column] = { + str(label): SPECIAL_TOKEN_IDS.user_start + index + for index, label in enumerate(action.values) + } + for column, action in binding.previous_action_inputs.items(): + if action not in binding.actions: + raise ValueError( + f"{column}: previous_action_inputs references an unknown action" + ) + if action in maps: + labels = list(binding.actions[action].values) + initial = binding.initial_inputs.get(column) + if initial is not None and initial not in labels: + labels.append(initial) + maps[column] = { + str(label): SPECIAL_TOKEN_IDS.user_start + index + for index, label in enumerate(labels) + } + counts[column] = SPECIAL_TOKEN_IDS.user_start + len(labels) + types[column] = "Int64" + normalization = config.global_training.normalization + statistics = ( + normalization.statistics + if normalization and normalization.source == "statistics" + else {} + ) + return DatasetMetadata( + window_length=config.global_training.context_length + + max(1, config.global_training.target_offset), + column_data_types=types, + n_classes=counts, + id_maps=maps, + selected_columns_statistics={ + column: statistics.get(column, {"mean": 0.0, "std": 1.0}) + for column in columns + if column not in counts + and (normalization is not None or column not in interface.input_columns) + }, + ) + + +def _fit_input_metadata(config, name, binding, metadata, root): + """Fit missing raw codecs/moments on the same episode-safe training split.""" + from types import SimpleNamespace + + from sequifier.config.policy_config import ResolvedPolicyBinding + from sequifier.io.policy_dataset import PolicyDataCatalog + from sequifier.io.preference_dataset import prepare_preferences + from sequifier.rl.offline_data import prepare_offline + + phase = next( + ( + phase + for phase in config.training_plan.phases + if phase.algorithm != "supervised" + and phase_interface(config, phase) == name + ), + None, + ) + if phase is None or phase.algorithm not in {"policy_imitation", "iql", "dpo"}: + return metadata + interface = config.model.interfaces[name] + static = ( + interface.static_conditioning.columns if interface.static_conditioning else [] + ) + observations = [ + column + for column in interface.input_columns + if column not in binding.previous_action_inputs + and not (binding.return_conditioned and column == "__policy_return") + ] + categorical = [ + column + for column in observations + static + if "int" in metadata.column_data_types[column].lower() + ] + missing_codecs = set(categorical) - metadata.n_classes.keys() + missing_statistics = ( + set(observations + static) + - set(categorical) + - metadata.selected_columns_statistics.keys() + ) + if not missing_codecs and ( + not missing_statistics or config.global_training.normalization is not None + ): + return metadata + provisional = ResolvedPolicyBinding( + **binding.model_dump(), + context_length=config.global_training.context_length, + interface_name=name, + observation_columns=observations, + static_columns=static, + categorical_columns=categorical, + observation_n_classes={ + column: metadata.n_classes[column] + for column in observations + if column in metadata.n_classes + }, + static_n_classes={ + column: metadata.n_classes[column] + for column in static + if column in metadata.n_classes + }, + id_maps={ + column: mapping + for column, mapping in metadata.id_maps.items() + if column in observations + static + }, + input_schema={}, + ) + sources, preferences = {}, {} + for dataset_name, dataset in config.dataset_training.items(): + for part_name, part in dataset.parts.items(): + ref = f"{dataset_name}.{part_name}" + if isinstance(part, SequencePartSpecModel): + sources[ref] = resolve_sequence_source( + SequenceSourceConfig.model_validate( + part.model_dump(exclude={"type"}) + ), + root, + ) + elif isinstance(part, PreferencePartSpecModel): + preferences[ref] = str(root / part.manifest) + phase = phase.model_copy(deep=True) + for key in ("sources", "validation_sources"): + refs = getattr(phase, key, None) + if refs is not None: + expanded = [ + ref.model_copy(update={"source": full}) + for ref in refs + for full, _ in source_parts(config, ref.source) + ] + if any(ref.source not in sources for ref in expanded): + return metadata # Prepared sources already own frozen preprocessing metadata. + setattr(phase, key, expanded) + for key in ("preferences", "validation_preferences"): + ref = getattr(phase, key, None) + if ref is not None: + parts = source_parts(config, ref.source) + if len(parts) != 1: + raise ValueError(f"{phase.name}: select one preference part") + setattr(phase, key, ref.model_copy(update={"source": parts[0][0]})) + run = SimpleNamespace( + validation_fraction=config.global_training.validation_fraction, + seed=config.seed, + preferences=preferences, + sources=sources, + ) + catalog = PolicyDataCatalog( + sources, provisional, config.global_training.episode_cache_size + ) + data = ( + prepare_preferences(run, phase, provisional, catalog, {}) + if phase.algorithm == "dpo" + else prepare_offline(run, phase, provisional, catalog, {}) + ) + fitting = data.fitting_dataset() + fitted = fitting.fit_categorical_config() + values = metadata.model_dump() + values["n_classes"].update(fitted.categorical_n_classes) + values["id_maps"].update(fitted.id_maps) + if ( + config.global_training.normalization is None + and not metadata.depth_layouts + and not metadata.image_sources + ): + fitting.use_config(fitted) + for column, statistics in fitting.normalization_statistics().items(): + values["selected_columns_statistics"].setdefault(column, statistics) + return DatasetMetadata.model_validate(values) + + +def _supervised_subset(config): + phases = [ + phase + for phase in config.training_plan.phases + if phase.algorithm == "supervised" + ] + names = {ref.source.partition(".")[0] for phase in phases for ref in phase.sources} + if config.evaluation: + names.update( + ref.source.partition(".")[0] + for ref in config.evaluation.sources + if ref.source.partition(".")[0] in names + ) + return config.model_copy( + update={ + "dataset_training": { + name: dataset + for name, dataset in config.dataset_training.items() + if name in names + }, + "training_plan": config.training_plan.model_copy(update={"phases": phases}), + "policy_bindings": {}, + "policy_binding": None, + } + ) + + +def resolve_training_run(authored: SequifierConfig): + from sequifier.artifacts.model_artifact import read_execution_artifact + from sequifier.artifacts.model_config import ( + execution_config, + resolved_config_from_model_config, + ) + from sequifier.rl.attachment import apply_binding, resolve_binding + + root = Path(authored.project_root).resolve() + config = authored.model_copy(deep=True) + config.project_root = str(root) + source = None + if config.initialize_from: + config.initialize_from = str((root / config.initialize_from).resolve()) + source, _ = resolved_config_from_model_config( + read_execution_artifact(config.initialize_from).model_config.values, + device=config.device, + ) + if config.model is not None and config.model != source.model: + raise ValueError("declared model differs from initialize_from model") + config.model = source.model.model_copy(deep=True) + for field in ( + "context_length", + "training_objective", + "target_offset", + "forward_attention", + ): + actual = getattr(source.global_training, field) + if ( + field in config.global_training.model_fields_set + and getattr(config.global_training, field) != actual + ): + raise ValueError( + f"global_training.{field} differs from initialize_from" + ) + setattr(config.global_training, field, actual) + assert config.model is not None + if config.policy_binding is not None: + if len(config.model.interfaces) != 1: + raise ValueError("policy_binding requires exactly one model interface") + config.policy_bindings = { + next(iter(config.model.interfaces)): config.policy_binding + } + config.policy_binding = None + for dataset in config.dataset_training.values(): + if dataset.model_interface is None: + if len(config.model.interfaces) != 1: + raise ValueError("dataset model_interface is required when ambiguous") + dataset.model_interface = next(iter(config.model.interfaces)) + if dataset.model_interface not in config.model.interfaces: + raise ValueError( + f"unknown dataset model_interface {dataset.model_interface!r}" + ) + training = config.global_training + if training.context_length is None: + training.context_length = config.model.backbone.architecture.max_context_length + if training.training_objective is None: + if any( + phase.algorithm == "supervised" for phase in config.training_plan.phases + ): + raise ValueError( + "supervised phases require global_training.training_objective" + ) + training.training_objective = "policy" + training.target_offset = 0 + training = GlobalTrainingSpecModel.model_validate(training.model_dump()) + config.global_training = training + + # All declared metadata for an interface is an assertion of the same facts. + metadata = {} + declarations = training.metadata_config_path + if isinstance(declarations, str): + if len(config.model.interfaces) != 1: + raise ValueError( + "global_training.metadata_config_path must map interface names when ambiguous" + ) + declarations = {next(iter(config.model.interfaces)): declarations} + for name, path in (declarations or {}).items(): + if name not in config.model.interfaces: + raise ValueError(f"unknown metadata model_interface {name!r}") + metadata[name] = load_dataset_metadata(str(root / path)) + part_metadata = {} + for dataset_name, dataset in config.dataset_training.items(): + for part_name, part in dataset.parts.items(): + path = getattr(part, "metadata_config_path", None) + if path: + value = load_dataset_metadata(str(root / path)) + part_metadata[f"{dataset_name}.{part_name}"] = value + existing = metadata.get(dataset.model_interface) + if existing is not None: + for key in ( + "column_data_types", + "n_classes", + "id_maps", + "selected_columns_statistics", + "normalize_real_columns", + ): + if getattr(existing, key) != getattr(value, key): + raise ValueError( + f"{dataset_name}.{part_name}: interface metadata differs at {key}" + ) + metadata[dataset.model_interface] = value + interfaces = {} + for name, interface in config.model.interfaces.items(): + if source is not None: + interfaces[name] = source.interface(name).model_copy(deep=True) + if name in metadata: + candidate = _resolve_interface( + name, + interface, + metadata[name], + training, + metadata[name].storage_layout, + ) + for key in ( + "column_data_types", + "id_maps", + "selected_columns_statistics", + "target_decoder_ids", + ): + if getattr(candidate, key) != getattr(interfaces[name], key): + raise ValueError( + f"{name}: metadata differs from initialization at {key}" + ) + else: + value = metadata.get(name) + if value is None: + binding = config.policy_bindings.get(name) + if binding is None: + raise ValueError(f"{name}: model interface requires metadata") + value = _inferred_metadata(config, name, binding) + binding = config.policy_bindings.get(name) + if binding is not None: + value = _fit_input_metadata(config, name, binding, value, root) + interfaces[name] = _resolve_interface( + name, interface, value, training, value.storage_layout + ) + resolved = execution_config( + config.model, + interfaces, + training, + device=config.device, + project_root=str(root), + model_name=config.model_name, + seed=config.seed, + ) + for name, binding in config.policy_bindings.items(): + if name not in interfaces: + raise ValueError(f"policy_bindings: unknown interface {name!r}") + fitted = resolve_binding(resolved, name, binding) + apply_binding(resolved, name, fitted) + resolved.policy_bindings[name] = fitted + for name, interface in interfaces.items(): + if interface.policy is not None: + resolved.policy_bindings.setdefault(name, interface.policy) + + sequences, preferences = {}, {} + for dataset_name, dataset in config.dataset_training.items(): + for part_name, part in dataset.parts.items(): + ref = f"{dataset_name}.{part_name}" + if isinstance(part, PreferencePartSpecModel): + preferences[ref] = str((root / part.manifest).resolve()) + elif isinstance(part, SequencePartSpecModel): + sequences[ref] = resolve_sequence_source( + SequenceSourceConfig.model_validate( + part.model_dump(exclude={"type"}) + ), + root, + ) + elif isinstance(part, DatasetPartSpecModel): + sequences[ref] = resolve_sequence_source( + SequenceSourceConfig( + metadata_config_path=part.metadata_config_path, + split_index=0, + format="prepared", + encoding="model", + ), + root, + ) + for phase in config.training_plan.phases: + phase.optimization = phase.optimization_configs() + if phase.algorithm == "supervised": + for ref in phase.sources: + for _, part in source_parts(config, ref.source): + if not isinstance(part, DatasetPartSpecModel): + raise ValueError( + f"{phase.name}: supervised sources require prepared parts" + ) + continue + name = phase_interface(config, phase) + phase.model_interface = name + if name not in resolved.policy_bindings: + raise ValueError( + f"{phase.name}: interface {name!r} requires a policy binding" + ) + for key in ("sources", "validation_sources"): + refs = getattr(phase, key, None) + if refs is None: + continue + expanded = [] + for ref in refs: + for full, _ in source_parts(config, ref.source): + if full not in sequences: + raise ValueError( + f"{phase.name}: {full} is not a sequence source" + ) + expanded.append(ref.model_copy(update={"source": full})) + setattr(phase, key, expanded) + if phase.algorithm == "dpo": + for key in ("preferences", "validation_preferences"): + ref = getattr(phase, key) + if ref is None: + continue + parts = source_parts(config, ref.source) + if len(parts) != 1 or parts[0][0] not in preferences: + raise ValueError(f"{phase.name}: select one preference part") + setattr(phase, key, ref.model_copy(update={"source": parts[0][0]})) + for environment in ( + getattr(phase, "environment", None), + getattr(getattr(phase, "evaluation", None), "environment", None), + ): + if ( + environment + and environment.units != resolved.policy_bindings[name].units + ): + raise ValueError( + f"{phase.name}: environment units differ from policy units" + ) + reference = getattr(phase, "reference", None) + if reference and reference.path: + reference.path = str((root / reference.path).resolve()) + archive = getattr(phase, "archive_directory", None) + if archive: + phase.archive_directory = str((root / archive).resolve()) + for previous, current in zip( + config.training_plan.phases, config.training_plan.phases[1:] + ): + if ( + current.optimizer_state == "continue" + and previous.algorithm != "supervised" + and previous.model_interface != current.model_interface + ): + raise ValueError("optimizer continuation cannot change model_interface") + if all(phase.algorithm == "supervised" for phase in config.training_plan.phases): + subset = _supervised_subset(config).model_copy(update={"initialize_from": None}) + subset = SequifierConfig.model_validate(subset.model_dump()) + native = resolve_sequifier_config(subset, part_metadata) + for dataset in native.dataset_training.values(): + previous = interfaces[dataset.model_interface] + dataset.interface.policy = previous.policy + dataset.interface.prediction_distributions = ( + previous.prediction_distributions + ) + resolved = native + resolved.policy_bindings = { + name: interface.policy + for name, interface in interfaces.items() + if interface.policy is not None + } + resolved.authored = config + resolved.initialize_from = config.initialize_from + resolved.model_path = str( + (root / (config.model_path or f"models/{config.model_name}/model.pt")).resolve() + ) + resolved.report_path = str( + ( + root + / ( + config.report_path + or f"outputs/training/{config.model_name}/report.json" + ) + ).resolve() + ) + resolved.phase_directory = str( + ( + root + / (config.phase_directory or f"outputs/training/{config.model_name}/phases") + ).resolve() + ) + resolved.sequence_sources = sequences + resolved.preference_sources = preferences + if any(phase.algorithm != "supervised" for phase in config.training_plan.phases): + supervised = { + phase.name: phase + for phase in resolve_supervised_phases( + [ + phase + for phase in config.training_plan.phases + if phase.algorithm == "supervised" + ] + ) + } + resolved.training_plan = [ + supervised.get(phase.name, phase) for phase in config.training_plan.phases + ] + for key in ( + "export_generative_model", + "export_embedding_model", + "embedding_layer_names", + "export_onnx", + "export_pt", + "export_with_dropout", + ): + setattr(resolved, key, getattr(config, key)) + return resolved + + +def policy_run_config( + config, phases, *, incoming=None, index=0, destination=None, resuming=None +): + """Select resolved inputs for one native policy lifecycle.""" + from sequifier.artifacts.model_artifact import read_execution_artifact + from sequifier.artifacts.model_config import resolved_config_from_model_config + from sequifier.artifacts.model_export import model_execution_config + from sequifier.config.components import ResumeConfig + from sequifier.rl.attachment import apply_binding, resolve_binding + from sequifier.rl.run_config import ResolvedPolicyRun + + names = {phase.model_interface for phase in phases} + if len(names) != 1: + raise ValueError("a native policy lifecycle requires one interface") + name = next(iter(names)) + execution = config + initialization = incoming if incoming is not None else config.initialize_from + if initialization: + execution, _ = resolved_config_from_model_config( + read_execution_artifact(initialization).model_config.values, + device=config.device, + interface_name=name, + ) + apply_binding( + execution, + name, + resolve_binding(execution, name, config.policy_bindings[name]), + ) + binding = execution.interface(name).policy + if binding is None: + raise ValueError("selected interface has no policy binding") + training = config.global_training + resume = training.resume + root = Path(config.project_root) + checkpoint = ( + (root / resume.checkpoint_path).resolve() + if resume and resume.checkpoint_path + else root / f"checkpoints/{config.model_name}/run.pt" + ) + if resuming is not None: + checkpoint = ( + Path(config.phase_directory) / f"{index:03d}-{phases[0].name}" / "run.pt" + ) + return ResolvedPolicyRun( + project_root=config.project_root, + model_name=config.model_name, + device=config.device, + seed=config.seed, + enforce_determinism=training.enforce_determinism, + model=execution.model, + policy=binding, + execution_model=model_execution_config(execution), + policy_interface=name, + initialize_from=initialization, + normalization=training.normalization, + units=binding.units, + sources=config.sequence_sources, + preferences=config.preference_sources, + phases=phases, + validation_fraction=training.validation_fraction, + episode_cache_size=training.episode_cache_size, + resume=ResumeConfig( + checkpoint_path=str(checkpoint), + policy=("if_exists" if resuming else "never") + if resuming is not None + else (resume.policy if resume else "never"), + mode=resume.mode if resume else "reset_environments", + ), + model_path=str(destination or config.model_path), + report_path=config.report_path + if destination is None + else str(Path(destination).with_name("report.json")), + checkpoint_path=str(checkpoint), + phase_directory=config.phase_directory + if destination is None + else str(Path(destination).parent / "native"), + ) diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index 2b62309e..ddabae20 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -958,7 +958,7 @@ IQL and online PPO through the same commands, see [policy learning](../policy-le `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated -adapter, and a decoder. Datasets own data and training policy; `model` +adapter, and a decoder. Phases own batching and optimization; datasets own data and losses; `model` owns architecture. ```console @@ -975,40 +975,60 @@ project_root: . model_name: event-model device: cuda seed: 1010 - global_training: read_format: parquet training_objective: causal context_length: 128 inference_batch_size: 256 - batch_size: 64 - learning_rate: 0.0001 - model: backbone: architecture: dim_model: 128 max_context_length: 512 num_layers: 6 - attention: {type: mha, n_heads: 8, n_kv_heads: 8, output_projection: true} - feed_forward: {dim: 512, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: rope, theta: 10000} + attention: + type: mha + n_heads: 8 + n_kv_heads: 8 + output_projection: true + feed_forward: + dim: 512 + activation: swiglu + normalization: + type: rmsnorm + norm_first: true + position_encoding: + type: rope + theta: 10000 dropout: 0.1 shared_layer_groups: [] interface: - input_columns: [event] - target_columns: [event] - ingestion: {type: embedding, output_dim: 128} - decoder: {type: linear, prediction_length: 1, support: 1} - + input_columns: + - event + target_columns: + - event + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 + support: 1 dataset: - part: {metadata_config_path: configs/metadata_configs/events.json} - criterion: {event: CrossEntropyLoss} - + part: + metadata_config_path: configs/metadata_configs/events.json + criterion: + event: CrossEntropyLoss training_plan: - epochs: 5 - + phases: + - epochs: 5 + mode: sequential + name: supervised + algorithm: supervised + batch_size: 64 + optimization: + model: + learning_rate: 0.0001 evaluation: true ``` @@ -1062,7 +1082,6 @@ project_root: . model_name: event-model device: cuda seed: 1010 - global_training: read_format: parquet training_objective: causal @@ -1070,62 +1089,108 @@ global_training: target_offset: 1 window_stride: 1 inference_batch_size: 256 - batch_size: 64 - accumulation_steps: 4 - learning_rate: 0.0001 - optimizer: {name: AdamW, weight_decay: 0.01} - scheduler: {name: StepLR, step_size: 1, gamma: 0.99} - scheduler_step_on: epoch - reset_optimization_on_phase: true - gradient_clip: 1.0 save_interval_epochs: 1 - model: backbone: architecture: dim_model: 128 max_context_length: 512 num_layers: 6 - attention: {type: mha, n_heads: 8, n_kv_heads: 8, output_projection: true} - feed_forward: {dim: 512, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: rope, theta: 10000} + attention: + type: mha + n_heads: 8 + n_kv_heads: 8 + output_projection: true + feed_forward: + dim: 512 + activation: swiglu + normalization: + type: rmsnorm + norm_first: true + position_encoding: + type: rope + theta: 10000 dropout: 0.1 shared_layer_groups: [] interfaces: event_prediction: - input_columns: [event] - target_columns: [event] - categorical_decoder_special_tokens: {event: [other]} - ingestion: {type: embedding, output_dim: 128} - decoder: {type: linear, prediction_length: 1, support: 1} - + input_columns: + - event + target_columns: + - event + categorical_decoder_special_tokens: + event: + - other + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 + support: 1 dataset_training: events: model_interface: event_prediction parts: - original: {metadata_config_path: configs/metadata_configs/events.json} - increment: {metadata_config_path: configs/metadata_configs/events-increment.json} - criterion: {event: CrossEntropyLoss} - loss_weights: {event: 1.0} + original: + metadata_config_path: configs/metadata_configs/events.json + increment: + metadata_config_path: configs/metadata_configs/events-increment.json + criterion: + event: CrossEntropyLoss + loss_weights: + event: 1.0 freeze: - backbone: {freeze: [attention.qkv]} - + backbone: + freeze: + - attention.qkv training_plan: phases: - - name: incremental_finetuning - epochs: 2 - mode: sequential - sources: [{source: events.increment}] - - name: complete_retraining - epochs: 5 - mode: interleaved - selection: round_robin - sources: [{source: events, batches_per_selection: 4}] - + - name: incremental_finetuning + epochs: 2 + mode: sequential + sources: + - source: events.increment + batch_size: 64 + accumulation_steps: 4 + optimization: + model: + learning_rate: 0.0001 + optimizer: + name: AdamW + weight_decay: 0.01 + scheduler: + name: StepLR + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip: 1.0 + optimizer_state: reset + - name: complete_retraining + epochs: 5 + mode: interleaved + selection: round_robin + sources: + - source: events + batches_per_selection: 4 + batch_size: 64 + accumulation_steps: 4 + optimization: + model: + learning_rate: 0.0001 + optimizer: + name: AdamW + weight_decay: 0.01 + scheduler: + name: StepLR + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip: 1.0 + optimizer_state: reset evaluation: - sources: [{source: events}] - + sources: + - source: events export_generative_model: true export_embedding_model: false export_onnx: true @@ -1289,27 +1354,38 @@ target in each dataset must have a positive weight. ## Optimizer plans -The single-optimizer form above continues to apply one optimizer to all -parameters and uses `global_training.learning_rate`. To select parameters or +The single-optimizer form applies one optimizer to all +parameters and uses the role’s `learning_rate`. To select parameters or use multiple optimizers, replace it with an ordered plan. Each group has its -own learning rate; omit the run-wide `learning_rate` in this form: +own learning rate; use each group’s `learning_rate` in this form: ```yaml -global_training: - optimizer: - groups: - - id: hidden_matrices - select: - semantic_groups: ["attention.*", "feed_forward.*"] - component: backbone - parameter_kind: weight - ndim: 2 - optimizer: {name: Muon, momentum: 0.95} - learning_rate: 0.02 - - id: remainder - select: otherwise - optimizer: {name: AdamW, weight_decay: 0.01} - learning_rate: 0.0003 +training_plan: + phases: + - name: supervised + algorithm: supervised + optimization: + model: + optimizer: + groups: + - id: hidden_matrices + select: + semantic_groups: + - attention.* + - feed_forward.* + component: backbone + parameter_kind: weight + ndim: 2 + optimizer: + name: Muon + momentum: 0.95 + learning_rate: 0.02 + - id: remainder + select: otherwise + optimizer: + name: AdamW + weight_decay: 0.01 + learning_rate: 0.0003 ``` Each supplied selector condition must match. `semantic_groups` matches the @@ -1341,38 +1417,83 @@ valid with `CyclicLR`. ## Optimization across phases -By default, every training phase starts with a new optimizer, scheduler, and -gradient scaler. Model weights and global progress counters continue across -phase boundaries, while optimizer moments and scheduler position do not. Set -`global_training.reset_optimization_on_phase: false` to keep the optimizer, -scheduler, and gradient scaler operating continuously across all phases. A -checkpoint restores the active optimization lifecycle exactly in either mode. - -For an epoch-stepped scheduler in reset mode, do not provide -`global_training.scheduler.total_steps` because Sequifier owns each phase's -scheduler duration. When a scheduler such as `OneCycleLR` accepts `total_steps`, -Sequifier sets it to the active phase's `epochs` in reset mode, or the sum of all -phase epochs in continuous mode. A continuous epoch-stepped scheduler may still -provide `total_steps` for compatibility, but it must equal the sum of all phase -epochs. Other scheduler arguments remain shared across phases. This example uses -a single optimizer: +Each phase owns `batch_size` (32 by default), `accumulation_steps` (1), and +`optimization.`. Supervised phases have the single role `model`, meaning +the trainable model parameters. Imitation, DPO and GRPO have `actor`; PPO also +has `value`; IQL has `actor`, `value`, and `critic`. Unknown roles fail validation. +Omitted roles use Adam, learning rate 0.0003 and gradient clip 1.0. Set +`gradient_clip: null` to disable clipping. Optimizer plans take their rates from +their groups. Supervised roles default to StepLR (`step_size: 1`, `gamma: 0.99`); +policy roles default to ConstantLR (`factor: 1`, `total_iters: 1`). ```yaml -global_training: - scheduler: - name: OneCycleLR - max_lr: 0.003 - pct_start: 0.2 - cycle_momentum: false - scheduler_step_on: epoch - reset_optimization_on_phase: false +training_plan: + phases: + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: [{source: observations.train}] + batch_size: 64 + accumulation_steps: 1 + optimizer_state: reset + optimization: + model: + optimizer: {name: Adam} + learning_rate: 0.001 + scheduler: {name: StepLR, arguments: {step_size: 1, gamma: 0.99}} + scheduler_step_on: epoch + gradient_clip: 1.0 ``` -Batch-stepped schedulers may configure `total_steps` explicitly because phase -update counts can depend on source interleaving, dataset transitions, and -gradient accumulation. In reset mode, that value applies independently to each -phase; configuration emits a warning to make that scope explicit. In continuous -mode, it applies once across the full run. +`optimizer_state: reset | continue` describes the transition **into** this phase; +its default is `reset`. Reset creates new optimizer, scheduler and scaler state. +Continue preserves these states from the immediately preceding phase and requires +that phase's `handoff: last`, identical algorithm, role configurations, parameter +ownership and freeze selections. The first phase cannot continue. Supervised +continuation requires the same dataset ownership; policy continuation requires +the same interface. Mixed plans reject continuation because their artifact +handoffs do not preserve optimizer state. Policy OneCycleLR continuation is +also rejected. Native checkpoints restore the active lifecycle in either mode. Supervised +`handoff: best` requires an evaluation monitor and selects the preceding phase’s +best weights before starting the next phase; its monitor state is phase-local. + +Supervised `scheduler_step_on: epoch` advances once after a completed phase epoch; +ReduceLROnPlateau receives the configured evaluation monitor at that boundary +and requires `evaluation.monitor`. `batch` advances once per applied optimizer +update, after accumulation, rather than once per microbatch. Policy roles only +accept `scheduler_step_on: optimizer_step`, their default. Policy plateau and +compound schedulers are unsupported. Constructor arguments are validated before +training; the shared optimizer shape does not imply shared scheduler capabilities. + +For epoch OneCycleLR, the resolved `total_steps` is the phase's epoch count, or +the sum of epochs in its contiguous continuation chain. An explicit value must +match. Batch OneCycleLR requires an explicit `total_steps`. Preflight counts loader +batches using the same sequential/interleaved source scheduler, phase seeds and +accumulation rules as training, including partial updates flushed at dataset +boundaries. The budget sums over a contiguous continuation chain. This calculation +does not consume samples or advance loader RNG. Early stopping and explicitly +skipped updates may consume less than the authored maximum. +Policy OneCycleLR requires the exact maximum authored optimizer-update budget: + +| Algorithm | Optimizer updates per role | +| --- | --- | +| Imitation | epochs × ceil(training examples / (batch_size × accumulation_steps)) | +| IQL / DPO | updates | +| PPO | optimization_epochs × sum(ceil(rollout size / (minibatch_size × accumulation_steps))) | +| GRPO | optimization_epochs × sum(ceil(group trajectories / (batch_size × accumulation_steps))) | + +PPO `rollout_length` counts collected transitions; `minibatch_size` counts +transitions in each optimization microbatch. They are distinct budgets. IQL +uses `batch_size: 64` by default and one update per sampled batch; it does not +accept accumulation. DPO counts preference pairs; GRPO counts trajectories. +PPO defaults to `minibatch_size: 64`; other batched algorithms default to 32. +Early stopping, skipped updates or KL limits can consume less than the maximum. + +The authored schema is separate from resolved training phases. Checkpoints and +plan fingerprints contain resolved roles, batching and scheduler budgets; +reproduction does not depend on global optimizer values. `global_training` +contains execution/distribution, resume, data/window and model contracts only. ## Training plans @@ -1721,34 +1842,18 @@ healthy active update. Momentum and weight decay cannot update parameters whose gradients are absent. -## Policy runs - -Policy training uses one `PolicyRunConfig` with top-level ordered `phases`, -`model`, `policy_interface`, `context_length`, optional named `sources` and -`preferences`, and run-wide output/resume settings. Supported algorithms are -`policy_imitation`, `iql`, `ppo`, `dpo`, and `grpo`. Epochs, offline updates, -PPO environment steps and GRPO groups are explicit per-phase budgets. +## Policy and mixed plans -Each phase selects `handoff: best | last` (default `best`). One frozen actor -representation and causal history contract spans the run; all optimizer/critic -state starts fresh at each handoff. Return conditioning supports imitation only. -Standalone legacy policy runs use their existing phase schema. Use -`run_type: unified` for supervised and policy phases in one plan, or -`initialize_from` plus `policy_binding.actions` to post-train an ordinary -supervised artifact. See the [unified learning guide](../supervised-policy-learning.md) -for typed actions, source bindings, prediction distributions, and recovery. +All algorithms use `training_plan.phases`. Declare policy sequence/preference +parts in `dataset_training`, and attach actions with `policy_binding` (one +interface) or `policy_bindings` (named interfaces). Run-wide settings remain in +`global_training`; no alternate run discriminator is needed. -See the [policy learning guide](../policy-learning.md) for full fields, objective -formulas, environment capabilities, preference manifests, checkpoint boundaries, -selection monitors and exact/non-exact continuation. Examples: - -- [BC](../example_inputs/configs/train-bc-toy.yaml) -- [Return-conditioned imitation](../example_inputs/configs/train-return-conditioned.yaml) -- [IQL](../example_inputs/configs/train-iql-from-table.yaml) -- [PPO](../../src/sequifier/templates/train-ppo.yaml) -- [DPO](../../src/sequifier/templates/train-dpo.yaml) -- [GRPO](../../src/sequifier/templates/train-grpo.yaml) -- [Imitation → IQL → DPO → GRPO → PPO](../example_inputs/configs/train-multiphase-control.yaml) +See the complete [supervised template](../../src/sequifier/templates/train-supervised.yaml), +[policy template](../../src/sequifier/templates/train-policy.yaml), and +[supervised-to-policy template](../../src/sequifier/templates/train-supervised-to-policy.yaml). +The [policy guide](../policy-learning.md) describes algorithm budgets and +[the mixed-plan guide](../supervised-policy-learning.md) describes artifact handoffs. # Infer Command Guide @@ -1870,7 +1975,7 @@ fragments or define the same field twice. | --- | --- | --- | | `model_path` | Required | Model path, or a list of compatible model paths. | | `model_type` | Required | `generative` or `embedding`. | -| `device` | Required | ONNX: `cpu` or `cuda`; PT also supports `mps`. ONNX CUDA requires an installed runtime exposing `CUDAExecutionProvider`. | +| `device` | `cpu` | ONNX: `cpu` or `cuda`; PT also supports `mps`. ONNX CUDA requires an installed runtime exposing `CUDAExecutionProvider`. | | `project_root` | Required | Base for project paths; normally `.`. | | `data_path` | Metadata test/last split | Input file or folder. Usually required with artifact-only inference because artifacts do not store split paths. | | `preprocessing_data_path` | `null` | Derives the generated metadata path. | @@ -1921,7 +2026,7 @@ stored folder window if the remaining layout and semantics match. | `decode_categories` | `true` | Decode categorical predictions to their original values. | | `sample_from_distribution_columns` | `null` | Sample these categorical targets instead of using argmax. | | `infer_with_dropout` | `false` | Enable dropout at inference; ONNX also requires dropout-preserving export. | -| `deterministic` | `false` | Request deterministic PyTorch algorithms. | +| `enforce_determinism` | `false` | Request deterministic PyTorch algorithms. | | `seed` | `1010` | Random seed. | | `autoregressive` | `false` | Feed predictions back for multi-step generation. | | `generation_steps` | `null` | Required positive step count when autoregressive is enabled. | @@ -1943,10 +2048,12 @@ its stored windows rather than this first-window reduction. ## CLI overrides `--data-path`, `--input-columns`, `--metadata-config-path`, `--model-path`, -`--seed`, `--dataset`, `--part`, and `--model-interface` override YAML values. +`--seed`, `--dataset`, `--part`, `--model-interface`, `--model-type`, +`--enforce-determinism`, and `--action-selection` override YAML values. `--randomize` takes precedence over `--seed`. `--skip-metadata` skips external metadata loading; inline metadata, a selected training route, or a self-describing -PT artifact must then provide the required metadata. +PT artifact must then provide the required metadata. `--skip-metadata` is +forecast-only and is rejected on policy inference. ## Outputs @@ -2020,16 +2127,75 @@ ORT installation. Outputs stay indexed by outer item coordinates. ## Policy model interfaces -`sequifier infer` recognizes a policy interface in a portable PyTorch actor. -Use the selected actor export rather than the policy run checkpoint. Use raw -CSV/Parquet observation rows and optional recorded action histories; alignment, vocabulary, and normalization come from the artifact. -`model_type: generative` and `model_type: policy` are accepted. Environment -execution uses a policy `rollout` section with `environment` (a version-3 adapter -supporting `seeded_reset`), `trajectory_path`, `seed_index`, and optional -`max_episode_steps`. Set `autoregressive: true` and `generation_steps`; policy -rollout does not configure forecast bindings. See the -[policy guide](../policy-learning.md#python-and-inference) and -[inference example](../example_inputs/configs/infer-policy.yaml). +`model_type` selects the authored route before artifact resolution. `generative` +and `embedding` use forecast schemas; `policy` uses `PolicyInferenceConfig` +directly. Unknown and cross-route fields fail validation even when set to their +default values. CLI overrides and Python configuration loaders follow the same +selection rules. + +Policy inference requires one portable PT artifact and one unambiguous policy +interface. Supply raw CSV/Parquet observation rows; history alignment, vocabulary +and normalization come from that artifact. Policy supports the common model path, +interface, data path, formats, device, seed, `enforce_determinism`, batch size fields, plus: + +| Policy field | Default / meaning | +| --- | --- | +| `output_path` | null; write policy predictions here instead of the generated output path | +| `action_selection` | `deterministic`; choose the policy mode. `stochastic` samples. | +| `desired_return` | null; fixed return budget for a return-conditioned policy | +| `desired_return_column` | `desired_return`; per-row budget when no fixed budget is supplied | +| `previous_action_columns` | {}; input-column mapping for recorded previous actions | +| `sequence_id_column`, `item_position_column` | `sequenceId`, `itemPosition`; order raw history rows | +| `output_probabilities` | false; include policy distribution outputs | +| `rollout` | null; execute the policy in a version-3 online environment | + +Policy assertions may specify `metadata_config_path`, `input_columns`, +`static_columns`, `target_columns`, `column_data_types`, `target_column_types`, +`training_objective`, `context_length`, `target_offset`, `prediction_length`, and +`decoder_support`. Every supplied assertion must match the selected artifact. +Forecast fields `sample_from_distribution_columns`, `infer_with_dropout`, +`decode_categories`, dataset/part selection, image path overrides, +`autoregressive`, and `generation_steps` are rejected on policy requests. +Policy-only return/action/history options and the policy-only `output_path` +are rejected on forecast requests. Embedding requests also reject generative +sampling, category decoding, probability output, autoregression and rollout fields; +dropout remains supported for embeddings. + +```yaml +project_root: . +model_type: policy +model_path: models/policy/model.pt +model_interface: actor +data_path: observations.csv +read_format: csv +device: cpu +seed: 1010 +enforce_determinism: true +action_selection: stochastic +rollout: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + units: {} + max_steps: 8 + seed_index: 0 + trajectory_path: outputs/trajectory.json +``` + +Rollout schemas are selected by the inference route; do not declare `rollout.type`. +Forecast rollout requires `bindings` and a version-1 environment. Policy rollout +rejects bindings and requires the distinct version-3 environment protocol, +including implementation identity and matching units. Both use `max_steps` and +optional `max_episode_steps`. `seed_index` counts prepared windows in forecast +rollout and raw input rows in policy rollout, each zero-based in input order. + +`enforce_determinism` controls deterministic execution, independently of action +selection and seed. A seeded stochastic policy can reproduce its sampled sequence +under the same supported runtime, device and input ordering. Online evaluation +uses `action_selection` with the same meaning; its execution determinism is the +training run's `global_training.enforce_determinism`. See the +[environment protocols](../rollout.md) and [policy guide](../policy-learning.md). # Visualize Training Command Guide @@ -2357,113 +2523,51 @@ input schemas, and mixed supervised/policy plans, see the [unified learning guide](supervised-policy-learning.md). The configuration below documents the compatible single-categorical authoring surface. -`sequifier train` and `sequifier.rl_api.run_policy_training` execute the same -`PolicyRunConfig`. Its ordered `phases` may contain `policy_imitation`, `iql`, -`ppo`, `dpo`, and `grpo` in any compatible order. Each phase takes ownership -of its predecessor's selected actor. Optimizers, schedulers, critics, samplers, -and environments start fresh at phase boundaries. - -This legacy surface supports one fixed categorical action, scalar or categorical -observations/static fields, causal fixed-length histories, FP32, and local -single-process CPU/CUDA/MPS execution. Return conditioning is available only -for imitation. A run containing another algorithm requires an unconditioned -actor throughout. Mixed phases use the unified run configuration linked above. -For this legacy surface, text tokenizers, continuous/joint actions, images/depth, distributed training, -vectorized environments, and reward models are outside this contract. +`sequifier train` accepts one `SequifierConfig` with ordered +`training_plan.phases`. Supervised, imitation, IQL, PPO, DPO and GRPO use the same +model/interface and artifact contracts. A policy binding selects named real or +categorical action heads. See the [training guide](configs/train.md) for phase +ownership, scheduling budgets, optimizer groups and transition validation. +Policy execution currently uses a local CPU/CUDA/MPS process; standalone +supervised training also supports distributed execution. ## Authoring a run -Start with the [multi-phase example](example_inputs/configs/train-multiphase-control.yaml), -[single-phase DPO](../src/sequifier/templates/train-dpo.yaml), or -[single-phase GRPO](../src/sequifier/templates/train-grpo.yaml). -Generate their curated data with: +Use the [policy template](../src/sequifier/templates/train-policy.yaml), +[PPO template](../src/sequifier/templates/train-ppo.yaml), or +[multiphase example](example_inputs/configs/train-multiphase-control.yaml). +The latter contains imitation → IQL → DPO → GRPO → PPO. Generate its data with: ```shell python documentation/example_inputs/policy_control_data.py data sequifier train --config-path documentation/example_inputs/configs/train-multiphase-control.yaml ``` -The example defines imitation → IQL → DPO → GRPO → PPO. Remove or reorder phase -entries to author smaller runs, such as imitation → IQL, imitation → PPO, -imitation → DPO, imitation → GRPO, or DPO → GRPO. The input preferences come from -an explicitly supplied sign-matching rule; Sequifier never derives preferences -from recorded reward. These are illustrative configurations, without a claim -that every algorithm order improves every task. - -```yaml -project_root: . -model_name: control -policy_interface: actor -context_length: 4 -seed: 1010 -device: cpu -# model uses the ordinary backbone/interfaces architecture schema. -model: { ... } -sources: - decisions: - paths: [data/control-decisions.csv] - columns: - episode_id: episodeId - group: contextId -preferences: - rankings: data/control-preferences.json -phases: - - name: imitate - algorithm: policy_imitation - sources: [decisions] - epochs: 4 - - name: rank - algorithm: dpo - preferences: rankings - updates: 100 - beta: 0.1 - reference: {source: phase_start} -resume: {policy: if_exists, mode: exact} -``` - -Run-wide fields: - -| Field | Meaning / default | -| --- | --- | -| `model`, `policy_interface`, `context_length`, `model_name`, `phases` | Required architecture, selected interface, history length, run name, ordered phases | -| `project_root`, `device`, `seed`, `enforce_determinism` | `.`, `cpu`, 1010, true; seed in `[0, 2**32)` | -| `sources` | Named `SequenceSourceConfig` objects; paths/manifest, format, explicit column mappings, prepared metadata | -| `preferences` | Named paths to version-1 preference-pair JSON manifests | -| `units` | Frozen observation/static units; `{}` means unitless; online training/evaluation adapters must match | -| `initialize_from` | Optional portable actor with matching full representation and architecture | -| `normalization` | Optional explicit frozen `source: identity` or `source: statistics` with exact real-field `statistics: {column: {mean: ..., std: ...}}` | -| `validation_fraction`, `episode_cache_size` | 0.2 and 8; split independent components before fitting codecs/statistics | -| `resume` | `{policy: never, mode: reset_environments}`; policy also `if_exists`/`required`, mode also `exact` | -| `model_path` | `models//actor.pt` | -| `report_path` | `outputs/training//report.json` | -| `checkpoint_path` | `checkpoints//run.pt` | -| `phase_directory` | `outputs/training//phases`; subdirectories `000-`, etc. | - -The model must contain exactly `policy_interface`. Its input columns list the -policy observations followed by static fields; its target is the action column. -Use a scalar categorical decoder with `support: 1`, `prediction_length: 1`. -Declare `return_conditioned: false` for BC or any improvement algorithm. Standard -ingestion/backbone/decoder configuration remains available. Previous executable -actions enter the input as `__policy_previous_action`; its start token is -input-only. Return-conditioned imitation additionally uses `__policy_return`. - -The first offline phase fits missing categorical vocabularies and normalization -on its training episodes only. A declared normalization takes precedence over -fitting statistics. An online-first run must supply frozen normalization and -categorical counts/codecs, or a compatible `initialize_from` actor. Later phases -reuse the complete frozen representation. They may optionally declare -`representation` and online `normalization` as assertions; any disagreement -fails with a field-specific diagnostic before training. An online phase after -an offline phase normally omits `normalization` to reuse the fitted values. - -All input files, preference manifests, metadata, initial actors and references -are protected from output collisions. Source bytes, partitions, codecs, model, -units, reference weights, and adapter identities are checked on resume. +All sources live in `dataset_training..parts.`. Use `type: sequence` +for raw tables or exported sequence manifests, prepared metadata parts for +supervised data, and `type: preferences` with `manifest` for preference pairs. +Phase source references are objects such as `{source: decisions.default}`; +DPO `preferences` uses the same reference shape. Every data-bearing dataset +selects its model interface explicitly when more than one is available. + +Run-wide `context_length`, normalization, validation fraction (0.2), episode +cache size (8), determinism and resume settings live in `global_training`. +`resume` defaults to `{policy: never, mode: reset_environments}`; use `if_exists` +or `required` with a checkpoint path, and `exact` for supported online adapters. +`policy_binding` applies to a single model interface; use `policy_bindings` keyed +by interface name for multiple interfaces. Environment units must match the binding. +No optimizer defaults live in this run-wide block. + +Default outputs are `models//model.pt`, +`outputs/training//report.json`, and +`outputs/training//phases`. The resolved configuration records +role defaults and execution contracts. Fingerprints and checkpoints include +phase settings and validate source, reference and environment identities on resume. ## Selection and optimization Every phase has a unique identifier `name` and `handoff: best | last` (default -`best`). Best selection retains the earliest finite best on ties. The final +`last`). Best selection retains the earliest finite best on ties. The final update always evaluates. Missing finite evaluation for `best` fails explicitly. `last` selects the most recently completed optimizer transaction. The final export uses the last phase's selected actor, with metric, transaction and weight @@ -2689,7 +2793,7 @@ independent seeds and a whole-evaluation POSIX/main-thread deadline. It evaluate the live actor synchronously and restores every module's training mode and the training RNG state afterward. It requires `environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; -`selection: deterministic | stochastic` defaults to deterministic. Default +`action_selection: deterministic | stochastic` defaults to deterministic. Default `metric: mean_completed_return`, `direction: max` selects true completed returns. `mean_completed_length` and `success_rate` are also supported. Censored episodes and unavailable metrics are explicit; best selection fails when no finite @@ -2804,38 +2908,432 @@ source/reference provenance, selections, clocks and continuation modes. ## Python and inference +The Python API starts from the same authored schema as YAML: + ```python -from sequifier.config.train_config import load_train_config -from sequifier.rl_api import PolicyRunConfig, run_policy_training, load_policy - -config = load_train_config("configs/train-policy.yaml", {}, False) -if not isinstance(config, PolicyRunConfig): - raise TypeError("expected a policy run") -result = run_policy_training(config) -actor = load_policy(result.model_path) -for phase in result.phases: - print(phase.name, phase.selected_metric, phase.clocks) +from sequifier.config.train_config import SequifierConfig +from sequifier.config.training_resolution import resolve_training_run +from sequifier.train import run_training +import yaml + +with open("configs/train-policy.yaml") as stream: + authored = SequifierConfig.model_validate(yaml.safe_load(stream)) +resolved = resolve_training_run(authored) +result = run_training(resolved) ``` -`PolicyRunConfig.model_validate` accepts the same mapping directly. The single -`PolicyRunResult` exposes `policy`, ordered `phases`, `representation` and output -paths. `rl_api` also exports the phase schemas, preference records, online -capabilities, `PolicyWindowView`, and model loading/execution helpers. - -Use `sequifier infer` with the final artifact for categorical sequence inference -or rollout. `collect_policy_rollout` uses the same causal `PolicyHistory` and -`PolicyCursor` decision/step path as online training and evaluation. -Return-conditioned actors require -an explicit desired return. `PolicyRolloutConfig` declares `environment`, -`trajectory_path`, `seed_index` and optional `max_episode_steps`; the environment -must support seeded reset. Successors always come from the environment, so no -prediction bindings are configured. A true terminal may omit its successor; -truncation and collection cutoffs must retain it. -See [policy inference](configs/infer.md), [rollout](rollout.md), and the +`run_training` selects the supervised engine, native policy runner, or mixed-plan +coordinator from the resolved phase algorithms. `load_train_config(path, {}, False)` +combines YAML loading and resolution. `run_policy_training` takes an internal +resolved policy run, rather than a separate authored configuration. Direct Python +inference can validate `PolicyInferenceConfig`, resolve it with +`policy_inference_config(request.model_dump(exclude_unset=True))`, and call +`sequifier.rl.inference.infer_policy`; it uses the same schema as the YAML loader. +`rl_api` exports phase schemas, policy bindings, action declarations, environment +capabilities, sequence records and `load_policy`. + +Use `model_type: policy` with a selected portable PT artifact for raw CSV/Parquet +observation rows. `enforce_determinism` requests deterministic execution; +`action_selection: deterministic | stochastic` selects the policy mode or sampling. +The seed is independent of both. Seeded sampling can reproduce a sequence under +the same supported runtime, device and input order. Online evaluation uses +`action_selection`, defaulting to deterministic, with the run's execution settings. +Evaluation reports record that choice. + +Policy `rollout.environment` declares the version-3 environment, `max_steps` +sets the collection budget, and `seed_index` selects a zero-based raw input row. +`trajectory_path` is required; `max_episode_steps` is optional. Policy rollout +rejects forecast bindings and generation settings. It requires seeded observation +reset; successors come from actual environment transitions. True termination may +omit a successor; truncation and collection cutoffs retain it. Forecast rollout +instead selects a prepared model window and follows the separate version-1 +protocol. See [inference](configs/infer.md), [rollout](rollout.md), and the [Python PPO workflow](example_inputs/ppo_toy_control.py). -CPU determinism is the reference execution contract. CUDA and MPS reproducibility -is limited by the selected PyTorch backend/operators and matching device/RNG -topology. This implementation was statically checked; no tests or training runs -were written/executed for this change, so numerical and end-to-end acceptance -verification remains outstanding. +CPU tests cover the reference execution contract. CUDA and MPS reproducibility +also depends on backend/operators and matching device/RNG topology. + + +# Supervised models and policy learning + +Policy learning uses the same model interfaces, input tensors, decoder branches, +categorical codecs, and portable artifacts as supervised learning. A policy +binding identifies the output heads that form an environment action. The +binding does not automatically add inputs or replace a pretrained decoder. + +## Post-train a supervised artifact + +An ordinary supervised PT model, or a supported supervised run checkpoint, can +initialize a policy run. Its interface does not need existing policy metadata. + +```yaml +project_root: . +model_name: posttrained +initialize_from: models/supervised/model.pt +policy_binding: + actions: + mode: {type: categorical, values: ["off", "on"]} + price: {type: real, scale: 0.5} +dataset_training: + demonstrations: + parts: + recorded: {type: sequence, paths: [data/demonstrations.parquet], format: table} +training_plan: + phases: + - name: imitate + algorithm: policy_imitation + sources: [{source: demonstrations.recorded}] + epochs: 5 + batch_size: 64 + optimization: + actor: {learning_rate: 0.0003} +``` + +Use the actual interface and target names from the source artifact. Quote YAML +labels such as `"off"` and `"on"` if the YAML parser interprets them as booleans. +Omitting categorical `values` selects the artifact's entire decoder vocabulary, +including any executable special tokens. Explicit values select a subset and +are reported as an adaptation when that changes the original support. + +The artifact provides the architecture, context length, target positions, +normalization, and class maps. A separately supplied model must agree with it. +Original weights load strictly. Only explicitly introduced exploration parameters +are initialized. The run report records the transfer and its new/reused parameters. + +## Real and categorical heads + +Every real action head requires an explicit positive `scale`, measured in the +head's physical units. Its pretrained scalar output is the Gaussian location; +categorical heads retain their logits and persisted decoder-ID semantics. + +| Setting | Meaning | +| --- | --- | +| `scale` | Fixed Gaussian standard deviation by default. | +| `learn_scale: true` | Learn a global log scale, bounded by `min_scale` and `max_scale`. | +| `lower` / `upper` | Transform the Gaussian onto a positive, negative, or finite open interval using a softplus or logistic bijection. | +| `atoms: {0.0: 0.05}` | Assign fixed probability mass to an exact real value, alongside the continuous component. Probabilities must sum to less than one. | +| `active_when: {mode: [on]}` | Activate a field only for these values of an earlier action field. | +| `inactive_value: 0.0` | Deterministic value used when a field is inactive. | + +Bounds change the deterministic prediction function and are reported as an +adaptation. Samples are never clipped and then scored as unclipped Gaussians. +Collection retains transform latents for stable replay near numerical boundaries. +Externally supplied exact endpoints require a declared point mass. Inactive +fields are point masses with zero log-probability contribution, rather than +Gaussians with zero variance. + +The existing decoder determines factorization. Ordinary branches predict +independent fields; an autoregressive branch samples in its existing target +order, embedding each sampled prefix value. Recorded actions are scored with +their recorded prefix. Selecting a later autoregressive target requires selecting +all its predecessors. Composite branches and tied weights are retained. Hashed +categorical outputs are normalized over canonical executable candidates. + +Likelihoods sum over active action fields once per decision. Supervised loss +weights do not alter PPO ratios or DPO likelihoods. Independent categorical and +unbounded Gaussian entropy is exact; dependent/transformed entropy uses a +score-function estimator that includes the sampled prefix distribution. These +estimates can have higher variance. The GRPO sampled reference penalty retains +its existing interpretation. + +All five algorithms consume this joint action contract: imitation, IQL, PPO, +DPO, and GRPO. IQL projects real action fields and embeds categorical fields +before evaluating its scalar twin Q-functions. + +## Inputs, history, and output positions + +The full persisted execution schema describes scalar, static, depth, image, and +mask tensors. Online adapters can return nested JSON values, or array/tensor +payloads that can be converted to portable values. Supply images with the +model's existing crop, channel, frame-capacity, and byte conventions. Depth +arrays and their named masks must match the source execution schema. + +Prepared sources reuse depth masks and the existing image store. Table sources +can provide nested Parquet payloads. Use `columns.actions` to map logical action +names to table columns; `columns.previous_actions` maps preceding recorded actions. +Exported sequence shards preserve named action records and optional real latents. + +No previous-action input is inserted by default. To feed an existing model input +from a lagged action, declare, for example: + +```yaml +policy_binding: + previous_action_inputs: {last_quantity: quantity} + initial_inputs: {last_quantity: 0.0} + actions: + quantity: {type: real, scale: 0.5} +``` + +`initial_inputs` supplies the first decision's lagged value. The existing input's +codec and normalization are retained. Temporal padding, action activity, and +missing training labels are distinct. Complete relevant action tuples are +required for likelihood training; missing predecessors are not silently masked. + +`decision_position` defaults to `-1`. Other negative positions select existing +decoder positions while preserving support and prediction geometry. A model +with several prediction positions does not automatically execute an action block. +Only the selected current decision is executed by these runners. + +Forward attention is persisted separately from the learning objective. A +bidirectional encoder can consume an entirely observed history. Each historical +decision is reconstructed at its actual availability boundary, so later +observations cannot leak into its likelihood. An input unavailable at action time +must be supplied through an explicit task adaptation. + +Return conditioning is opt-in for attached models with a `__policy_return` +input. Offline conditioning needs a known reward tail. Online conditioning needs +`initial_desired_return`; subsequent budgets follow the configured return update. + +## Mixed training plans + +Supervised and policy phases use the same `training_plan.phases` structure. +See the full [supervised-to-policy template](../src/sequifier/templates/train-supervised-to-policy.yaml). +Supervised prepared parts and policy sequence parts both belong to +`dataset_training`; each phase references the appropriate parts. No additional +run discriminator is needed. `global_training` owns execution and model/data +contracts. Each phase owns batching and `optimization.`: + +```yaml +training_plan: + phases: + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: [{source: observations.train}] + batch_size: 64 + accumulation_steps: 1 + optimization: + model: {optimizer: {name: Adam}, learning_rate: 0.001} + - name: imitation + algorithm: policy_imitation + epochs: 5 + sources: [{source: demonstrations.recorded}] + batch_size: 64 + accumulation_steps: 1 + optimization: + actor: {optimizer: {name: Adam}, learning_rate: 0.0003} +``` + +The coordinator preflights model, source and environment contracts and commits +incoming artifacts before native phase execution. Phase-local checkpoints recover +active work; atomic receipts recover completed handoffs. Policy `model_interface` +selects an interface when source references are insufficient to identify it. +All interfaces remain in the model artifact; a shared backbone update can change +other interfaces' predictions. + +`optimizer_state` defaults to `reset`. Mixed-plan continuation is rejected because +artifact handoffs do not preserve optimization state. Native supervised-only and +compatible policy-only plans support explicit continuation under the +[training transition rules](configs/train.md#optimization-across-phases). +`handoff: last | best` selects phase weights; continuation requires the predecessor's +`last` handoff. Mixed plans execute locally. Exact online recovery requires the +adapter's snapshot capabilities. + +The Python API exposes `SequifierConfig`, `ResolvedSequifierConfig`, and +`run_learning_plan` from `sequifier.training_api`. The common entry point +`sequifier.train.run_training` dispatches a resolved plan to its appropriate +runner. `load_policy(..., binding=...)` attaches a task directly to a supervised +artifact; policy bindings never author a second model architecture. + +## Distribution-aware supervised losses and inference + +Existing MSE, MAE, cross-entropy, and hashed-component losses retain their +semantics. An interface may declare `prediction_distributions` for real targets +and select `GaussianNLLLoss` in `dataset_training.criterion`. This uses the same +real distribution implementation as policy likelihoods and stores learned scales +in the model artifact. + +Artifacts retain ordinary point-prediction execution. For an artifact containing +policy metadata, set `execution_mode: prediction` to select ordinary forecast +inference, or `execution_mode: policy` for action sampling. Joint policy +inference writes every executed field; `output_probabilities: true` also writes +the joint action log probability. Deterministic generation uses sequential +locations/argmax choices; it does not claim a global mode of an autoregressive +joint distribution. + +New model artifacts use format version 3; versions 1 and 2 remain readable. +Joint trajectories and PPO collection records use version 2. Existing +single-categorical configuration and inference APIs remain available as +compatibility surfaces. ONNX remains a prediction export, not a general training +checkpoint or an automatically exported stochastic sampler. + +This implementation has not been exercised with tests or training experiments +yet. Probability, architecture-parity, learning, and interruption acceptance +checks remain to be performed before release. + + +# Local numeric forecast rollouts + +For trajectory-backed categorical policy training and return-conditioned +execution through the online environment protocol, see [policy learning](policy-learning.md). + +The first rollout slice collects one seed-initialized episode or episode fragment +on CPU. It supports real scalar features and real static conditioning, with +one-step `causal` or `final_value` forecast models (`target_offset: 1`). Model +predictions and a persistent local adapter jointly produce the next input row. +There is no learning algorithm in this path. + +## Inference configuration + +Add this to an otherwise valid inference configuration for a model with inputs +`x`, `d` and target `x`: + +```yaml +device: cpu +model_type: generative +rollout: + max_steps: 3 + seed_index: 0 + trajectory_path: outputs/trajectory.json + environment: + factory: your_application.adapters:make_environment + config: {} # keyword arguments accepted by your factory + bindings: + - {input: x, source: prediction, key: x} + - {input: d, source: environment, key: d} +``` + +Replace the adapter factory with your application's importable `module:factory`. +It must implement the protocol below and provide the bound environment fields. + +Run the existing `sequifier infer --config-path configs/infer.yaml` command. +PT and existing schema-bearing ONNX models use the same coordinator and their +existing inference backends. This feature adds no export format. + +`seed_index` is required and selects one prepared model window, using zero-based +dataset/chunk order. Unlike legacy autoregression, selection happens before the +first-subsequence-per-sequence filter. It uses the usual inference window +selection rules, including `window_stride`. An out-of-range index fails. The +seed must have at least one valid position, end in a valid position, and contain +only left padding before its valid observations. Every feature and the attention +mask must have the same context length. Static values come from the existing +processed static feature table and stay constant. + +Each changing input must have exactly one binding. `prediction` keys name model +targets; `environment` keys name adapter outputs. Static inputs cannot have +bindings. Missing producers, duplicate input bindings, unknown predictions, and +extra or missing adapter fields fail clearly. Outputs that are only consumed by +the adapter do not need a feedback binding. + +The seed dataset uses the existing **preprocessed training representation**. +The adapter and stored trajectory use **original units**. The inference wrapper +decodes the seed and forecasts and encodes every successor using the fixed +training `mean`, `std`, and `normalize_real_columns` metadata. It uses the same +`std + 1e-9` transform as preprocessing, without fitting statistics on rollout +values. Float overflow, non-finite values, arrays in place of scalars, and +incompatible model inputs are rejected. Model padding stays zero with a false +attention mask. Features and attention masks slide together, appending a valid +successor position even for the last executed prediction. + +With `rollout` present, inference writes the trajectory to `trajectory_path` +(relative to `project_root` unless absolute). It replaces that file atomically +after successful collection and validation. This path collects one selected +seed with one model; it does not produce legacy flat prediction/probability +files. Omitting `rollout` preserves existing inference output and configuration +rules. + +## Adapter protocol + +Adapters declare `protocol_version = 1` and implement: + +```python +from sequifier.rollout_api import ResetRequest, StepRequest, StepResult + +def reset(self, request: ResetRequest) -> None: ... +def step(self, request: StepRequest) -> StepResult: ... +def close(self) -> None: ... +``` + +A `module:factory` receives `adapter.config` as keyword arguments and is called +once per collection. The adapter stays alive for all steps and closes on success +or failure, including reset, prediction, and result-validation failures. A +factory that fails during construction owns cleanup of any partial resources. +Python callers can pass a supplied adapter instance to `collect_rollout`. + +`reset` receives the episode ID, complete seed history, validity mask, static +inputs, and random seed. It initializes the environment from this history and +must not execute a transition or replace the seed. `step` receives the current +observation, all named forecasts, static inputs, and zero-based step ID. It +returns external fields, one finite scalar reward, and separate Python boolean +`terminated` and `truncated` flags. Reset and step requests contain independent +copies of coordinator data. Adapters must return the final pre-reset successor +and must not autoreset. + +Every prediction executes exactly once before collection can end. A boundary +stops further prediction and stepping. `generation_steps` is a collection +budget: exhausting it leaves `collection_cutoff: true` when the episode is +unfinished, without fabricating termination or truncation. Optional +`rollout.max_episode_steps` is an episode time limit; reaching it records +`truncated: true`. An adapter can also report its own truncation. Both boundary +flags may be true together; termination takes precedence for bootstrap. + +## Trajectory format and Python API + +The version-1 JSON artifact contains the domain seed history and mask, static +values, input bindings, prediction names, episode ID, and ordered transitions. +Each transition records the observation, predictions (the executed forecast +actions), external fields, reward, successor, step ID, and separate boundary +flags. `bootstrap_allowed` is false on termination and true otherwise, including +time-limit truncation with its recorded final successor. The artifact also +contains `total_return`, `collection_cutoff`, and inference provenance, including +the exact preprocessing statistics and seed origin. + +Seed history plus successors reconstruct every later history and attention mask; +they are not duplicated in each record. Artifacts record one episode fragment +and do not promise simulator snapshots or exact resumability. + +```python +from sequifier.rollout_api import load_trajectory, validate_trajectory + +trajectory = load_trajectory("outputs/trajectory.json") +validated = validate_trajectory(trajectory) +``` + +Saving and loading validate format/version, finite scalar data, seed shape and +mask, producer coverage, successor bindings, observation continuity, contiguous +episode/step IDs, total return, bootstrap semantics, and absence of transitions +after an episode boundary. The same validation is available for in-memory data. + +For direct Python collection, a predictor is a callable receiving a copied +`SeedHistory` in original units and returning a dictionary of scalar forecasts. +The collector owns adapter closure. `history_validator`, if provided, validates +the initial history and every successor (including a terminal successor) for +the predictor's representation; the inference integration uses the fixed codec +and existing `ExecutionSchema` here. + +```python +from sequifier.rollout_api import collect_rollout, save_trajectory + +# predictor, adapter, seed and config are provided by your application. +trajectory = collect_rollout( + predictor, adapter, seed, config, prediction_columns=["x"], max_steps=1000, +) +save_trajectory(trajectory, config.trajectory_path) +``` + +Multimodal and categorical observations, reset-only initialization, vectorized +environments, remote transport, learning algorithms, and new export support are +outside this slice. + +## Policy environments and reproducibility + +Policy training, online evaluation and policy rollout use `environment` with +`factory`, `config`, `implementation_id`, and `units` (empty means unitless). +This is a version-3 online environment, distinct from the forecast version-1 +protocol above. Policy rollout requires seeded observation reset; training +requires ordinary reset. Exact resume requires snapshot/restore capabilities. +GRPO also requires reproducible group starts. See the +[policy protocol reference](policy-learning.md#online-adapters). + +The inference route selects the rollout schema. Policy rejects forecast +`bindings`; forecast rejects policy implementation/units declarations. Neither +schema accepts a rollout discriminator. Both use `max_steps`, `trajectory_path`, +`seed_index`, and optional `max_episode_steps`. Forecast `seed_index` indexes +prepared windows; policy `seed_index` indexes raw input rows before sequence +history preparation. Both indices are zero-based and out-of-range values fail +before environment creation. + +Use `enforce_determinism` for execution and policy `action_selection` for mode +versus sampling. A fixed seed can make stochastic actions reproducible under +the same supported runtime and ordering. diff --git a/src/sequifier/helpers.py b/src/sequifier/helpers.py index 2dd5ca64..0cd3f19e 100644 --- a/src/sequifier/helpers.py +++ b/src/sequifier/helpers.py @@ -952,9 +952,8 @@ def configure_determinism(seed: int, strict: bool = False) -> None: torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True + torch.use_deterministic_algorithms(strict) if strict: - torch.use_deterministic_algorithms(True, warn_only=True) - os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index 2c6827da..75362b87 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -168,9 +168,9 @@ def infer(args: Any, args_config: dict[str, Any]) -> None: skip_metadata = args_config.get("skip_metadata", False) config = load_inferer_config(config_path, args_config, skip_metadata) - from sequifier.config.policy_infer_config import PolicyInferenceConfig + from sequifier.config.policy_infer_config import ResolvedPolicyInferenceConfig - if isinstance(config, PolicyInferenceConfig): + if isinstance(config, ResolvedPolicyInferenceConfig): from sequifier.rl.inference import infer_policy infer_policy(config) @@ -191,7 +191,7 @@ def infer(args: Any, args_config: dict[str, Any]) -> None: selected_columns_statistics = {} normalize_real_columns = True - configure_determinism(config.seed, config.deterministic) + configure_determinism(config.seed, config.enforce_determinism) # Seed once for this CLI run, before creating any ORT sessions. onnxruntime.set_seed(config.seed) @@ -1272,14 +1272,14 @@ def predict(history): # Instantiate only after validating the seed; collect_rollout owns close(), # including reset, model, adapter, successor validation, and collection failures. - adapter = load_adapter(rollout.adapter) + adapter = load_adapter(rollout.environment) trajectory = collect_rollout( predict, adapter, seed, rollout, config.target_columns, - config.generation_steps, + rollout.max_steps, random_seed=config.seed, history_validator=validate_history, metadata={ @@ -1291,8 +1291,9 @@ def predict(history): "context_length": config.window_view.context_length, "preprocessing": codec.to_dict(), "rollout": rollout.model_dump(mode="json"), - "collection_budget": config.generation_steps, + "collection_budget": rollout.max_steps, "seed": config.seed, + "enforce_determinism": config.enforce_determinism, "seed_origin": { "data_id": data_id, "seed_index": rollout.seed_index, diff --git a/src/sequifier/io/actions.py b/src/sequifier/io/actions.py index 3617ab81..2601001b 100644 --- a/src/sequifier/io/actions.py +++ b/src/sequifier/io/actions.py @@ -22,14 +22,6 @@ def action_component_key(name, config): def encode_action(value, config): - if not config.is_joint: - value = domain_scalar(value, "action") - try: - return config.action_values.index(value) - except ValueError as exc: - raise ValueError( - f"action {value!r} is outside the fixed vocabulary" - ) from exc if not isinstance(value, Mapping) or { key for key in value if key in config.actions } != set(config.actions): @@ -72,8 +64,6 @@ def action_batch(values, config, *, device=None, encoded=True): values = list(values) if not encoded: values = [encode_action(value, config) for value in values] - if not config.is_joint: - return torch.tensor(values, dtype=torch.long, device=device) result = { name: torch.tensor( [value[name] for value in values], @@ -119,8 +109,6 @@ def action_record(actions, index=0): def decode_action(action, config): - if not config.is_joint: - return {config.action_column: config.action_values[action]} return { name: field.values[int(action[name])] if field.type == "categorical" diff --git a/src/sequifier/io/episode_corpus.py b/src/sequifier/io/episode_corpus.py index a392c8d8..05930bae 100644 --- a/src/sequifier/io/episode_corpus.py +++ b/src/sequifier/io/episode_corpus.py @@ -10,7 +10,6 @@ from sequifier.fingerprints import fingerprint from sequifier.io.policy_window import action_index from sequifier.io.sequence import SequenceEpisode -from sequifier.io.values import numeric_scalar from sequifier.trajectory.importers import source_episodes from sequifier.trajectory.sources import ResolvedSequenceSource @@ -48,9 +47,6 @@ def __init__(self, sources, config, cache_size=8): self.cache_size = cache_size self.cache = OrderedDict() self.metadata = [] - categorical = ( - set(config.categorical_columns) | config.categorical_n_classes.keys() - ) seen = set() for index in range(len(self.sources)): episode = self[index] @@ -68,12 +64,9 @@ def __init__(self, sources, config, cache_size=8): if episode.previous_action is not None: action_index(episode.previous_action, config) for column, values in episode.observations.items(): - if not config.is_joint and column not in categorical: - for value in values: - numeric_scalar(value, column) + pass for column, value in episode.static_inputs.items(): - if not config.is_joint and column not in categorical: - numeric_scalar(value, column) + pass self.metadata.append( EpisodeMetadata( episode.episode_id, diff --git a/src/sequifier/io/episode_features.py b/src/sequifier/io/episode_features.py index 6b9d5861..f3dae585 100644 --- a/src/sequifier/io/episode_features.py +++ b/src/sequifier/io/episode_features.py @@ -1,15 +1,13 @@ """Fit and validate frozen policy features over an episode view.""" import polars as pl -import torch -from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.io.policy_window import encode_feature from sequifier.io.preprocessing import ( _finite_mean_and_std, create_id_map, get_combined_statistics, - normalize_values, ) from sequifier.trajectory.returns import return_to_go @@ -42,7 +40,7 @@ def values(self, episode, length, column): else [episode.static_inputs[column]] ) - def fit_categorical_config(self) -> CategoricalPolicyConfig: + def fit_categorical_config(self) -> ResolvedPolicyBinding: """Fit only declared raw categorical features, using the training episodes.""" values = self.config.model_dump() for column in self.config.categorical_columns: @@ -75,9 +73,9 @@ def fit_categorical_config(self) -> CategoricalPolicyConfig: else "static_n_classes" ) values[key][column] = max(mapping.values()) + 1 - return CategoricalPolicyConfig.model_validate(values) + return ResolvedPolicyBinding.model_validate(values) - def use_config(self, config: CategoricalPolicyConfig) -> None: + def use_config(self, config: ResolvedPolicyBinding) -> None: if self._validated_config == config: return self.config = config @@ -96,40 +94,14 @@ def validate_normalization(self, statistics): """Check FP32 transforms episode-by-episode without constructing windows.""" if self._validated_statistics == statistics: return - if self.config.is_joint: - from sequifier.io.policy_window import PolicyWindowView - - view = PolicyWindowView(self.config) - for episode in self.episodes: - for step in range(len(episode.actions)): - view.episode_inputs(episode, step, None) - self._validated_statistics = {k: dict(v) for k, v in statistics.items()} - return - for episode, length in zip(self.episodes, self.lengths): - for column in self.config.observation_columns + self.config.static_columns: - if column in self.config.categorical_n_classes: - continue - values = self.values(episode, length, column) - tensor = torch.tensor(values, dtype=torch.float32) - transformed = normalize_values(tensor, statistics[column]) - if ( - not torch.isfinite(tensor).all() - or not torch.isfinite(transformed).all() - ): - raise ValueError( - f"episode {episode.episode_id}, {column}: non-finite FP32 input/normalization" - ) - if self.config.return_conditioned: - returns = return_labels(episode, length, self.config.returns.gamma) - if not torch.isfinite( - torch.tensor(returns, dtype=torch.float32) - / self.config.returns.scale - ).all(): - raise ValueError( - f"episode {episode.episode_id}: non-finite conditioned return" - ) + from sequifier.io.policy_window import PolicyWindowView + view = PolicyWindowView(self.config) + for episode in self.episodes: + for step in range(len(episode.actions)): + view.episode_inputs(episode, step, None) self._validated_statistics = {k: dict(v) for k, v in statistics.items()} + return def normalization_statistics(self): """Use the same stable sample moments as supervised preprocessing.""" diff --git a/src/sequifier/io/folder_dataset.py b/src/sequifier/io/folder_dataset.py index c37f917d..a5bdd0ac 100644 --- a/src/sequifier/io/folder_dataset.py +++ b/src/sequifier/io/folder_dataset.py @@ -60,7 +60,7 @@ def __init__(self, data_path: str, config: Any, shuffle: bool, format_hint: str) if config.image_sources or config.preconditioning_image is not None else None ) - self.batch_size = global_training(config).batch_size + self.batch_size = config.batch_size self.shuffle = shuffle self._epoch_state = shared_int(0) self._start_batch_state = shared_int(0) diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py index 082ca393..6b65bd94 100644 --- a/src/sequifier/io/iql_dataset.py +++ b/src/sequifier/io/iql_dataset.py @@ -24,8 +24,6 @@ class IQLTransitionDataset: """Index eligible decisions while retaining the source's bounded episode cache.""" def __init__(self, decisions): - if decisions.config.return_conditioned and not decisions.config.is_joint: - raise ValueError("IQL requires an unconditioned actor") self.decisions = decisions self.view = PolicyWindowView(decisions.config) self.cumulative_lengths = decisions.cumulative_lengths diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index d2bd2136..be2bcea0 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -7,17 +7,12 @@ import torch from torch.utils.data import Dataset, Sampler -from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.io.batch import SequifierBatch from sequifier.io.episode_corpus import EpisodeCorpus, EpisodeView, episode_content_key from sequifier.io.episode_features import EpisodeFeatures, return_labels from sequifier.io.partitioning import split_components -from sequifier.io.policy_window import ( - PolicyWindowView, - action_global_ids, - action_index, - action_target, -) +from sequifier.io.policy_window import PolicyWindowView, action_index class EpisodeDecisions(EpisodeFeatures): @@ -26,7 +21,7 @@ class EpisodeDecisions(EpisodeFeatures): def __init__( self, sources: Sequence | EpisodeView, - config: CategoricalPolicyConfig, + config: ResolvedPolicyBinding, *, episode_cache_size: int = 8, ): @@ -204,27 +199,14 @@ def imitation_example(dataset, index) -> SequifierBatch: inputs = PolicyWindowView(dataset.config).episode_inputs( episode, step, returns, action_ids ) - if dataset.config.is_joint: - from sequifier.io.actions import action_batch - - actions = action_batch([action_ids[step]], dataset.config) - metadata = dict(inputs.metadata) - metadata["target_valid_mask"] = torch.zeros_like(inputs.attention_valid_mask) - metadata["target_valid_mask"][0, -1] = True - return SequifierBatch( - inputs.inputs, - {name: value[:, None] for name, value in actions.items()}, - metadata, - ) - targets = torch.zeros((1, dataset.config.context_length), dtype=torch.int64) - targets[0, -1] = action_global_ids(dataset.config)[action_ids[step]] + from sequifier.io.actions import action_batch + + actions = action_batch([action_ids[step]], dataset.config) metadata = dict(inputs.metadata) metadata["target_valid_mask"] = torch.zeros_like(inputs.attention_valid_mask) metadata["target_valid_mask"][0, -1] = True return SequifierBatch( inputs.inputs, - {action_target(dataset.config): targets}, + {name: value[:, None] for name, value in actions.items()}, metadata, - torch.tensor([episode_index], dtype=torch.int64), - torch.tensor([episode.item_positions[step]], dtype=torch.int64), ) diff --git a/src/sequifier/io/policy_materialization.py b/src/sequifier/io/policy_materialization.py index c033b128..cd22a62e 100644 --- a/src/sequifier/io/policy_materialization.py +++ b/src/sequifier/io/policy_materialization.py @@ -26,7 +26,9 @@ def encode_value(value, column, config, *, force_raw=False): if value in special and special[value] < config.categorical_n_classes[column]: return special[value] unknown = SPECIAL_TOKEN_IDS.unknown - if unknown in mapping.values(): + if unknown in mapping.values() or unknown < config.categorical_n_classes.get( + column, 0 + ): return unknown raise ValueError(f"{column}: value is outside the persisted categorical codec") return value diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py index b21c5f10..d6e04728 100644 --- a/src/sequifier/io/policy_window.py +++ b/src/sequifier/io/policy_window.py @@ -1,16 +1,11 @@ """Policy alignment over the shared named-tensor and model-window contracts.""" from collections.abc import Mapping, Sequence -from typing import cast -import torch - -from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view from sequifier.io.batch import SequifierInputs from sequifier.io.preprocessing import normalize_values -from sequifier.io.values import domain_fields, domain_scalar, numeric_scalar -from sequifier.special_tokens import SPECIAL_TOKEN_IDS PREVIOUS_ACTION = "__policy_previous_action" DESIRED_RETURN = "__policy_return" @@ -20,80 +15,33 @@ def normalize_policy_features(features, metadata, interface): """Apply persisted policy transforms identically in training and prediction.""" features = dict(features) mask = metadata["attention_valid_mask"] + if interface.policy.return_conditioned and interface.policy.input_encoding == "raw": + features[DESIRED_RETURN] = ( + features[DESIRED_RETURN] / interface.policy.returns.scale + ) for column, statistics in interface.selected_columns_statistics.items(): - if column not in features or ( - interface.policy.is_joint and not interface.normalize_real_columns - ): + if column not in features or (not interface.normalize_real_columns): continue - if interface.policy.is_joint and interface.policy.input_encoding == "model": + if interface.policy.input_encoding == "model": continue value = normalize_values(features[column], statistics) if value.ndim >= 2 and value.shape[:2] == mask.shape: valid = mask.reshape(*mask.shape, *([1] * (value.ndim - 2))) value = value.masked_fill(~valid, 0.0) features[column] = value - if interface.policy.return_conditioned and not interface.policy.is_joint: - features[DESIRED_RETURN] = ( - features[DESIRED_RETURN] / interface.policy.returns.scale - ) return features -def action_index( - value, config: CategoricalPolicyConfig -) -> int | dict[str, int | float]: - if config.is_joint: - from sequifier.io.actions import encode_action - - return encode_action(value, config) - value = domain_scalar(value, "recorded action") - try: - return config.action_values.index(value) - except ValueError as exc: - raise ValueError( - f"action {value!r} is outside action_values {config.action_values!r}" - ) from exc - - -def action_global_ids(config: CategoricalPolicyConfig) -> list[int]: - offset = SPECIAL_TOKEN_IDS.user_start - return list(range(offset, offset + len(config.action_values))) - +def action_index(value, config: ResolvedPolicyBinding) -> int | dict[str, int | float]: + from sequifier.io.actions import encode_action -def action_target(config: CategoricalPolicyConfig) -> str: - names = config.observation_columns + config.static_columns - return "__policy_action" if config.action_column in names else config.action_column + return encode_action(value, config) -def encode_feature(value, column: str, config: CategoricalPolicyConfig): - if config.is_joint: - from sequifier.io.policy_materialization import encode_value +def encode_feature(value, column: str, config: ResolvedPolicyBinding): + from sequifier.io.policy_materialization import encode_value - return encode_value(value, column, config) - mapping = config.id_maps.get(column) - if mapping is not None: - try: - # JSON object keys are strings; preprocessing accepts both forms. - value = mapping[value] if value in mapping else mapping[str(value)] - except (KeyError, TypeError) as exc: - unknown = SPECIAL_TOKEN_IDS.labels_by_id[SPECIAL_TOKEN_IDS.unknown] - if unknown not in mapping: - if ( - config.categorical_encoding == "global" - and SPECIAL_TOKEN_IDS.unknown not in mapping.values() - ): - return SPECIAL_TOKEN_IDS.unknown - raise ValueError( - f"{column}: value {value!r} is outside the fixed vocabulary" - ) from exc - value = mapping[unknown] - count = config.categorical_n_classes.get(column) - if count is not None: - domain_scalar(value, column) - if isinstance(value, str) or value != int(value) or not 0 <= value < count: - raise ValueError(f"{column}: categorical ID must be in [0, {count})") - return int(value) - return numeric_scalar(value, column) + return encode_value(value, column, config) class PolicyWindowView: @@ -103,7 +51,7 @@ class PolicyWindowView: to the persisted input vocabulary here; the start token is input-only. """ - def __init__(self, config: CategoricalPolicyConfig): + def __init__(self, config: ResolvedPolicyBinding): self.config = config self.view = ModelWindowView(config.context_length, "policy", 0) self.resolved = resolve_window_view( @@ -123,7 +71,7 @@ def episode_inputs( returns = return_labels(episode, length, self.config.returns.gamma) start = max(0, step + 1 - self.config.context_length) initial = ( - (None if self.config.is_joint else len(self.config.action_values)) + (None) if episode.starts_episode else action_index(episode.previous_action, self.config) ) @@ -150,7 +98,7 @@ def successor_inputs(self, episode, step: int, successor) -> SequifierInputs: ] + [successor] previous = [ ( - (None if self.config.is_joint else len(self.config.action_values)) + (None) if episode.starts_episode else action_index(episode.previous_action, self.config) ) @@ -183,90 +131,17 @@ def inputs( static_inputs: Mapping, ) -> SequifierInputs: config = self.config - if config.is_joint: - from sequifier.io.policy_materialization import materialize_history + from sequifier.io.policy_materialization import materialize_history - if config.return_conditioned: - if desired_returns is None or len(desired_returns) != len(observations): - raise ValueError( - "return-conditioned histories require one return per observation" - ) - observations = [ - {**row, DESIRED_RETURN: value} - for row, value in zip(observations, desired_returns) - ] - return materialize_history( - config, observations, previous_actions, static_inputs - ) - previous_actions = cast(Sequence[int], previous_actions) - length = len(observations) - if not length or len(previous_actions) != length: - raise ValueError( - "policy observations and previous actions must have matching nonempty lengths" - ) if config.return_conditioned: - if desired_returns is None or len(desired_returns) != length: + if desired_returns is None or len(desired_returns) != len(observations): raise ValueError( - "return-conditioned windows require one return per decision" + "return-conditioned histories require one return per observation" ) - elif desired_returns is not None: - raise ValueError("behavior cloning must not consume return inputs") - start = max(0, length - config.context_length) - padding = config.context_length - (length - start) - rows = [ - domain_fields(row, set(config.observation_columns), "policy observation") - for row in observations[start:] - ] - static = domain_fields( - static_inputs, set(config.static_columns), "policy static inputs" - ) - features = {} - for column in config.observation_columns + config.static_columns: - values = ( - [row[column] for row in rows] - if column in config.observation_columns - else [static[column]] * len(rows) - ) - features[column] = torch.tensor( - [ - [0] * padding - + [encode_feature(value, column, config) for value in values] - ], - dtype=torch.int64 - if column in config.categorical_n_classes - else torch.float32, - ) - ids = action_global_ids(config) - bos = ids[-1] + 1 - preceding = [] - for value in previous_actions[start:]: - if ( - isinstance(value, bool) - or not isinstance(value, int) - or not 0 <= value <= len(ids) - ): - raise ValueError( - "previous actions require decoder IDs or the explicit start token" - ) - preceding.append(bos if value == len(ids) else ids[value]) - features[PREVIOUS_ACTION] = torch.tensor( - [[bos] * padding + preceding], dtype=torch.int64 + observations = [ + {**row, DESIRED_RETURN: value} + for row, value in zip(observations, desired_returns) + ] + return materialize_history( + config, observations, previous_actions, static_inputs ) - if desired_returns is not None: - features[DESIRED_RETURN] = torch.tensor( - [ - [0.0] * padding - + [ - numeric_scalar(value, "desired_return") - for value in desired_returns[start:] - ] - ], - dtype=torch.float32, - ) - if any( - value.is_floating_point() and not torch.isfinite(value).all() - for value in features.values() - ): - raise ValueError("policy inputs exceed the finite float32 range") - metadata = self.resolved.build_masks(torch.tensor([padding], dtype=torch.int64)) - return SequifierInputs(features, metadata) diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index cf29aff4..5a8cb6cc 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -186,12 +186,16 @@ def use_config(self, config): def prepare_preferences(run, phase, config, catalog, cache): - key = ("preferences", phase.preferences, phase.validation_preferences) + key = ( + "preferences", + phase.preferences.source, + phase.validation_preferences.source if phase.validation_preferences else None, + ) if key in cache: return cache[key] - pairs = load_preferences(run.preferences[phase.preferences]).pairs + pairs = load_preferences(run.preferences[phase.preferences.source]).pairs validation = ( - load_preferences(run.preferences[phase.validation_preferences]).pairs + load_preferences(run.preferences[phase.validation_preferences.source]).pairs if phase.validation_preferences else None ) diff --git a/src/sequifier/io/prepared_policy_data.py b/src/sequifier/io/prepared_policy_data.py index 2e58abb1..c7609bce 100644 --- a/src/sequifier/io/prepared_policy_data.py +++ b/src/sequifier/io/prepared_policy_data.py @@ -6,7 +6,7 @@ from sequifier.io.episode_features import EpisodeFeatures if TYPE_CHECKING: - from sequifier.rl.policy import CategoricalPolicy + from sequifier.rl.policy import Policy class PreparedPolicyData(Protocol): @@ -18,4 +18,4 @@ def contract(self) -> list: ... def fitting_dataset(self) -> EpisodeFeatures: ... - def validate(self, actor: "CategoricalPolicy") -> None: ... + def validate(self, actor: "Policy") -> None: ... diff --git a/src/sequifier/io/sequifier_dataset_from_file.py b/src/sequifier/io/sequifier_dataset_from_file.py index 3c66a2fd..80a04781 100644 --- a/src/sequifier/io/sequifier_dataset_from_file.py +++ b/src/sequifier/io/sequifier_dataset_from_file.py @@ -45,7 +45,7 @@ class SequifierDatasetFromFile(IterableDataset): def __init__(self, data_path: str, config: Any, shuffle: bool = True): super().__init__() self.config = config - self.batch_size = global_training(config).batch_size + self.batch_size = config.batch_size self.shuffle = shuffle self._epoch_state = shared_int(0) self._start_batch_state = shared_int(0) diff --git a/src/sequifier/make.py b/src/sequifier/make.py index cfe43b36..cebb3da3 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -39,12 +39,6 @@ training_objective: causal context_length: 48 inference_batch_size: 10 - batch_size: 10 - learning_rate: 0.0001 - optimizer: {name: AdamW} - scheduler: {name: StepLR, step_size: 1, gamma: 0.99} - scheduler_step_on: epoch - reset_optimization_on_phase: true model: backbone: @@ -64,12 +58,26 @@ part: {metadata_config_path: PLEASE FILL} criterion: {EXAMPLE_TARGET_COLUMN_NAME: MSELoss} -training_plan: {epochs: 10} +training_plan: + phases: + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: [{source: default.default}] + batch_size: 10 + accumulation_steps: 1 + optimization: + model: + learning_rate: 0.0001 + optimizer: {name: AdamW} + scheduler: {name: StepLR, arguments: {step_size: 1, gamma: 0.99}} + scheduler_step_on: epoch evaluation: true -export_generative_model: PLEASE FILL # true or false -export_embedding_model: PLEASE FILL # true or false +export_generative_model: true +export_embedding_model: false """ infer_config_string = """project_root: . diff --git a/src/sequifier/rl/attachment.py b/src/sequifier/rl/attachment.py index 94614b02..7f7936dd 100644 --- a/src/sequifier/rl/attachment.py +++ b/src/sequifier/rl/attachment.py @@ -1,6 +1,6 @@ """Attach a policy task to a resolved model without rebuilding its interface.""" -from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.model.execution_schema import ExecutionSchema @@ -44,16 +44,14 @@ def resolve_binding(execution, interface_name, binding): for item in schema.inputs if item.key not in static and item.key != "attention_valid_mask" ] - conditioned = ( - binding.return_conditioned - if "__policy_return" in temporal - and "return_conditioned" in binding.model_fields_set - else False - ) + conditioned = binding.return_conditioned + if conditioned and "__policy_return" not in temporal: + raise ValueError( + "return conditioning requires an explicit __policy_return model input" + ) values.update( actions=actions, action_order=[name for name in interface.target_columns if name in actions], - alignment="model_inputs", return_conditioned=conditioned, observation_columns=[ name @@ -84,7 +82,7 @@ def resolve_binding(execution, interface_name, binding): for column in binding.previous_action_inputs: if column not in temporal: raise ValueError(f"{column}: previous action binding is not a model input") - return CategoricalPolicyConfig.model_validate(values) + return ResolvedPolicyBinding.model_validate(values) def apply_binding(execution, interface_name, policy): @@ -104,18 +102,15 @@ def apply_binding(execution, interface_name, policy): interface.policy = policy authored = execution.model.interfaces[interface_name] authored.prediction_distributions = distributions - authored.policy = PolicySchemaConfig.model_validate( - policy.model_dump(include=set(PolicySchemaConfig.model_fields)) - ) def attach_policy(network, execution, interface_name, binding): - from sequifier.rl.policy import CategoricalPolicy + from sequifier.rl.policy import Policy original_parameters = {name for name, _ in network.named_parameters()} policy = resolve_binding(execution, interface_name, binding) apply_binding(execution, interface_name, policy) - actor = CategoricalPolicy.from_network(network, execution, interface_name) + actor = Policy.from_network(network, execution, interface_name) actor.transfer_report = { "mode": "adapt" if binding.previous_action_inputs diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 1b5112f2..45a14e9c 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -11,14 +11,14 @@ restore_model_artifact, save_model_artifact, ) -from sequifier.rl.policy import CategoricalPolicy +from sequifier.rl.policy import Policy def phase_actor_path(directory, index, name): return Path(directory) / f"{index:03d}-{name}" / "actor.pt" -def validate_policy_weights(policy: CategoricalPolicy) -> None: +def validate_policy_weights(policy: Policy) -> None: for name, value in policy.network.state_dict().items(): if not torch.isfinite(value).all(): raise ValueError(f"policy tensor {name!r} must be finite") @@ -27,7 +27,7 @@ def validate_policy_weights(policy: CategoricalPolicy) -> None: def save_policy( - policy: CategoricalPolicy, path: str | Path, *, metadata: dict | None = None + policy: Policy, path: str | Path, *, metadata: dict | None = None ) -> None: validate_policy_weights(policy) save_model_artifact( @@ -54,7 +54,7 @@ def load_policy( device: str = "cpu", interface_name: str | None = None, binding=None, -) -> CategoricalPolicy: +) -> Policy: if isinstance(path, ModelArtifact): network, config, artifact = restore_model_artifact( path, device=device, interface_name=interface_name @@ -77,12 +77,12 @@ def load_policy( raise ValueError("Select one policy model_interface") interface_name = candidates[0] assert interface_name is not None - if binding is not None and binding.is_joint: + if binding is not None: from sequifier.rl.attachment import attach_policy policy = attach_policy(network, config, interface_name, binding) else: - policy = CategoricalPolicy.from_network(network, config, interface_name) + policy = Policy.from_network(network, config, interface_name) policy.units = dict(artifact.metadata.provenance.get("policy_units", {})) validate_policy_weights(policy) policy.eval() diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index eaa82679..86378b78 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -86,11 +86,7 @@ def decide(self, value=None, *, deterministic=False): decode_action(index, actor.config), index, float(distribution.log_prob(action)[0]), - float( - distribution.entropy(self.generator)[0] - if actor.config.is_joint - else distribution.entropy()[0] - ), + float(distribution.entropy(self.generator)[0]), float(estimate[0]), ) diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py index 9f93ade2..d68dc207 100644 --- a/src/sequifier/rl/history.py +++ b/src/sequifier/rl/history.py @@ -5,8 +5,7 @@ from copy import deepcopy from dataclasses import dataclass -from sequifier.io.policy_window import PolicyWindowView, encode_feature -from sequifier.io.values import domain_fields +from sequifier.io.policy_window import PolicyWindowView @dataclass(frozen=True) @@ -89,68 +88,27 @@ def _frame( raise ValueError( "policy frame requires an episode ID and nonnegative integer step" ) - action_count = len(self.spec.action_values) - if not self.spec.is_joint and ( - not isinstance(previous_action, int) - or isinstance(previous_action, bool) - or not ( - previous_action == action_count - if step_id == 0 - else 0 <= previous_action < action_count - ) + if set(observation) != set(self.spec.observation_columns) or set(static) != set( + self.spec.static_columns ): - raise ValueError("policy start token is allowed only on the first decision") - if self.spec.is_joint: - if set(observation) != set(self.spec.observation_columns) or set( - static - ) != set(self.spec.static_columns): - raise ValueError( - "environment payload does not match the model input binding" - ) - frame = PolicyFrame( - tuple(deepcopy(observation).items()), - deepcopy(previous_action), - tuple(deepcopy(static).items()), - episode_id, - step_id, - desired_return, + raise ValueError( + "environment payload does not match the model input binding" ) - history_inputs(self.spec, [frame]) - return frame - observation = domain_fields( - observation, set(self.spec.observation_columns), "policy observation" - ) - static = domain_fields( - static, set(self.spec.static_columns), "policy static inputs" - ) - for column, value in {**observation, **static}.items(): - mapping = self.spec.id_maps.get(column) - if ( - self.strict_vocabulary - and mapping is not None - and value not in mapping - and str(value) not in mapping - ): - raise ValueError( - f"{column}: online input is outside its frozen vocabulary" - ) - encode_feature(value, column, self.spec) frame = PolicyFrame( - tuple(observation.items()), - previous_action, - tuple(static.items()), + tuple(deepcopy(observation).items()), + deepcopy(previous_action), + tuple(deepcopy(static).items()), episode_id, step_id, desired_return, ) - # Reject values outside FP32 before making them part of live history. history_inputs(self.spec, [frame]) return frame def reset(self, observation, static_inputs, episode_id, desired_return=None): frame = self._frame( observation, - None if self.spec.is_joint else len(self.spec.action_values), + None, static_inputs, episode_id, 0, @@ -162,8 +120,6 @@ def reset(self, observation, static_inputs, episode_id, desired_return=None): def advance(self, action, successor, desired_return=None): current = self.current - if not self.spec.is_joint and not 0 <= action < len(self.spec.action_values): - raise ValueError("history action is outside the actor vocabulary") frame = self._frame( successor, action, diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 4696bf31..c35f9d71 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -8,14 +8,12 @@ import torch from loguru import logger -from sequifier.config.policy_infer_config import PolicyInferenceConfig -from sequifier.config.policy_train_config import PolicyTableColumns +from sequifier.config.policy_infer_config import ResolvedPolicyInferenceConfig from sequifier.helpers import configure_determinism, normalize_path, write_data from sequifier.io.actions import action_record, decode_action, encode_action from sequifier.io.batch import SequifierInputs, collate_model_inputs from sequifier.io.policy_window import PolicyWindowView from sequifier.io.sequence import ordered_sequence_rows -from sequifier.io.values import domain_scalar from sequifier.rl.checkpoint import load_policy from sequifier.rl.distributions import action_generator from sequifier.rl.environment import load_online_adapter @@ -23,11 +21,10 @@ from sequifier.rollout.history import numeric_scalar from sequifier.trajectory.schema import SeedHistory from sequifier.trajectory.store import save_trajectory -from sequifier.trajectory.tables import table_episode -def infer_policy(config: PolicyInferenceConfig) -> None: - configure_determinism(config.seed, config.deterministic) +def infer_policy(config: ResolvedPolicyInferenceConfig) -> None: + configure_determinism(config.seed, config.enforce_determinism) policy = load_policy( config._artifacts.get(config.model_path) or normalize_path(config.model_path, config.project_root), @@ -40,8 +37,6 @@ def infer_policy(config: PolicyInferenceConfig) -> None: if not len(data): raise ValueError("policy inference input must be nonempty") required = set(spec.observation_columns + spec.static_columns) - if config.previous_action_column is not None and config.rollout is None: - required.add(config.previous_action_column) if spec.return_conditioned and config.desired_return is None: required.add(config.desired_return_column) if not spec.return_conditioned and config.desired_return is not None: @@ -87,22 +82,17 @@ def desired_return(config, spec, row): def infer_rollout(policy, data, config): spec = policy.config - assert config.generation_steps is not None rollout = config.rollout if rollout.seed_index >= len(data): raise ValueError("policy rollout seed_index is outside the input rows") row = data.row(rollout.seed_index, named=True) seed = SeedHistory( observations={ - column: [0.0] * (spec.context_length - 1) - + [row[column] if spec.is_joint else domain_scalar(row[column], column)] + column: [0.0] * (spec.context_length - 1) + [row[column]] for column in spec.observation_columns }, attention_valid_mask=[False] * (spec.context_length - 1) + [True], - static_inputs={ - column: row[column] if spec.is_joint else domain_scalar(row[column], column) - for column in spec.static_columns - }, + static_inputs={column: row[column] for column in spec.static_columns}, ) budget = desired_return(config, spec, row) trajectory = collect_policy_rollout( @@ -110,9 +100,9 @@ def infer_rollout(policy, data, config): load_online_adapter(rollout.environment), seed, rollout, - config.generation_steps, + rollout.max_steps, desired_return=budget, - deterministic=config.deterministic, + action_selection=config.action_selection, random_seed=config.seed, ) trajectory.metadata.update( @@ -120,6 +110,8 @@ def infer_rollout(policy, data, config): "model_path": config.model_path, "model_interface": config.model_interface, "seed_index": rollout.seed_index, + "enforce_determinism": config.enforce_determinism, + "seed": config.seed, } ) destination = normalize_path(rollout.trajectory_path, config.project_root) @@ -132,67 +124,8 @@ def infer_rollout(policy, data, config): def policy_windows(data, spec, config): # Rows without a sequence ID are independent single-observation decisions. # Within sequences, recorded actions determine the causal previous-action input. - if spec.is_joint: - yield from joint_policy_windows(data, spec, config) - return - indexed = data.with_row_index("__policy_row") - groups = ( - indexed.partition_by(config.sequence_id_column, maintain_order=True) - if config.sequence_id_column in data.columns - else [indexed.slice(i, 1) for i in range(len(indexed))] - ) - view = PolicyWindowView(spec) - for group in groups: - if config.item_position_column not in group.columns: - group = group.with_row_index(config.item_position_column) - group = ordered_sequence_rows(group, config.item_position_column) - first_position = group[config.item_position_column][0] - previous_column = config.previous_action_column - if previous_column is not None: - previous = group[previous_column].to_list() - if any( - value is None for value in previous[0 if first_position > 0 else 1 :] - ): - raise ValueError( - "null previous actions are allowed only at sequence start" - ) - actions = previous[1:] - else: - if first_position > 0: - raise ValueError( - "fragment inference requires an explicit preceding action" - ) - if len(group) > 1 and ( - spec.action_column not in group.columns - or spec.action_column in spec.observation_columns + spec.static_columns - ): - raise ValueError( - "sequential inference requires recorded actions or previous_action_column" - ) - actions = group[spec.action_column].to_list()[:-1] if len(group) > 1 else [] - label = "__inference_action" - while label in group.columns: - label += "_" - episode = table_episode( - group.with_columns(pl.Series(label, actions + [spec.action_values[0]])), - spec, - PolicyTableColumns( - action=label, - previous_action=previous_column, - item_position=config.item_position_column, - reward=None, - terminated=None, - truncated=None, - ), - sequence_id=0, - ) - returns = ( - [desired_return(config, spec, row) for row in group.iter_rows(named=True)] - if spec.return_conditioned - else None - ) - for step, row_index in enumerate(group["__policy_row"]): - yield row_index, view.episode_inputs(episode, step, returns) + yield from joint_policy_windows(data, spec, config) + return @torch.no_grad() @@ -207,29 +140,27 @@ def predict_policy(policy, data, config): policy.device ) distribution = policy.distribution(inputs) - sampled = distribution.select(config.deterministic, generator) + sampled = distribution.select( + config.action_selection == "deterministic", generator + ) actions = [action_record(sampled, i) for i in range(len(rows))] probabilities = ( - distribution.distribution.probs.cpu().tolist() - if config.output_probabilities and not spec.is_joint - else None + distribution.categorical_probabilities(sampled) + if config.output_probabilities + else {} ) for index, (row_index, action) in enumerate(zip(rows, actions)): output = { "__policy_row": row_index, **decode_action(action, spec), } - if spec.is_joint and config.output_probabilities: + if config.output_probabilities: output["action_log_probability"] = float( distribution.log_prob(sampled)[index] ) - if probabilities is not None: - output.update( - { - f"{spec.action_column}__probability_{action_id}": value - for action_id, value in enumerate(probabilities[index]) - } - ) + for name, values in probabilities.items(): + for action_id, value in enumerate(values[index]): + output[f"{name}__probability_{action_id}"] = float(value) outputs.append(output) predictions = pl.DataFrame(outputs).sort("__policy_row").drop("__policy_row") identity = [ @@ -254,9 +185,11 @@ def joint_policy_windows(data, spec, config): ) view = PolicyWindowView(spec) for group in groups: + if config.item_position_column not in group.columns: + group = group.with_row_index(config.item_position_column) group = ordered_sequence_rows(group, config.item_position_column) rows = group.to_dicts() - previous, observations = [], [] + previous, observations, budgets = [], [], [] for index, row in enumerate(rows): if spec.previous_action_inputs: if index: @@ -281,12 +214,14 @@ def joint_policy_windows(data, spec, config): else: previous.append(None) observations.append({name: row[name] for name in spec.observation_columns}) + if spec.return_conditioned: + budgets.append(desired_return(config, spec, row)) yield ( row["__policy_row"], view.inputs( observations, previous, - None, + budgets if spec.return_conditioned else None, {name: row[name] for name in spec.static_columns}, ), ) diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 4281949f..518d2b4d 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -70,9 +70,7 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): value, q1, q2, targets = make_iql_models( actor.execution_config, actor.interface_name, - actor.config.actions - if actor.config.is_joint - else len(actor.config.action_values), + actor.config.actions, ) for module in (value, q1, q2, *targets): module.eval() diff --git a/src/sequifier/rl/joint_distribution.py b/src/sequifier/rl/joint_distribution.py index a582c295..891685f8 100644 --- a/src/sequifier/rl/joint_distribution.py +++ b/src/sequifier/rl/joint_distribution.py @@ -190,6 +190,24 @@ def log_prob(self, actions): score = score + torch.where(active, component, 0.0) return score.float() + def categorical_probabilities(self, actions): + """Per-field conditional probabilities at the supplied action prefix.""" + parameters = self._parameters(actions) + result = {} + for name, field in self.actor.config.actions.items(): + if field.type == "categorical": + distribution = self._field(name, parameters) + assert isinstance(distribution, CategoricalActionDistribution) + probabilities = distribution.distribution.probs + if field.active_when: + point = torch.zeros_like(probabilities) + point[:, field.values.index(field.inactive_value)] = 1 + probabilities = torch.where( + self._active(name, actions)[:, None], probabilities, point + ) + result[name] = probabilities + return result + @property def entropy_is_exact(self): return not self.actor.autoregressive_actions and all( diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 7e7458d8..8abe7995 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -3,7 +3,6 @@ from dataclasses import replace from pathlib import Path -from sequifier.config.train_config import PolicyRunConfig from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism from sequifier.io.atomic import write_json_atomic @@ -22,6 +21,7 @@ from sequifier.rl.references import PolicyReferences from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_checkpoint import RunCheckpoint +from sequifier.rl.run_config import ResolvedPolicyRun from sequifier.rl.run_state import ( ActorRepresentation, PhaseContext, @@ -45,65 +45,53 @@ def _initialize_actor(config, data): config.initialize_from, device=config.device, interface_name=config.policy_interface, - binding=config.policy if config.policy.is_joint else None, + binding=config.policy, ) validate_declared_policy(config.policy, source.config, "initialize_from.policy") - if source.config.is_joint: - from sequifier.artifacts.model_config import validate_execution_attachment - from sequifier.artifacts.model_export import model_execution_config + from sequifier.artifacts.model_config import validate_execution_attachment + from sequifier.artifacts.model_export import model_execution_config - validate_execution_attachment( - config.execution_model, model_execution_config(source.execution_config) - ) - source.units = dict(config.units) - return source - actor = build_actor( - config, source.config, source.interface.selected_columns_statistics - ) - ActorRepresentation.from_actor(actor, config.units).validate_actor( - source, config.units, "initialize_from" + validate_execution_attachment( + config.execution_model, model_execution_config(source.execution_config) ) - actor.network.load_state_dict(source.network.state_dict(), strict=True) + source.units = dict(config.units) + return source else: - if config.policy.is_joint: - from sequifier.artifacts.model_config import ( - resolved_config_from_model_config, - ) + from sequifier.artifacts.model_config import resolved_config_from_model_config - execution, _ = resolved_config_from_model_config( - config.execution_model, - device=config.device, - interface_name=config.policy_interface, - ) - return build_actor( - config, - config.policy, - execution.interface( - config.policy_interface - ).selected_columns_statistics, - ) - first = data[0] - fitting = first.fitting_dataset() if first is not None else None - fitted = ( - fitting.fit_categorical_config() if fitting is not None else config.policy + execution, _ = resolved_config_from_model_config( + config.execution_model, + device=config.device, + interface_name=config.policy_interface, + ) + return build_actor( + config, + config.policy, + execution.interface(config.policy_interface).selected_columns_statistics, ) - if set(fitted.categorical_columns) - fitted.categorical_n_classes.keys(): - raise ValueError( - "online-first categorical fields require frozen class counts and codecs" - ) - if config.normalization is not None or fitting is None: - statistics = resolve_normalization(fitted, config.normalization) - else: - fitting.use_config(fitted) - statistics = fitting.normalization_statistics() - actor = build_actor(config, fitted, statistics) - return actor -def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: +def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: """Validate the whole run, execute phase-local lifecycles, and export its final actor.""" - if not isinstance(config, PolicyRunConfig): - raise TypeError("policy training requires PolicyRunConfig") + if not isinstance(config, ResolvedPolicyRun): + from sequifier.config.train_config import ( + ResolvedSequifierConfig, + SequifierConfig, + ) + from sequifier.config.training_resolution import ( + policy_run_config, + resolve_training_run, + ) + + if isinstance(config, SequifierConfig): + config = resolve_training_run(config) + if not isinstance(config, ResolvedSequifierConfig) or any( + phase.algorithm == "supervised" for phase in config.training_plan + ): + raise TypeError( + "policy training requires a resolved policy-only training plan" + ) + config = policy_run_config(config, config.training_plan) configure_determinism(config.seed, config.enforce_determinism) plan = _plan_contract(config) inputs = RunInputs.resolve(config) @@ -139,12 +127,6 @@ def run_policy_training(config: PolicyRunConfig) -> PolicyRunResult: if recovery is not None: require_equal(data_contract, recovery.data_contract, "resume.partitions") for phase in config.phases: - if phase.representation is not None: - require_equal( - representation.values["policy"], - phase.representation.model_dump(mode="json"), - f"phases.{phase.name}.representation", - ) normalization = getattr(phase, "normalization", None) if normalization is not None: require_equal( @@ -195,6 +177,9 @@ def commit(state): ) recovery.save(checkpoint, actor.device, state) + if recovery.continuation is not None: + actor._phase_continuation = recovery.continuation + context = PhaseContext( config, phase_index, @@ -217,12 +202,14 @@ def commit(state): provenance={ **result.provenance, "config_fingerprint": fingerprint(phase.model_dump(mode="json")), + "resolved_phase": phase.model_dump(mode="json"), "representation_fingerprint": representation.identity, "incoming_weight_hash": incoming_weight_hash, "actor_path": str(actor_path), "report_path": str(actor_path.with_name("report.json")), }, ) + recovery.continuation = getattr(actor, "_phase_continuation", None) recovery.finish_phase(actor, result) recovery.save(checkpoint, actor.device) recovery.write_reports() diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index 7a6e5c64..11449ed0 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -54,13 +54,17 @@ def validate(self, actor): def prepare_offline(run, phase, config, catalog, cache): # The cache belongs to one run: budgets and optimizers do not affect partitions. - key = ("offline", tuple(phase.sources), tuple(phase.validation_sources or [])) + key = ( + "offline", + tuple(ref.source for ref in phase.sources), + tuple(ref.source for ref in (phase.validation_sources or [])), + ) if key in cache: return cache[key] corpus = catalog.dataset - training = corpus(phase.sources) + training = corpus([ref.source for ref in phase.sources]) if phase.validation_sources: - validation = corpus(phase.validation_sources) + validation = corpus([ref.source for ref in phase.validation_sources]) else: training, validation = split_by_episode( training, run.validation_fraction, run.seed, group_identical=True diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 786c069d..67744fd5 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -8,22 +8,10 @@ import torch from torch import Tensor, nn -from sequifier.artifacts.model_config import execution_config -from sequifier.config.metadata import DatasetMetadata -from sequifier.config.policy_config import CategoricalPolicyConfig, PolicySchemaConfig -from sequifier.config.train_config import ( - GlobalTrainingSpecModel, - ResolvedSequifierConfig, - _resolve_interface, -) +from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.train_config import ResolvedSequifierConfig from sequifier.io.batch import SequifierInputs -from sequifier.io.policy_window import ( - DESIRED_RETURN, - PREVIOUS_ACTION, - action_global_ids, - action_target, - normalize_policy_features, -) +from sequifier.io.policy_window import normalize_policy_features from sequifier.io.preprocessing import normalization_statistics from sequifier.model.execution_schema import ExecutionSchema from sequifier.model.prediction_distribution import PredictionScales @@ -35,7 +23,7 @@ def policy_execution_config( - config: CategoricalPolicyConfig, + config: ResolvedPolicyBinding, model, device: str = "cpu", *, @@ -45,98 +33,26 @@ def policy_execution_config( **run_options, ): """Resolve policy alignment directly into the ordinary execution contract.""" - if config.is_joint: - if execution_model is None: - raise ValueError( - "joint policies require a resolved model execution contract" - ) - from sequifier.artifacts.model_config import resolved_config_from_model_config + if execution_model is None: + raise ValueError("joint policies require a resolved model execution contract") + from sequifier.artifacts.model_config import resolved_config_from_model_config - execution, _ = resolved_config_from_model_config( - execution_model, device=device, interface_name=config.interface_name - ) - from sequifier.rl.attachment import apply_binding - - apply_binding(execution, config.interface_name, config) - for key, value in run_options.items(): - if key in type(execution).model_fields: - setattr(execution, key, value) - return execution - columns = config.observation_columns + config.static_columns + [PREVIOUS_ACTION] - if config.return_conditioned: - columns.append(DESIRED_RETURN) - target = action_target(config) - categorical = list(config.categorical_n_classes) + [PREVIOUS_ACTION, target] - real = [column for column in columns if column not in categorical] - decoder_ids = action_global_ids(config) - action_width = decoder_ids[-1] + 1 - global_training = global_training or GlobalTrainingSpecModel( - training_objective="policy", - context_length=config.context_length, - target_offset=0, - inference_batch_size=1, - batch_size=1, - learning_rate=1e-3, - torch_compile="none", - ) - interface = model.interfaces[config.interface_name].model_copy( - update={ - "input_columns": columns, - "target_columns": [target], - "categorical_decoder_special_tokens": {target: []}, - "policy": PolicySchemaConfig.model_validate( - config.model_dump(include=set(PolicySchemaConfig.model_fields)) - ), - } - ) - metadata = DatasetMetadata( - window_length=config.context_length + 1, - column_data_types={ - column: "Int64" if column in categorical else "Float32" - for column in columns + [target] - }, - n_classes={ - **config.categorical_n_classes, - PREVIOUS_ACTION: action_width + 1, - target: action_width, - }, - id_maps={ - **config.id_maps, - target: { - str(value): global_id - for value, global_id in zip(config.action_values, decoder_ids) - }, - }, - selected_columns_statistics=statistics - if statistics is not None - else { - column: {"mean": 0.0, "std": 1.0} - for column in real - if column != DESIRED_RETURN - }, - ) - resolved = _resolve_interface( - config.interface_name, - interface, - metadata, - global_training, - metadata.storage_layout, - ) - if resolved.target_decoder_ids[target] != decoder_ids: - raise ValueError("policy decoder IDs must exactly match executable actions") - return execution_config( - model.model_copy(update={"interfaces": {config.interface_name: interface}}), - {config.interface_name: resolved}, - global_training, - device=device, - **run_options, + execution, _ = resolved_config_from_model_config( + execution_model, device=device, interface_name=config.interface_name ) + from sequifier.rl.attachment import apply_binding + + apply_binding(execution, config.interface_name, config) + for key, value in run_options.items(): + if key in type(execution).model_fields: + setattr(execution, key, value) + return execution -class CategoricalPolicy(nn.Module): +class Policy(nn.Module): """A convenience view; the network owns all trainable weights and execution.""" - config: CategoricalPolicyConfig + config: ResolvedPolicyBinding execution_config: ResolvedSequifierConfig interface_name: str network: ComposableTransformerNetwork @@ -151,17 +67,12 @@ def from_network(cls, network, execution_config, interface_name: str): raise ValueError( f"Model interface {interface_name!r} has no policy alignment" ) - if not interface.policy.is_joint and interface.target_decoder_ids[ - interface.target_columns[0] - ] != action_global_ids(interface.policy): - raise ValueError("policy decoder IDs must exactly match executable actions") policy = cls() policy.config = interface.policy policy.execution_config = execution_config policy.interface_name = interface_name policy.network = network - if policy.config.is_joint: - policy._resolve_actions() + policy._resolve_actions() return policy @property @@ -285,25 +196,12 @@ def action_decoder_ids(self, name): def set_normalization_statistics( self, statistics: Mapping[str, Mapping[str, float]] ) -> None: - if self.config.is_joint: - self.interface.selected_columns_statistics = ( - normalization_statistics(statistics) - if self.interface.normalize_real_columns - else {name: dict(values) for name, values in statistics.items()} - ) - return - expected = ( - set(self.config.observation_columns + self.config.static_columns) - - self.config.categorical_n_classes.keys() - ) - if set(statistics) != expected: - raise ValueError("normalization statistics must match real policy columns") - self.interface.selected_columns_statistics = normalization_statistics( - { - column: {key: float(values[key]) for key in ("mean", "std")} - for column, values in statistics.items() - } + self.interface.selected_columns_statistics = ( + normalization_statistics(statistics) + if self.interface.normalize_real_columns + else {name: dict(values) for name, values in statistics.items()} ) + return def model_inputs( self, inputs: SequifierInputs @@ -328,24 +226,17 @@ def prepared_logits(self, features, metadata) -> Tensor | dict[str, Tensor]: outputs = self.network( features, metadata, interface_name=self.interface_name ).logits - if self.config.is_joint: - return { - name: value.masked_fill( - ~metadata["attention_valid_mask"][:, -value.shape[1] :, None], 0.0 - ) - for name, value in outputs.items() - } - logits = outputs[self.interface.target_columns[0]] - return logits.masked_fill( - ~metadata["attention_valid_mask"][:, -logits.shape[1] :, None], 0.0 - ) + return { + name: value.masked_fill( + ~metadata["attention_valid_mask"][:, -value.shape[1] :, None], 0.0 + ) + for name, value in outputs.items() + } def distribution( self, inputs: SequifierInputs ) -> CategoricalActionDistribution | JointActionDistribution: - if self.config.is_joint: - return JointActionDistribution(self, inputs) - return CategoricalActionDistribution(self(inputs)[:, -1]) + return JointActionDistribution(self, inputs) def act(self, inputs: SequifierInputs, *, deterministic: bool, generator): distribution = self.distribution(inputs) @@ -388,15 +279,4 @@ def validate_inputs(self, inputs: SequifierInputs) -> None: for value in [*inputs.inputs.values(), *inputs.metadata.values()] ): raise ValueError("policy features and masks must be on the model device") - if self.config.is_joint: - return - ids = action_global_ids(self.config) - allowed = torch.tensor(ids + [ids[-1] + 1], device=self.device) - if not torch.isin(inputs.inputs[PREVIOUS_ACTION], allowed).all(): - raise ValueError( - "previous actions must be executable action IDs or the input-only start token" - ) - - -# Backward-compatible name for the original single-categorical API. -Policy = CategoricalPolicy + return diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py index 2cad9520..f5f82bbd 100644 --- a/src/sequifier/rl/ppo_buffer.py +++ b/src/sequifier/rl/ppo_buffer.py @@ -7,7 +7,7 @@ import torch -from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.io.actions import action_batch, decode_action from sequifier.io.atomic import write_json_atomic from sequifier.io.batch import SequifierInputs @@ -43,7 +43,7 @@ class PPORolloutBuffer: shared with the fixed old scores or targets held by this buffer. """ - spec: CategoricalPolicyConfig + spec: ResolvedPolicyBinding policy_version: int frames: tuple decisions: tuple[PPODecision, ...] @@ -136,7 +136,7 @@ def archive(self, path, *, include_scores): decisions.append(record) payload = { "format": "sequifier.ppo-collection", - "version": 2 if self.spec.is_joint else 1, + "version": 2, "policy_version": self.policy_version, "policy": self.spec.model_dump(mode="json"), "frames": [frame.state_dict() for frame in self.frames], @@ -152,10 +152,10 @@ def load_collection_record(path): "version" ) not in {1, 2}: raise ValueError("expected version-1 PPO collection record") - from sequifier.config.policy_config import CategoricalPolicyConfig + from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.rl.history import PolicyFrame - spec = CategoricalPolicyConfig.model_validate(payload["policy"]) + spec = ResolvedPolicyBinding.model_validate(payload["policy"]) frames = [PolicyFrame.from_state_dict(frame) for frame in payload["frames"]] for frame in frames: history_inputs(spec, [frame]) @@ -165,13 +165,7 @@ def load_collection_record(path): step["policy_version"] != payload["policy_version"] or (step["episode_id"], step["step_id"]) != (frame.episode_id, frame.step_id) - or (not spec.is_joint and not 0 <= step["action"] < len(spec.action_values)) - or step["domain_action"] - != ( - decode_action(step["action"], spec) - if spec.is_joint - else spec.action_values[step["action"]] - ) + or step["domain_action"] != (decode_action(step["action"], spec)) or step["bootstrap"] != (not step["terminated"]) ): raise ValueError("collection record violates the decision contract") diff --git a/src/sequifier/rl/ppo_collector.py b/src/sequifier/rl/ppo_collector.py index 7834c3d8..d9920245 100644 --- a/src/sequifier/rl/ppo_collector.py +++ b/src/sequifier/rl/ppo_collector.py @@ -96,9 +96,7 @@ def collect(self, horizon, policy_version): current_reference, successor_reference, decision.action, - decision.domain_action - if self.actor.config.is_joint - else self.actor.config.action_values[decision.action], + decision.domain_action, decision.log_probability, decision.value, decision.entropy, diff --git a/src/sequifier/rl/ppo_evaluation.py b/src/sequifier/rl/ppo_evaluation.py index 7276461e..cd4cc039 100644 --- a/src/sequifier/rl/ppo_evaluation.py +++ b/src/sequifier/rl/ppo_evaluation.py @@ -91,7 +91,8 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): for _ in range(evaluation.max_episode_steps): step = cursor.step( cursor.decide( - deterministic=evaluation.selection == "deterministic" + deterministic=evaluation.action_selection + == "deterministic" ) ) if cursor.ended: @@ -139,7 +140,7 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): "timed_out": timed_out, "episodes": episodes, "seconds": time.monotonic() - started, - "selection": evaluation.selection, + "action_selection": evaluation.action_selection, } if successes and all(success is not None for success in successes): result["success_rate"] = statistics.fmean(successes) diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 3f2a7971..8c0a8a16 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -15,7 +15,7 @@ require_exact_snapshot, require_group_start, ) -from sequifier.rl.policy import CategoricalPolicy, policy_execution_config +from sequifier.rl.policy import Policy, policy_execution_config from sequifier.rl.ppo_evaluation import require_evaluation_timer from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_state import require_equal @@ -154,7 +154,7 @@ def build_actor(config, fitted, statistics): model_name=config.model_name, seed=config.seed, ) - actor = CategoricalPolicy.from_network( + actor = Policy.from_network( build_transformer_network( execution, device=torch.device(config.device) ).network, diff --git a/src/sequifier/rl/references.py b/src/sequifier/rl/references.py index a5fac4c5..1d20a625 100644 --- a/src/sequifier/rl/references.py +++ b/src/sequifier/rl/references.py @@ -57,7 +57,7 @@ def _artifact(self, path): path, device="cpu", interface_name=self.config.policy_interface, - binding=self.config.policy if self.config.policy.is_joint else None, + binding=self.config.policy, ) self.recovery.representation.validate_actor( source, self.config.units, f"reference.{path}" diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 17f70488..7a6191bf 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -54,6 +54,7 @@ class RunCheckpoint: reference_contract: dict = field(default_factory=dict) completed: list[CompletedPhase] = field(default_factory=list) phase_state: dict | None = None + continuation: dict | None = None @property def results(self): @@ -83,13 +84,6 @@ def load(cls, path, config, plan, data_identity): ) try: saved_plan = payload["plan"] - if saved_plan != plan: - from sequifier.config.train_config import PolicyRunConfig - - upgraded = PolicyRunConfig.model_validate( - {**saved_plan, "resume": config.resume.model_dump()} - ) - saved_plan = upgraded.model_dump(mode="json", exclude={"resume"}) require_equal(plan, saved_plan, "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") if payload["format_version"] == 5: diff --git a/src/sequifier/rl/run_config.py b/src/sequifier/rl/run_config.py new file mode 100644 index 00000000..79fbf589 --- /dev/null +++ b/src/sequifier/rl/run_config.py @@ -0,0 +1,35 @@ +"""Resolved inputs for the native policy runner; never an authored config.""" + +from pydantic import BaseModel, ConfigDict + +from sequifier.config.components import ResumeConfig +from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.policy_plan import FrozenNormalizationConfig, PolicyPhase +from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.config.train_config import ModelSpecModel + + +class ResolvedPolicyRun(BaseModel): + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + project_root: str + model_name: str + device: str + seed: int + enforce_determinism: bool = False + model: ModelSpecModel + policy: ResolvedPolicyBinding + execution_model: dict + policy_interface: str + initialize_from: str | None = None + normalization: FrozenNormalizationConfig | None = None + sources: dict[str, SequenceSourceConfig] + preferences: dict[str, str] + units: dict[str, str] + phases: list[PolicyPhase] + validation_fraction: float + episode_cache_size: int + resume: ResumeConfig + model_path: str + report_path: str + checkpoint_path: str + phase_directory: str diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 59f84cf6..29faa73b 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -8,13 +8,13 @@ import torch -from sequifier.config.policy_config import CategoricalPolicyConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_plan import PhaseConfig -from sequifier.rl.policy import CategoricalPolicy +from sequifier.rl.policy import Policy from sequifier.runtime.random_state import capture_rng_state, restore_rng_state if TYPE_CHECKING: - from sequifier.config.train_config import PolicyRunConfig + from sequifier.rl.run_config import ResolvedPolicyRun PhaseType = TypeVar("PhaseType", bound=PhaseConfig) DataType = TypeVar("DataType") @@ -84,7 +84,7 @@ def values(self): @property def policy(self): - return CategoricalPolicyConfig.model_validate(self.values["policy"]) + return ResolvedPolicyBinding.model_validate(self.values["policy"]) @property def identity(self): @@ -120,7 +120,7 @@ def as_dict(self): @dataclass(frozen=True) class PolicyRunResult: - policy: CategoricalPolicy + policy: Policy phases: tuple[PolicyPhaseResult, ...] representation: ActorRepresentation model_path: str @@ -130,12 +130,12 @@ class PolicyRunResult: @dataclass(frozen=True) class PhaseContext(Generic[PhaseType, DataType]): - run: "PolicyRunConfig" + run: "ResolvedPolicyRun" index: int phase: PhaseType - actor: CategoricalPolicy + actor: Policy data: DataType - reference: CategoricalPolicy | None + reference: Policy | None reference_identity: str | None resume_state: dict | None commit: Callable[[dict], None] @@ -234,6 +234,37 @@ def __init__(self, context, *, modules=None, frozen=None): role: self.configs[role].build(module) for role, module in self.modules.items() } + continuation = getattr(actor, "_phase_continuation", None) + if ( + context.phase.optimizer_state == "continue" + and not context.resume_state + and continuation is None + ): + raise ValueError("continued phase requires the preceding optimizer state") + if ( + context.phase.optimizer_state == "continue" + and not context.resume_state + and continuation + ): + require_equal( + context.phase.algorithm, + continuation["algorithm"], + "optimization.algorithm", + ) + require_equal( + sorted(self.networks), + sorted(continuation["networks"]), + "optimization.roles", + ) + for name, module in self.networks.items(): + module.load_state_dict(continuation["networks"][name], strict=True) + for role, (optimizer, scheduler) in self.optimization.items(): + optimizer.load_state_dict( + continuation["optimization"][role]["optimizer"] + ) + scheduler.load_state_dict( + continuation["optimization"][role]["scheduler"] + ) self.steps = dict.fromkeys(self.modules, 0) self.sampler = torch.Generator().manual_seed(context.run.seed + context.index) self.state = context.resume_state or {} @@ -287,7 +318,7 @@ def step(self): role: float( torch.nn.utils.clip_grad_norm_( module.parameters(), - self.configs[role].gradient_clip, + self.configs[role].gradient_clip or float("inf"), error_if_nonfinite=True, ) ) @@ -347,6 +378,28 @@ def checkpoint(self, final, **extra): ) def finish(self, clocks, provenance): + next_index = self.context.index + 1 + preserve = ( + next_index < len(self.context.run.phases) + and self.context.run.phases[next_index].optimizer_state == "continue" + ) + if preserve: + self.context.actor._phase_continuation = { + "algorithm": self.context.phase.algorithm, + "networks": { + name: clone_weights(module) + for name, module in self.networks.items() + }, + "optimization": { + role: { + "optimizer": optimizer.state_dict(), + "scheduler": scheduler.state_dict(), + } + for role, (optimizer, scheduler) in self.optimization.items() + }, + } + elif hasattr(self.context.actor, "_phase_continuation"): + del self.context.actor._phase_continuation return self.selection.finish( self.context, self.transaction, clocks, self.metrics, provenance ) diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index eefa39ba..b8e54a59 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -3,6 +3,7 @@ import math import warnings from copy import deepcopy +from typing import Literal from uuid import uuid4 from sequifier.config.policy_infer_config import PolicyRolloutConfig @@ -17,19 +18,19 @@ validate_online_adapter, validate_reset, ) -from sequifier.rl.policy import CategoricalPolicy +from sequifier.rl.policy import Policy from sequifier.trajectory.schema import SeedHistory, Trajectory, Transition def collect_policy_rollout( - policy: CategoricalPolicy, + policy: Policy, adapter: OnlineEnvironmentAdapter, seed: SeedHistory, config: PolicyRolloutConfig, max_steps: int, *, desired_return: float | None = None, - deterministic: bool = True, + action_selection: Literal["deterministic", "stochastic"] = "deterministic", random_seed: int = 1010, ) -> Trajectory: """Execute a supplied initial observation, preserving actual episode boundaries.""" @@ -39,8 +40,8 @@ def collect_policy_rollout( require_seeded_reset(adapter) if (not isinstance(max_steps, int)) or max_steps < 1: raise ValueError("policy collection budget must be a positive integer") - if not isinstance(deterministic, bool): - raise ValueError("policy deterministic selection must be a boolean") + if action_selection not in {"deterministic", "stochastic"}: + raise ValueError("action_selection must be deterministic or stochastic") spec, units = policy.config, config.environment.units if units != getattr(policy, "units", {}): raise ValueError("rollout environment units differ from the policy") @@ -92,7 +93,7 @@ def collect_policy_rollout( transitions, previous_actions, log_probabilities = [], [], [] for step_id in range(max_steps): current = cursor.history.current - decision = cursor.decide(deterministic=deterministic) + decision = cursor.decide(deterministic=action_selection == "deterministic") result = cursor.step(decision) truncated = result.truncated or ( config.max_episode_steps is not None @@ -119,16 +120,12 @@ def collect_policy_rollout( name: action_metadata[action_latent_key(name, spec)] for name, field in spec.actions.items() if field.type == "real" - } - if spec.is_joint - else {}, + }, action_components={ name: action_metadata[action_component_key(name, spec)] for name, field in spec.actions.items() if field.type == "real" - } - if spec.is_joint - else {}, + }, external_inputs=successor or {}, reward=result.reward, successor_observation=successor, @@ -141,10 +138,8 @@ def collect_policy_rollout( break last = transitions[-1] return Trajectory( - format="sequifier.policy-trajectory" - if spec.is_joint - else "sequifier.scalar-policy-trajectory", - version=2 if spec.is_joint else 1, + format="sequifier.policy-trajectory", + version=2, episode_id=episode_id, seed=seed, bindings=[ @@ -156,14 +151,13 @@ def collect_policy_rollout( collection_cutoff=not (last.terminated or last.truncated), total_return=math.fsum(step.reward for step in transitions), metadata={ - "policy_alignment": spec.alignment, "policy_execution": { "config": spec.model_dump(mode="json"), "initial_desired_return": desired_return, "remaining_returns": returns, "previous_action_ids": previous_actions, "action_log_probabilities": log_probabilities, - "deterministic": deterministic, + "action_selection": action_selection, }, }, ) diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 7f85f6e8..19d933e7 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -5,7 +5,6 @@ from sequifier.config.policy_plan import PolicyImitationPhaseConfig from sequifier.io.batch import collate_sequence_batches from sequifier.io.policy_dataset import imitation_example -from sequifier.io.policy_window import action_global_ids, action_target from sequifier.rl.batch_order import BatchOrder from sequifier.rl.offline_data import OfflineData from sequifier.rl.run_state import PhaseContext, PhaseSession @@ -13,12 +12,7 @@ def action_nll(actor, batch): batch = batch.to(actor.device) - if actor.config.is_joint: - actions = {name: value[:, -1] for name, value in batch.targets.items()} - return -actor.evaluate_actions(batch.model_inputs, actions).float() - labels = batch.targets[action_target(actor.config)][:, -1] - ids = torch.tensor(action_global_ids(actor.config), device=actor.device) - actions = (labels[:, None] == ids[None, :]).long().argmax(dim=-1) + actions = {name: value[:, -1] for name, value in batch.targets.items()} return -actor.evaluate_actions(batch.model_inputs, actions).float() diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index 859a091c..f13006ab 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -2,10 +2,8 @@ from sequifier.config.policy_config import ( ActionFieldConfig, - CategoricalPolicyConfig, PolicyBinding, - PolicyConfig, - PolicySchemaConfig, + ResolvedPolicyBinding, ReturnDefinition, ) from sequifier.config.policy_infer_config import ( @@ -22,7 +20,6 @@ OnlineEnvironmentConfig, OnlineEvaluationConfig, PolicyImitationPhaseConfig, - PolicyResumeConfig, PPOPhaseConfig, ReferenceConfig, ) @@ -31,7 +28,7 @@ SequenceSourceConfig, ) from sequifier.config.sequence_import_config import SequenceImportConfig -from sequifier.config.train_config import PolicyRunConfig +from sequifier.config.train_config import ResolvedSequifierConfig, SequifierConfig from sequifier.io.batch import SequifierBatch, SequifierInputs, collate_sequence_batches from sequifier.io.policy_dataset import PolicySequenceDataset, split_by_episode from sequifier.io.policy_window import PolicyWindowView, action_index @@ -48,7 +45,7 @@ OnlineStepResult, ) from sequifier.rl.learner import run_policy_training -from sequifier.rl.policy import CategoricalPolicy, Policy +from sequifier.rl.policy import Policy from sequifier.rl.ppo_buffer import load_collection_record from sequifier.rl.ppo_evaluation import evaluate_policy from sequifier.rl.run_state import ( @@ -70,8 +67,8 @@ __all__ = [ "ActorRepresentation", "CategoricalActionDistribution", - "CategoricalPolicy", - "CategoricalPolicyConfig", + "Policy", + "ResolvedPolicyBinding", "DPOPhaseConfig", "EnvironmentCapabilities", "FrozenNormalizationConfig", @@ -84,14 +81,14 @@ "OnlineResetResult", "OnlineStepResult", "PolicyImitationPhaseConfig", - "PolicyInferenceConfig", "PolicyRolloutConfig", + "PolicyInferenceConfig", "OptimizationConfig", "PolicyPhaseResult", - "PolicyResumeConfig", - "PolicyRunConfig", + "SequifierConfig", + "ResolvedSequifierConfig", "PolicyRunResult", - "PolicySchemaConfig", + "ResolvedPolicyBinding", "PolicySequenceDataset", "PolicyTableColumns", "PolicyWindowView", @@ -124,4 +121,4 @@ "split_by_episode", ] -__all__ += ["ActionFieldConfig", "Policy", "PolicyBinding", "PolicyConfig"] +__all__ += ["ActionFieldConfig", "Policy", "PolicyBinding"] diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 41f7e3f5..d5149323 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -35,12 +35,13 @@ from sequifier.training.loader_state import LoaderStateService from sequifier.training.loss import LossService from sequifier.training.metrics_service import MetricsService -from sequifier.training.optimization import OptimizationRuntime +from sequifier.training.optimization import GradientPolicy, OptimizationRuntime from sequifier.training.runtime import ( DatasetFreezingPlan, DatasetRuntimeRegistry, build_dataset_runtimes, ) +from sequifier.training.scheduler_budgets import validate_supervised_scheduler_budgets from sequifier.training.state import RunState @@ -66,10 +67,15 @@ class TrainingRun: context: RunContext def start_phase(self, phase_index: int) -> None: - """Start a phase with fresh optimizer, scheduler, and scaler state.""" + """Enter the phase with its resolved reset or continuation policy.""" optimizer_step = self.optimization.optimizer_step - self.optimization = self.optimization_factory(phase_index) + if self.config.training_plan[phase_index].optimizer_state == "reset": + self.optimization = self.optimization_factory(phase_index) + self.optimization.gradient_policy = GradientPolicy( + self.config.training_plan[phase_index].accumulation_steps, + self.config.training_plan[phase_index].optimization["model"].gradient_clip, + ) self.optimization.optimizer_step = optimizer_step self.optimization_phase_index = phase_index @@ -219,36 +225,39 @@ def build( runtime_metadata=built.runtime_metadata, freezing_plan=freezing_plan, ) - parameters: Any = tuple(strategy.prepare_optimizer_parameters(network)) - optimizer_plan = isinstance(config.global_training.optimizer, OptimizerPlan) - catalog = ( - ParameterCatalog(network) - if optimizer_plan or self.semantic_optimizer_grouping - else None - ) - if self.semantic_optimizer_grouping and not optimizer_plan: - assert catalog is not None - parameters = semantic_optimizer_groups( - catalog, - parameters={id(parameter) for parameter in parameters}, - ) + for dataset in datasets.values(): + for part in dataset.parts.values(): + part.factory.batch_size = config.training_plan[ + state.phase_index + ].batch_size - reset_each_phase = config.global_training.reset_optimization_on_phase - total_epochs = sum(phase.epochs for phase in config.training_plan) + parameters: Any = tuple(strategy.prepare_optimizer_parameters(network)) + catalog = ParameterCatalog(network) def optimization_factory(phase_index: int) -> OptimizationRuntime: + phase = config.training_plan[phase_index] + role = phase.optimization["model"] + selected_parameters = parameters + if self.semantic_optimizer_grouping and not isinstance( + role.optimizer, OptimizerPlan + ): + selected_parameters = semantic_optimizer_groups( + catalog, parameters={id(parameter) for parameter in parameters} + ) return OptimizationRuntime.create( - config.global_training, + role, str(execution.device), - parameters, + selected_parameters, parameter_catalog=catalog, - phase_epochs=( - config.training_plan[phase_index].epochs - if reset_each_phase - else total_epochs - ), + accumulation_steps=phase.accumulation_steps, + layer_type_dtypes=config.global_training.layer_type_dtypes, ) + validate_supervised_scheduler_budgets(config, datasets, state.phase_index) + for phase_index in range(len(config.training_plan)): + if phase_index != state.phase_index: + optimization_factory(phase_index) + optimization = optimization_factory(state.phase_index) if loaded is not None: restorer.restore_optimization(loaded, optimization, network, strategy) diff --git a/src/sequifier/sequifier.py b/src/sequifier/sequifier.py index c75539e7..8c8da347 100644 --- a/src/sequifier/sequifier.py +++ b/src/sequifier/sequifier.py @@ -122,6 +122,15 @@ def setup_parser() -> ArgumentParser: parser_infer.add_argument("-mp", "--model-path", type=str) parser_infer.add_argument("-s", "--seed", type=int) + parser_infer.add_argument( + "--model-type", choices=("generative", "embedding", "policy") + ) + parser_infer.add_argument( + "--enforce-determinism", action="store_true", default=None + ) + parser_infer.add_argument( + "--action-selection", choices=("deterministic", "stochastic") + ) parser_infer.add_argument("--dataset", type=str) parser_infer.add_argument("--part", type=str) parser_infer.add_argument("--model-interface", type=str) diff --git a/src/sequifier/templates/infer-policy.yaml b/src/sequifier/templates/infer-policy.yaml index f3a84faf..6b1f44a5 100644 --- a/src/sequifier/templates/infer-policy.yaml +++ b/src/sequifier/templates/infer-policy.yaml @@ -1,10 +1,10 @@ project_root: . -model_path: models/policy/actor.pt +model_path: models/policy/model.pt data_path: PLEASE_FILL -model_type: generative +model_type: policy read_format: csv write_format: csv device: cpu -deterministic: true -# Omit desired_return for behavior cloning. desired_return: 10.0 +enforce_determinism: true +action_selection: deterministic diff --git a/src/sequifier/templates/train-dpo.yaml b/src/sequifier/templates/train-dpo.yaml index 013a145a..ccc03d1f 100644 --- a/src/sequifier/templates/train-dpo.yaml +++ b/src/sequifier/templates/train-dpo.yaml @@ -2,43 +2,80 @@ project_root: . model_name: dpo-control device: cpu seed: 1010 -context_length: 4 model: backbone: architecture: dim_model: 32 num_layers: 1 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} + attention: + n_heads: 4 + feed_forward: + dim: 64 positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [cue] - target_columns: [action] - policy: - observation_columns: [cue] - action_values: [left, right] - return_conditioned: false + input_columns: + - cue + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -normalization: {source: identity} -phases: -- {name: preferences, algorithm: dpo, preferences: rankings, updates: 100, beta: 0.1} -sources: + feature_embedding_dims: + cue: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 4 + normalization: + source: identity +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right +dataset_training: decisions: - paths: [data/control-decisions.csv] - columns: - episode_id: episodeId - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {cue: next_cue} - group: contextId -preferences: {rankings: data/control-preferences.json} + parts: + default: + type: sequence + paths: + - data/control-decisions.csv + columns: + episode_id: episodeId + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: + cue: next_cue + group: contextId + model_interface: actor + rankings: + parts: + default: + type: preferences + manifest: data/control-preferences.json +training_plan: + phases: + - name: preferences + algorithm: dpo + preferences: + source: rankings.default + updates: 100 + beta: 0.1 diff --git a/src/sequifier/templates/train-grpo.yaml b/src/sequifier/templates/train-grpo.yaml index 81367eb2..5b1a5017 100644 --- a/src/sequifier/templates/train-grpo.yaml +++ b/src/sequifier/templates/train-grpo.yaml @@ -2,49 +2,79 @@ project_root: . model_name: grpo-control device: cpu seed: 1010 -context_length: 4 model: backbone: architecture: dim_model: 32 num_layers: 1 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} + attention: + n_heads: 4 + feed_forward: + dim: 64 positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [cue] - target_columns: [action] - policy: - observation_columns: [cue] - action_values: [left, right] - return_conditioned: false + input_columns: + - cue + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -normalization: {source: identity} -phases: -- name: group_improvement - algorithm: grpo - total_groups: 32 - groups_per_update: 2 - group_size: 4 - max_steps: 8 - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - evaluation: + feature_embedding_dims: + cue: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 4 + normalization: + source: identity +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right +training_plan: + phases: + - name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_update: 2 + group_size: 4 + max_steps: 8 environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 - metric: score_mean + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 + seeds: + - 17 + - 29 + - 43 + - 71 + max_episode_steps: 8 + timeout_seconds: 30 + monitor: + metric: score_mean + mode: max diff --git a/src/sequifier/templates/train-iql.yaml b/src/sequifier/templates/train-iql.yaml index 9fd42196..77a146f6 100644 --- a/src/sequifier/templates/train-iql.yaml +++ b/src/sequifier/templates/train-iql.yaml @@ -8,38 +8,67 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} + attention: + n_heads: 4 + feed_forward: + dim: 128 positional_encoding_scope: global interfaces: actor: - input_columns: [OBSERVATION_COLUMN] - target_columns: [action] - policy: - observation_columns: [OBSERVATION_COLUMN] - action_values: [0.0, 1.0] - return_conditioned: false + input_columns: + - OBSERVATION_COLUMN + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -context_length: 16 -sources: - demonstrations.recorded: - paths: [PLEASE_FILL.csv] - columns: - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: {OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN} -validation_fraction: 0.2 -resume: {policy: if_exists} -phases: -- name: offline_improvement - algorithm: iql - sources: [demonstrations.recorded] - updates: 10000 - batch_size: 64 + feature_embedding_dims: + OBSERVATION_COLUMN: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 16 + validation_fraction: 0.2 + resume: + policy: if_exists +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - 0.0 + - 1.0 +dataset_training: + demonstrations: + parts: + recorded: + type: sequence + paths: + - PLEASE_FILL.csv + columns: + reward: reward + terminated: terminated + truncated: truncated + bootstrap_allowed: bootstrap_allowed + successors: + OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN + model_interface: actor +training_plan: + phases: + - name: offline_improvement + algorithm: iql + sources: + - source: demonstrations.recorded + updates: 10000 + batch_size: 64 diff --git a/src/sequifier/templates/train-policy-from-supervised.yaml b/src/sequifier/templates/train-policy-from-supervised.yaml index 25c40414..98412ff5 100644 --- a/src/sequifier/templates/train-policy-from-supervised.yaml +++ b/src/sequifier/templates/train-policy-from-supervised.yaml @@ -2,18 +2,32 @@ project_root: . model_name: posttrained device: cpu initialize_from: models/supervised/model.pt -policy_interface: main +global_training: + resume: + policy: if_exists + mode: reset_environments policy_binding: actions: - mode: {type: categorical, values: ["off", "on"]} - quantity: {type: real, scale: 0.5} -sources: + mode: + type: categorical + values: + - 'off' + - 'on' + quantity: + type: real + scale: 0.5 +dataset_training: demonstrations: - paths: [data/demonstrations.parquet] - format: table -phases: + parts: + default: + type: sequence + paths: + - data/demonstrations.parquet + format: table +training_plan: + phases: - name: imitation algorithm: policy_imitation - sources: [demonstrations] + sources: + - source: demonstrations.default epochs: 5 -resume: {policy: if_exists, mode: reset_environments} diff --git a/src/sequifier/templates/train-policy.yaml b/src/sequifier/templates/train-policy.yaml index 6f851cb8..ac7fc514 100644 --- a/src/sequifier/templates/train-policy.yaml +++ b/src/sequifier/templates/train-policy.yaml @@ -8,35 +8,68 @@ model: dim_model: 32 num_layers: 2 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} + attention: + n_heads: 4 + feed_forward: + dim: 128 + activation: swiglu positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [OBSERVATION_COLUMN] - target_columns: [action] - policy: - observation_columns: [OBSERVATION_COLUMN] - action_values: [0.0, 1.0] - return_conditioned: true + input_columns: + - OBSERVATION_COLUMN + - __policy_previous_action + - __policy_return + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -context_length: 4 -sources: - demonstrations.recorded: - paths: [PLEASE_FILL.csv] -validation_fraction: 0.2 -episode_cache_size: 8 -resume: {policy: never} -phases: -- algorithm: policy_imitation - epochs: 80 - sources: [demonstrations.recorded] - name: policy_imitation - batch_size: 64 - optimization: {actor: {learning_rate: 0.005}} + feature_embedding_dims: + OBSERVATION_COLUMN: 1 + __policy_previous_action: 6 + __policy_return: 1 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 4 + validation_fraction: 0.2 + episode_cache_size: 8 + resume: + policy: never +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: true + actions: + action: + type: categorical + values: + - 0.0 + - 1.0 +dataset_training: + demonstrations: + parts: + recorded: + type: sequence + paths: + - PLEASE_FILL.csv + model_interface: actor +training_plan: + phases: + - algorithm: policy_imitation + epochs: 80 + sources: + - source: demonstrations.recorded + name: policy_imitation + batch_size: 64 + optimization: + actor: + learning_rate: 0.005 diff --git a/src/sequifier/templates/train-ppo.yaml b/src/sequifier/templates/train-ppo.yaml index 64860f35..c4ae2a35 100644 --- a/src/sequifier/templates/train-ppo.yaml +++ b/src/sequifier/templates/train-ppo.yaml @@ -2,55 +2,85 @@ project_root: . model_name: online-ppo device: cpu seed: 1010 -context_length: 4 model: backbone: architecture: dim_model: 32 num_layers: 1 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} + attention: + n_heads: 4 + feed_forward: + dim: 64 positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [cue] - target_columns: [action] - policy: - observation_columns: [cue] - action_values: [left, right] - return_conditioned: false + input_columns: + - cue + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} -policy_interface: actor -normalization: {source: identity} -resume: {policy: if_exists, mode: exact} -phases: -- name: online_control - algorithm: ppo - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: + feature_embedding_dims: + cue: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 4 + normalization: + source: identity + resume: + policy: if_exists + mode: exact +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right +training_plan: + phases: + - name: online_control + algorithm: ppo environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 + seeds: + - 17 + - 29 + - 43 + - 71 + max_episode_steps: 8 + timeout_seconds: 30 diff --git a/src/sequifier/templates/train-supervised-to-policy.yaml b/src/sequifier/templates/train-supervised-to-policy.yaml new file mode 100644 index 00000000..49d55a05 --- /dev/null +++ b/src/sequifier/templates/train-supervised-to-policy.yaml @@ -0,0 +1,104 @@ +project_root: . +model_name: supervised-to-policy +device: cpu +seed: 1010 +model: + backbone: + architecture: + dim_model: 32 + num_layers: 2 + max_context_length: 4 + attention: + n_heads: 4 + feed_forward: + dim: 128 + activation: swiglu + positional_encoding_scope: global + dropout: 0.0 + interfaces: + actor: + input_columns: + - OBSERVATION_COLUMN + - __policy_previous_action + - __policy_return + target_columns: + - action + ingestion: + type: embedding + output_dim: 8 + feature_embedding_dims: + OBSERVATION_COLUMN: 1 + __policy_previous_action: 6 + __policy_return: 1 + decoder: + type: linear + prediction_length: 1 + support: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 4 + validation_fraction: 0.2 + episode_cache_size: 8 + resume: + policy: never + training_objective: causal + torch_compile: none +policy_binding: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: true + actions: + action: + type: categorical + values: + - 0.0 + - 1.0 +dataset_training: + demonstrations: + parts: + recorded: + type: sequence + paths: + - PLEASE_FILL.csv + model_interface: actor + observations: + model_interface: actor + parts: + train: + metadata_config_path: configs/metadata_configs/observations.json + criterion: + action: CrossEntropyLoss +training_plan: + phases: + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - source: observations.train + batch_size: 64 + accumulation_steps: 1 + optimization: + model: + optimizer: + name: Adam + learning_rate: 0.001 + scheduler: + name: StepLR + arguments: + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip: 1.0 + - algorithm: policy_imitation + epochs: 80 + sources: + - source: demonstrations.recorded + name: policy_imitation + batch_size: 64 + optimization: + actor: + learning_rate: 0.005 diff --git a/src/sequifier/templates/train-supervised.yaml b/src/sequifier/templates/train-supervised.yaml new file mode 100644 index 00000000..578b2d80 --- /dev/null +++ b/src/sequifier/templates/train-supervised.yaml @@ -0,0 +1,57 @@ +project_root: . +model_name: supervised +device: cpu +global_training: + training_objective: causal + context_length: 48 + inference_batch_size: 10 +model: + backbone: + architecture: + dim_model: 128 + num_layers: 3 + attention: + n_heads: 16 + feed_forward: + dim: 128 + dropout: 0.2 + interface: + input_columns: + - EXAMPLE_INPUT_COLUMN_NAME + target_columns: + - EXAMPLE_TARGET_COLUMN_NAME + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 +dataset: + part: + metadata_config_path: PLEASE FILL + criterion: + EXAMPLE_TARGET_COLUMN_NAME: MSELoss +training_plan: + phases: + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - source: default.default + batch_size: 10 + accumulation_steps: 1 + optimization: + model: + learning_rate: 0.0001 + optimizer: + name: AdamW + scheduler: + name: StepLR + arguments: + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch +evaluation: true +export_generative_model: true +export_embedding_model: false diff --git a/src/sequifier/train.py b/src/sequifier/train.py index f07818be..f7750f5b 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -19,12 +19,10 @@ ModelArtifact, load_execution_model, ) -from sequifier.config.learning_run import LearningRunConfig # noqa: E402 -from sequifier.config.train_config import PolicyRunConfig # noqa: E402 -from sequifier.config.train_config import load_train_config # noqa: E402 from sequifier.config.train_config import ( # noqa: E402 ResolvedSequifierConfig as TrainModel, ) +from sequifier.config.train_config import load_train_config # noqa: E402 from sequifier.distributed.env import setup_distributed_env # noqa: E402 from sequifier.helpers import ( # noqa: E402 configure_determinism, @@ -129,7 +127,7 @@ def train_worker( run = None try: run = RunBuilder(semantic_optimizer_grouping=semantic_optimizer_grouping).build( - config, execution, integrations + config, execution, integrations, initial_artifact=config.initialize_from ) loguru_logger.info( f"--- Starting Training for model: {run.context.model_name} | " @@ -164,7 +162,7 @@ def _mp_train_worker_wrapper( @beartype def run_training( - config: TrainModel | PolicyRunConfig | LearningRunConfig, + config: TrainModel, *, integration_specs: tuple[IntegrationSpec, ...] = (), integration_instances: tuple[Any, ...] = (), @@ -172,28 +170,23 @@ def run_training( ) -> Any: """Launch canonical service-composed training locally or across workers.""" - if isinstance(config, LearningRunConfig): + if isinstance(config, TrainModel) and any( + phase.algorithm != "supervised" for phase in config.training_plan + ): if integration_specs or integration_instances or semantic_optimizer_grouping: raise ValueError( - "mixed learning plans currently use phase-local native integration settings" + "policy phases do not support custom integrations or parameter grouping" ) + from sequifier.config.training_resolution import policy_run_config + from sequifier.rl.learner import run_policy_training from sequifier.training.coordinator import run_learning_plan + if ( + all(phase.algorithm != "supervised" for phase in config.training_plan) + and len({phase.model_interface for phase in config.training_plan}) == 1 + ): + return run_policy_training(policy_run_config(config, config.training_plan)) return run_learning_plan(config) - if isinstance(config, PolicyRunConfig): - if integration_specs or integration_instances or semantic_optimizer_grouping: - raise ValueError( - "policy training does not yet support custom integrations or parameter grouping" - ) - from sequifier.rl.learner import run_policy_training - - configure_logger(config.project_root, config.model_name, 0) - configure_determinism( - config.seed, - config.enforce_determinism, - ) - run_policy_training(config) - return if not isinstance(config, TrainModel): raise TypeError("Training requires a canonical resolved config.") spec = config.global_training diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index 78b578f9..827fb4f1 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -120,14 +120,6 @@ def strip_provenance(value: Any) -> Any: "resume_training": { "seed": config.seed, "read_format": training.read_format, - "batch_size": training.batch_size, - "accumulation_steps": training.accumulation_steps, - "learning_rate": training.learning_rate, - "optimizer": training.optimizer.model_dump(mode="python"), - "scheduler": training.scheduler.model_dump(mode="python"), - "scheduler_step_on": training.scheduler_step_on, - "reset_optimization_on_phase": (training.reset_optimization_on_phase), - "gradient_clip": training.gradient_clip, "distributed": training.distributed, "data_parallelism": training.data_parallelism, "world_size": training.world_size, diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 70678c0c..74232758 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -16,12 +16,7 @@ read_execution_artifact, save_model_artifact, ) -from sequifier.config.learning_run import LearningRunConfig -from sequifier.config.train_config import ( - PolicyRunConfig, - SequifierConfig, - resolve_sequifier_config, -) +from sequifier.config.train_config import SequifierConfig, resolve_sequifier_config from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save, write_json_atomic from sequifier.trajectory.sources import file_checksum @@ -39,6 +34,10 @@ def _supervised_config(config, phase, index, incoming, resume): from sequifier.artifacts.model_config import resolved_config_from_model_config from sequifier.config.metadata import load_dataset_metadata + authored = config.authored + # The coordinator stores resolved phases; the native builder owns source expansion. + phase_spec = authored.training_plan.phases[index].model_copy(deep=True) + phase_spec.optimization = phase.optimization model = config.model source = None forward_attention = None @@ -70,12 +69,19 @@ def _supervised_config(config, phase, index, incoming, resume): "global_training": training.model_dump(), "dataset_training": { name: value.model_dump() - for name, value in config.dataset_training.items() + for name, value in authored.dataset_training.items() + if name in {ref.source.partition(".")[0] for ref in phase.sources} }, - "training_plan": {"phases": [phase.model_dump(exclude={"handoff"})]}, - "evaluation": config.evaluation.model_dump() if config.evaluation else None, + "training_plan": {"phases": [phase_spec.model_dump()]}, + "evaluation": authored.evaluation.model_dump() + if authored.evaluation + else None, "export_pt": True, "export_onnx": config.export_onnx, + "export_generative_model": config.export_generative_model, + "export_embedding_model": config.export_embedding_model, + "embedding_layer_names": config.embedding_layer_names, + "export_with_dropout": config.export_with_dropout, } ) metadata = { @@ -161,49 +167,15 @@ def _run_supervised(config, phase, index, incoming, destination, resume): def _policy_config( config, phase, index, incoming, destination, resume, execution_model=None ): - if incoming is None and execution_model is None: - raise ValueError("a policy phase requires a model execution contract") - interface = phase.model_interface or config.policy_interface - if interface is None: - if execution_model is not None: - interfaces = execution_model["interfaces"] - else: - if incoming is None: - raise ValueError("policy initialization requires a model artifact") - interfaces = read_execution_artifact(incoming).model_config.values[ - "interfaces" - ] - if len(interfaces) != 1: - raise ValueError("select policy_interface or a phase model_interface") - interface = next(iter(interfaces)) - return PolicyRunConfig.model_validate( - { - "project_root": config.project_root, - "model_name": f"{config.model_name}-phase-{index:03d}-{phase.name}", - "device": config.device, - "seed": config.seed + index, - "initialize_from": incoming, - "policy_interface": interface, - "execution_model": execution_model, - "policy_binding": config.policy_bindings[interface].model_dump( - exclude_unset=True - ) - if interface in config.policy_bindings - else None, - "sources": { - name: value.model_dump() for name, value in config.sources.items() - }, - "preferences": config.preferences, - "units": config.units, - "phases": [phase.model_dump()], - "validation_fraction": config.validation_fraction, - "episode_cache_size": config.episode_cache_size, - "resume": { - "policy": "if_exists" if resume else "never", - "mode": config.resume.mode, - }, - "model_path": str(destination), - } + from sequifier.config.training_resolution import policy_run_config + + return policy_run_config( + config, + [phase], + incoming=incoming, + index=index, + destination=destination, + resuming=resume, ) @@ -214,9 +186,17 @@ def _run_policy(config, phase, index, incoming, destination, resume): return run_policy_training(run).phases[0].as_dict() -def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: - checkpoint = Path(config.checkpoint_path) - plan = config.model_dump(mode="json", exclude={"resume"}) +def run_learning_plan(config) -> LearningRunResult: + phases = config.training_plan + resume = config.global_training.resume + from sequifier.config.components import ResumeConfig + + resume = resume or ResumeConfig() + checkpoint = Path(config.project_root) / ( + resume.checkpoint_path or f"checkpoints/{config.model_name}/learning-run.pt" + ) + plan = config.model_dump(mode="json") + plan["global_training"].pop("resume", None) identity = fingerprint(plan) state: dict[str, Any] = { "artifact_type": "sequifier_learning_run", @@ -229,7 +209,7 @@ def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: else None, "active": False, } - if config.resume.policy != "never" and checkpoint.is_file(): + if resume.policy != "never" and checkpoint.is_file(): state = torch.load(checkpoint, map_location="cpu", weights_only=False) if ( state.get("artifact_type") != "sequifier_learning_run" @@ -237,9 +217,9 @@ def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: or state.get("plan") != identity ): raise ValueError("learning run checkpoint differs from the requested plan") - elif config.resume.policy == "required": + elif resume.policy == "required": raise FileNotFoundError(checkpoint) - if not 0 <= len(state["completed"]) <= len(config.phases): + if not 0 <= len(state["completed"]) <= len(phases): raise ValueError("learning run checkpoint exceeds the phase plan") if ( state["incoming"] @@ -256,7 +236,7 @@ def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: else None ) dependencies = {} - for index, phase in enumerate(config.phases): + for index, phase in enumerate(phases): if phase.algorithm == "supervised": resolved = _supervised_config( config, phase, index, state["incoming"], False @@ -297,7 +277,7 @@ def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: if config.initialize_from: protected.add(Path(config.initialize_from).resolve()) outputs = [ - Path(config.checkpoint_path), + checkpoint, Path(config.model_path), Path(config.report_path), ] @@ -311,8 +291,8 @@ def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: directory = Path(config.phase_directory) if any(directory == source or directory in source.parents for source in protected): raise ValueError("phase_directory overlaps a learning input") - for index in range(len(state["completed"]), len(config.phases)): - phase = config.phases[index] + for index in range(len(state["completed"]), len(phases)): + phase = phases[index] incoming = state["incoming"] if incoming and file_checksum(incoming) != state["incoming_sha256"]: raise ValueError( @@ -321,7 +301,7 @@ def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: destination = ( Path(config.phase_directory) / f"{index:03d}-{phase.name}" / "model.pt" ) - resuming = state["active"] and config.resume.policy != "never" + resuming = state["active"] and resume.policy != "never" if not state["active"]: state["active"] = True atomic_save(checkpoint, state) @@ -384,7 +364,7 @@ def run_learning_plan(config: LearningRunConfig) -> LearningRunResult: ) return LearningRunResult( config.model_path, - config.checkpoint_path, + str(checkpoint), config.report_path, tuple(state["completed"]), ) diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index f5b8661c..006b3722 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -350,6 +350,25 @@ def flush_metrics( for phase_index, phase in enumerate(config.training_plan): if phase_index < state.phase_index: continue + if phase_index != state.phase_index: + previous = config.training_plan[phase_index - 1] + if previous.handoff == "best": + selected = run.distributed.gather_objects( + state.best_model_state_dict + )[0] + if selected is None: + raise ValueError( + f"{previous.name}: best handoff has no finite evaluation" + ) + run.distributed.restore_model_state(run.network, selected) + state.best_validation_loss = float("inf") + state.best_model_state_dict = None + state.epochs_without_improvement = 0 + for dataset in run.datasets.values(): + for part in dataset.parts.values(): + if part.factory.batch_size != phase.batch_size: + part.factory.loaders.clear() + part.factory.batch_size = phase.batch_size sources = [ build_source_runtime(source, run.datasets) for source in phase.sources @@ -378,10 +397,7 @@ def flush_metrics( state.phase_index = phase_index state.phase_epoch = phase_epoch state.phase_epoch_complete = False - if ( - config.global_training.reset_optimization_on_phase - and run.optimization_phase_index != phase_index - ): + if run.optimization_phase_index != phase_index: run.start_phase(phase_index) state.optimizer_step = run.optimization.optimizer_step state.epoch = current_epoch diff --git a/src/sequifier/training/optimization.py b/src/sequifier/training/optimization.py index 4427ae29..d2ffcd5b 100644 --- a/src/sequifier/training/optimization.py +++ b/src/sequifier/training/optimization.py @@ -2,7 +2,6 @@ from __future__ import annotations -import inspect from copy import deepcopy from dataclasses import dataclass from typing import Any, Iterable @@ -98,7 +97,8 @@ def create( parameters: Iterable[nn.Parameter] | list[dict[str, Any]], *, parameter_catalog: ParameterCatalog | None = None, - phase_epochs: int | None = None, + accumulation_steps: int = 1, + layer_type_dtypes: dict[str, str] | None = None, ) -> "OptimizationRuntime": optimizer = create_optimizer( training.optimizer, @@ -112,12 +112,6 @@ def create( scheduler_arguments = resolve_plan_scheduler_arguments( training.optimizer, training.scheduler ) - if ( - training.scheduler_step_on == "epoch" - and phase_epochs is not None - and "total_steps" in inspect.signature(scheduler_class).parameters - ): - scheduler_arguments["total_steps"] = phase_epochs scheduler = scheduler_class(optimizer, **scheduler_arguments) # GradScaler cannot unscale FP16 optimizer gradients. Explicitly pre-cast # FP16 parameters use ordinary, unscaled backward/optimizer steps. @@ -127,8 +121,7 @@ def create( for parameter in group["params"] ) use_scaler = not has_fp16_parameters and bool( - training.layer_type_dtypes - and "float16" in training.layer_type_dtypes.values() + layer_type_dtypes and "float16" in layer_type_dtypes.values() ) return cls( optimizer=optimizer, @@ -136,7 +129,7 @@ def create( scaler=GradScaler(device=device.split(":")[0], enabled=use_scaler), scheduler_policy=SchedulerPolicy(training.scheduler_step_on), gradient_policy=GradientPolicy( - accumulation_steps=training.accumulation_steps or 1, + accumulation_steps=accumulation_steps, clip_norm=training.gradient_clip, ), ) diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 40aa3c1d..0f0f6b6b 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -66,6 +66,7 @@ class PartLoaderFactory: config: ResolvedSequifierConfig dataset_name: str part_name: str + batch_size: int = 1 loaders: dict[Literal["training", "validation"], DataLoader] = field( default_factory=dict, init=False, @@ -81,8 +82,9 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: if split in self.loaders: return self.loaders[split] - dataset_config: Any = dataset_part_view( - self.config, self.dataset_name, self.part_name + dataset_config: Any = replace( + dataset_part_view(self.config, self.dataset_name, self.part_name), + batch_size=self.batch_size, ) if split != "training" or not _training_plan_uses_part( self.config, self.dataset_name, self.part_name diff --git a/src/sequifier/training/scheduler_budgets.py b/src/sequifier/training/scheduler_budgets.py new file mode 100644 index 00000000..d5f4c4bd --- /dev/null +++ b/src/sequifier/training/scheduler_budgets.py @@ -0,0 +1,111 @@ +"""Preflight scheduler budgets using the same source ordering as training.""" + +from dataclasses import dataclass + +import torch + +from sequifier.config.train_config import ResolvedTrainingSource +from sequifier.io.batch import SequifierBatch +from sequifier.training.runtime import ( + DatasetRuntime, + RuntimeBatch, + SourceScheduler, + build_source_runtime, +) + + +@dataclass +class _CountedSource: + """Supply batch identities to the scheduler without consuming training data.""" + + config: ResolvedTrainingSource + dataset: DatasetRuntime + batches: int + + def set_epoch(self, epoch, split): + pass + + def iter_batches(self, split="training"): + batch = RuntimeBatch( + self.dataset.name, + self.config.part or "", + SequifierBatch( + {}, + {}, + { + "attention_valid_mask": torch.empty(0, dtype=torch.bool), + "target_valid_mask": torch.empty(0, dtype=torch.bool), + }, + ), + ) + for _ in range(self.batches): + yield batch + + +def _phase_update_budget(config, datasets, phase_index): + phase = config.training_plan[phase_index] + for dataset in datasets.values(): + for part in dataset.parts.values(): + if part.factory.batch_size != phase.batch_size: + part.factory.loaders.clear() + part.factory.batch_size = phase.batch_size + sources = [build_source_runtime(source, datasets) for source in phase.sources] + counted = [ + _CountedSource(source.config, source.dataset, source.num_batches()) + for source in sources + ] + scheduler = SourceScheduler( + phase, counted, seed=config.seed + phase_index, phase_index=phase_index + ) + updates = 0 + for epoch in range(1, phase.epochs + 1): + active_dataset, accumulated = None, 0 + for batch in scheduler.iter_epoch(epoch): + if active_dataset != batch.dataset: + updates += int(accumulated > 0) + accumulated = 0 + active_dataset = batch.dataset + accumulated += 1 + if accumulated == phase.accumulation_steps: + updates += 1 + accumulated = 0 + updates += int(accumulated > 0) + return updates + + +def validate_supervised_scheduler_budgets(config, datasets, active_phase_index): + """Validate maximum applied-update budgets, including dataset-boundary flushes.""" + phase_budgets = {} + for index, phase in enumerate(config.training_plan): + role = phase.optimization["model"] + if role.scheduler.name != "OneCycleLR" or role.scheduler_step_on != "batch": + continue + start = index + while start > 0 and config.training_plan[start].optimizer_state == "continue": + start -= 1 + end = index + 1 + while ( + end < len(config.training_plan) + and config.training_plan[end].optimizer_state == "continue" + ): + end += 1 + for selected in range(start, end): + if selected not in phase_budgets: + phase_budgets[selected] = _phase_update_budget( + config, datasets, selected + ) + budget = sum(phase_budgets[selected] for selected in range(start, end)) + if role.scheduler.arguments["total_steps"] != budget: + raise ValueError( + f"{phase.name}: OneCycleLR.total_steps must equal optimizer-update budget {budget}" + ) + for dataset in datasets.values(): + for part in dataset.parts.values(): + if ( + part.factory.batch_size + != config.training_plan[active_phase_index].batch_size + ): + part.factory.batch_size = config.training_plan[ + active_phase_index + ].batch_size + part.factory.loaders.clear() diff --git a/src/sequifier/training_api.py b/src/sequifier/training_api.py index ef862776..61414a18 100644 --- a/src/sequifier/training_api.py +++ b/src/sequifier/training_api.py @@ -1,6 +1,6 @@ """Stable update-aware primitives for training integrations.""" -from sequifier.config.learning_run import LearningRunConfig +from sequifier.config.train_config import ResolvedSequifierConfig, SequifierConfig from sequifier.integration.contexts import StepIdentity from sequifier.integration.controls import TrainingDirective from sequifier.training.coordinator import LearningRunResult, run_learning_plan @@ -23,7 +23,8 @@ "StepIdentity", "StepResult", "TrainingDirective", - "LearningRunConfig", + "SequifierConfig", + "ResolvedSequifierConfig", "LearningRunResult", "run_learning_plan", ] diff --git a/src/sequifier/trajectory/export.py b/src/sequifier/trajectory/export.py index 33fbf311..3479a776 100644 --- a/src/sequifier/trajectory/export.py +++ b/src/sequifier/trajectory/export.py @@ -19,6 +19,7 @@ def import_sequences(config) -> Path: import pyarrow.parquet as pq source = config.resolved_source() + policy = config.resolved_policy() destination = (Path(config.project_root) / config.output_path).resolve() if destination.exists(): raise ValueError( @@ -34,11 +35,9 @@ def import_sequences(config) -> Path: bootstrap_allowed="__bootstrap", group="__group", episode_id="episodeId", - successors={ - name: "__next_" + name for name in config.policy.observation_columns - }, + successors={name: "__next_" + name for name in policy.observation_columns}, ) - fields = config.policy.observation_columns + config.policy.static_columns + fields = policy.observation_columns + policy.static_columns reserved = { value for value in columns.model_dump().values() if isinstance(value, str) } | set(columns.successors.values()) @@ -46,47 +45,31 @@ def import_sequences(config) -> Path: raise ValueError( "policy feature names collide with sequence import metadata columns" ) - references = source_episodes(source, config.policy) + references = source_episodes(source, policy) if not references: raise ValueError("sequence import requires at least one episode") - real = ( - set(fields) - - set(config.policy.categorical_columns) - - config.policy.categorical_n_classes.keys() - ) - action_type = ( - pa.struct( - [ - pa.field( - name, - pa.string() - if field.type == "categorical" and isinstance(field.values[0], str) - else pa.float64(), - ) - for name, field in config.policy.actions.items() - ] - + [ - pa.field(action_latent_key(name, config.policy), pa.float64()) - for name, field in config.policy.actions.items() - if field.type == "real" - ] - + [ - pa.field(action_component_key(name, config.policy), pa.int64()) - for name, field in config.policy.actions.items() - if field.type == "real" - ] - ) - if config.policy.is_joint - else ( - pa.string() - if isinstance(config.policy.action_values[0], str) - else pa.float64() - ) + action_type = pa.struct( + [ + pa.field( + name, + pa.string() + if field.type == "categorical" and isinstance(field.values[0], str) + else pa.float64(), + ) + for name, field in policy.actions.items() + ] + + [ + pa.field(action_latent_key(name, policy), pa.float64()) + for name, field in policy.actions.items() + if field.type == "real" + ] + + [ + pa.field(action_component_key(name, policy), pa.int64()) + for name, field in policy.actions.items() + if field.type == "real" + ] ) - types = { - name: pa.float64() - for name in ({"__reward"} if config.policy.is_joint else real | {"__reward"}) - } + types = {name: pa.float64() for name in ({"__reward"})} types.update( {name: pa.bool_() for name in ("__terminated", "__truncated", "__bootstrap")} ) @@ -168,7 +151,7 @@ def flush(): "episodes": len(seen), "decisions": sum(shard["decisions"] for shard in shards), "columns": columns.model_dump(), - "policy_schema": config.policy.model_dump(mode="json"), + "policy_schema": policy.model_dump(mode="json"), "shards": shards, "provenance": { "source": source.model_dump(mode="json"), diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 40b0b73e..560be0bb 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from pathlib import Path -from sequifier.config.policy_config import PolicySchemaConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.sequence import SequenceEpisode, TransitionColumns @@ -14,7 +14,7 @@ def sequence_from_trajectory( - trajectory: Trajectory, policy: PolicySchemaConfig, action_semantics="policy" + trajectory: Trajectory, policy: ResolvedPolicyBinding, action_semantics="policy" ) -> SequenceEpisode: episode = validate_trajectory(trajectory) if ( @@ -47,8 +47,6 @@ def sequence_from_trajectory( for name, value in step.action_components.items() }, } - if policy.is_joint - else step.predictions[policy.action_column] for step in steps ), static_inputs=episode.seed.static_inputs, @@ -70,7 +68,7 @@ def sequence_from_trajectory( @dataclass(frozen=True) class TrajectoryReference: path: Path - policy: PolicySchemaConfig + policy: ResolvedPolicyBinding action_semantics: str = "policy" namespace: str = "" @@ -84,7 +82,7 @@ def load(self) -> SequenceEpisode: def source_episodes( - source: SequenceSourceConfig | ResolvedSequenceSource, policy: PolicySchemaConfig + source: SequenceSourceConfig | ResolvedSequenceSource, policy: ResolvedPolicyBinding ): resolved: ResolvedSequenceSource = ( resolve_source(source) if isinstance(source, SequenceSourceConfig) else source diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index bb56a437..38f159bb 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -170,85 +170,9 @@ def validate_temporal_contract(self): if self.version != 2: raise ValueError("joint policy trajectories require version 2") _validate_joint_execution(self) - if self.format == "sequifier.scalar-policy-trajectory": - if ( - sum(self.seed.attention_valid_mask) != 1 - or len(self.prediction_columns) != 1 - ): - raise ValueError( - "policy episodes require one initial observation and one action head" - ) - if "policy_execution" in self.metadata: - _validate_policy_execution(self) return self -def _validate_policy_execution(trajectory: Trajectory) -> None: - from sequifier.config.policy_config import CategoricalPolicyConfig - - execution = trajectory.metadata["policy_execution"] - if not isinstance(execution, dict): - raise ValueError("policy_execution must be an object") - if not isinstance(execution.get("deterministic"), bool): - raise ValueError( - "policy execution selection must declare a deterministic boolean" - ) - spec = CategoricalPolicyConfig.model_validate(execution.get("config")) - if ( - trajectory.prediction_columns != [spec.action_column] - or set(trajectory.seed.observations) != set(spec.observation_columns) - or set(trajectory.seed.static_inputs) != set(spec.static_columns) - ): - raise ValueError("policy execution schema does not match trajectory fields") - budgets = TypeAdapter(list[FiniteFloat]).validate_python( - execution.get("remaining_returns"), strict=True - ) - previous_actions = TypeAdapter(list[StrictInt]).validate_python( - execution.get("previous_action_ids"), strict=True - ) - log_probabilities = TypeAdapter(list[FiniteFloat]).validate_python( - execution.get("action_log_probabilities"), strict=True - ) - count = len(trajectory.transitions) - if ( - len(previous_actions) != count - or len(log_probabilities) != count - or any(value > 1e-6 for value in log_probabilities) - ): - raise ValueError( - "policy execution action IDs/log probabilities must match decision count" - ) - expected_previous = len(spec.action_values) - for index, step in enumerate(trajectory.transitions): - action = step.predictions[spec.action_column] - if ( - action not in spec.action_values - or previous_actions[index] != expected_previous - ): - raise ValueError( - "policy previous actions must lag executed categorical actions by one decision" - ) - expected_previous = spec.action_values.index(action) - if spec.return_conditioned: - initial = TypeAdapter(FiniteFloat).validate_python( - execution.get("initial_desired_return"), strict=True - ) - if len(budgets) != count + 1 or budgets[0] != initial: - raise ValueError( - "policy return budgets must include the initial and every successor budget" - ) - for index, step in enumerate(trajectory.transitions): - expected = remaining_return(budgets[index], step.reward, spec.returns.gamma) - if not math.isclose( - budgets[index + 1], expected, rel_tol=1e-12, abs_tol=1e-12 - ): - raise ValueError( - "policy return budgets violate the recorded reward/gamma arithmetic" - ) - elif budgets or execution.get("initial_desired_return") is not None: - raise ValueError("behavior cloning must not consume desired-return inputs") - - def validate_trajectory(value: Trajectory | dict) -> Trajectory: """Revalidate even an existing (possibly mutated) trajectory object.""" return Trajectory.model_validate( @@ -257,13 +181,13 @@ def validate_trajectory(value: Trajectory | dict) -> Trajectory: def _validate_joint_execution(trajectory): - from sequifier.config.policy_config import CategoricalPolicyConfig + from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.io.actions import encode_action execution = trajectory.metadata.get("policy_execution") if not isinstance(execution, dict): raise ValueError("policy trajectories require execution metadata") - spec = CategoricalPolicyConfig.model_validate(execution.get("config")) + spec = ResolvedPolicyBinding.model_validate(execution.get("config")) if trajectory.prediction_columns != spec.action_columns: raise ValueError("trajectory action fields differ from the binding") if set(trajectory.seed.observations) != set(spec.observation_columns) or set( @@ -299,7 +223,7 @@ def _validate_joint_execution(trajectory): if recorded_previous != expected: raise ValueError("previous actions must lag executed actions") expected = encode_action(step.predictions, spec) - if not isinstance(execution.get("deterministic"), bool): + if execution.get("action_selection") not in {"deterministic", "stochastic"}: raise ValueError("trajectory must declare its selection mode") budgets = TypeAdapter(list[FiniteFloat]).validate_python( execution.get("remaining_returns"), strict=True diff --git a/src/sequifier/trajectory/tables.py b/src/sequifier/trajectory/tables.py index 7708783c..68a0406f 100644 --- a/src/sequifier/trajectory/tables.py +++ b/src/sequifier/trajectory/tables.py @@ -8,7 +8,7 @@ import polars as pl from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata -from sequifier.config.policy_config import PolicySchemaConfig +from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.io.images import ImageStore from sequifier.io.preprocessing import denormalize_values @@ -78,7 +78,7 @@ def load(self, identity): return pl.from_arrow(rows.slice(start - self.offsets[first], end - start)) -def model_decoder(metadata, action_column, action_values): +def model_decoder(metadata, actions): """Decode model-encoded scalars using one fixed set of inverse codecs.""" inverse_maps = ( { @@ -112,7 +112,8 @@ def decode(value, column): return SPECIAL_TOKEN_IDS.labels_by_id[value] raise ValueError(f"{column}: encoded ID has no domain label") value = inverse[int(value)] - if column == action_column: + if column in actions and actions[column].type == "categorical": + action_values = actions[column].values if isinstance(value, str) and not isinstance(action_values[0], str): try: value = numeric_scalar(float(value), column) @@ -141,7 +142,7 @@ def decode(value, column): @dataclass(frozen=True) class TableEpisodeReference: source: SequenceSourceConfig - policy: PolicySchemaConfig + policy: ResolvedPolicyBinding sequence_id: str | int table: IndexedTable metadata: DatasetMetadata | None = None @@ -156,8 +157,7 @@ def load(self) -> SequenceEpisode: ) decode = model_decoder( self.metadata if self.prepared or self.source.encoding == "model" else None, - columns.action or self.policy.action_column, - self.policy.action_values, + self.policy.actions, ) preceding = None @@ -167,7 +167,7 @@ def load(self) -> SequenceEpisode: raise ValueError("prepared episode contains no split-owned decisions") first = decisions[columns.item_position][0] previous = data.filter(pl.col(columns.item_position) == first - 1) - if len(previous) and self.policy.is_joint: + if len(previous): mapping = columns.actions or { name: name for name in self.policy.action_columns } @@ -175,11 +175,6 @@ def load(self) -> SequenceEpisode: name: decode(previous[column][0], name) for name, column in mapping.items() } - elif len(previous): - preceding = decode( - previous[columns.action or self.policy.action_column][0], - columns.action or self.policy.action_column, - ) data = decisions static_data = None if self.static_table is not None: @@ -225,18 +220,11 @@ def table_features(data, policy, columns, static_data): (policy.static_columns, columns.static_inputs), ) ) - action_column = columns.action or policy.action_column - action_columns = ( - ( - columns.actions - or ( - {name: name for name in policy.action_columns} - if columns.action is None - else {name: action_column for name in policy.action_columns} - ) - ) - if policy.is_joint - else {policy.action_column: action_column} + action_column = columns.action + action_columns = columns.actions or ( + {name: name for name in policy.action_columns} + if columns.action is None + else {name: action_column for name in policy.action_columns} ) inputs = set(observations.values()) if static_data is None: @@ -293,7 +281,7 @@ def table_episode( ) -> SequenceEpisode: """Canonical table-to-episode validation shared by training and inference.""" data = ordered_sequence_rows(data, columns.item_position) - if policy.is_joint and decode is domain_scalar: + if decode is domain_scalar: def identity_decode(value, column): return value @@ -302,23 +290,16 @@ def identity_decode(value, column): if data.is_empty(): raise ValueError("sequence source changed while loading") - action_column = columns.action or policy.action_column - action_columns = ( - ( - columns.actions - or ( - {name: name for name in policy.action_columns} - if columns.action is None - else {name: action_column for name in policy.action_columns} - ) - ) - if policy.is_joint - else {policy.action_column: action_column} + action_column = columns.action + action_columns = columns.actions or ( + {name: name for name in policy.action_columns} + if columns.action is None + else {name: action_column for name in policy.action_columns} ) observations, static_inputs = table_features(data, policy, columns, static_data) def canonical_action(value): - if value is None or not policy.is_joint: + if value is None: return value return { name: entry @@ -327,11 +308,7 @@ def canonical_action(value): } def values(column, decoder=decode): - if ( - policy.is_joint - and policy.input_encoding == "model" - and column in observations.values() - ): + if policy.input_encoding == "model" and column in observations.values(): return tuple(data[column].to_list()) return tuple(decoder(value, column) for value in data[column]) @@ -351,48 +328,38 @@ def constant(entries, label): return entries[0] positions = values(columns.item_position, lambda value, _: value) - if policy.is_joint: - if set(action_columns) != set(policy.actions): - raise ValueError("table action mappings must cover every action field") - actions = tuple( - canonical_action(dict(row[columns.action])) - if columns.action is not None and not columns.actions - else { - name: decode(row[column], name) - for name, column in action_columns.items() - } + if set(action_columns) != set(policy.actions): + raise ValueError("table action mappings must cover every action field") + actions = tuple( + canonical_action(dict(row[columns.action])) + if columns.action is not None + and not columns.actions + and isinstance(row[columns.action], dict) + else { + name: decode(row[column], name) for name, column in action_columns.items() + } + for row in data.iter_rows(named=True) + ) + if columns.previous_actions: + if set(columns.previous_actions) != set(policy.actions): + raise ValueError("previous action mappings must cover every action field") + previous = tuple( + {name: row[column] for name, column in columns.previous_actions.items()} for row in data.iter_rows(named=True) ) - if columns.previous_actions: - if set(columns.previous_actions) != set(policy.actions): - raise ValueError( - "previous action mappings must cover every action field" - ) - previous = tuple( - {name: row[column] for name, column in columns.previous_actions.items()} - for row in data.iter_rows(named=True) - ) - preceding = ( - None - if all(value is None for value in previous[0].values()) - else previous[0] + preceding = ( + None + if all(value is None for value in previous[0].values()) + else previous[0] + ) + if previous[1:] != actions[:-1]: + raise ValueError( + "previous actions must lag recorded actions by one decision" ) - if previous[1:] != actions[:-1]: - raise ValueError( - "previous actions must lag recorded actions by one decision" - ) - else: - actions = values(action_column) if columns.previous_action is not None: preceding_actions = values( columns.previous_action, - lambda value, _: None - if value is None - else ( - canonical_action(value) - if policy.is_joint - else decode(value, action_column) - ), + lambda value, _: None if value is None else (canonical_action(value)), ) if preceding_actions[1:] != actions[:-1]: raise ValueError( @@ -407,9 +374,7 @@ def constant(entries, label): static = { name: constant( ( - value - if policy.is_joint and policy.input_encoding == "model" - else decode(value, column) + value if policy.input_encoding == "model" else decode(value, column) for value in static_data[column] ), f"static input {name}", @@ -442,7 +407,7 @@ def constant(entries, label): successors.append( { name: row[column] - if policy.is_joint and policy.input_encoding == "model" + if policy.input_encoding == "model" else decode(row[column], observations[name]) for name, column in columns.successors.items() } @@ -506,7 +471,6 @@ def table_episodes(source, policy, paths, *, prepared=False): image_store = None if ( prepared - and policy.is_joint and metadata is not None and (metadata.image_sources or metadata.preconditioning_image is not None) ): diff --git a/src/sequifier/visualize_training.py b/src/sequifier/visualize_training.py index 8d628a57..38acf6a6 100644 --- a/src/sequifier/visualize_training.py +++ b/src/sequifier/visualize_training.py @@ -21,8 +21,6 @@ class DataContinuityError(Exception): """Non-monotonic training batch or epoch sequence.""" - pass - @dataclass class TrainingMetrics: From 0dce86c3dd5e985014f23c65e0cfaf9005fef86c Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 10 Nov 2026 17:07:37 +0100 Subject: [PATCH 25/71] clean up --- documentation/config-capability-inventory.md | 20 +-- documentation/configs/infer.md | 21 ++- documentation/configs/train.md | 13 ++ .../example_inputs/configs/infer-policy.yaml | 2 +- .../example_inputs/configs/infer-ppo-toy.yaml | 2 +- .../configs/rollout-ppo-toy.yaml | 4 +- .../example_inputs/iql_toy_control.py | 2 +- documentation/policy-learning.md | 31 +++-- documentation/rollout.md | 14 +- release-notes/unreleased-phase-ownership.md | 16 ++- src/sequifier/config/infer_config.py | 67 ++------- src/sequifier/config/inference_common.py | 74 +++++++++- src/sequifier/config/policy_infer_config.py | 127 +++++++++++++----- src/sequifier/config/policy_plan.py | 88 +++++++++--- src/sequifier/config/policy_train_config.py | 12 ++ src/sequifier/config/rollout_config.py | 19 ++- src/sequifier/config/train_config.py | 100 +++++++++++++- src/sequifier/config/training_resolution.py | 87 ++++++------ src/sequifier/consolidated-docs.md | 79 ++++++++--- src/sequifier/infer.py | 6 +- src/sequifier/io/preference_dataset.py | 10 +- src/sequifier/rl/inference.py | 8 +- src/sequifier/rl/offline_data.py | 6 +- src/sequifier/rl/ppo_evaluation.py | 6 +- src/sequifier/rl/run_state.py | 6 +- src/sequifier/rl/training.py | 2 +- src/sequifier/sequifier.py | 9 ++ src/sequifier/templates/infer-policy.yaml | 2 +- 28 files changed, 604 insertions(+), 229 deletions(-) diff --git a/documentation/config-capability-inventory.md b/documentation/config-capability-inventory.md index 180d7a12..db93d1e8 100644 --- a/documentation/config-capability-inventory.md +++ b/documentation/config-capability-inventory.md @@ -309,7 +309,7 @@ This inventory records every directly declared field in the baseline run, policy | Baseline field | New owner / behavior | Acceptance coverage | | --- | --- | --- | | `sources` | training_plan.phases[].sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `validation_sources` | training_plan.phases[].validation_sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `validation_sources` | training_plan.phases[].evaluation.sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | ## `policy_plan.PolicyImitationPhaseConfig` @@ -317,15 +317,15 @@ This inventory records every directly declared field in the baseline run, policy | --- | --- | --- | | `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `epochs` | training_plan.phases[].epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `monitor` | training_plan.phases[].monitor | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `direction` | training_plan.phases[].direction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `monitor` | training_plan.phases[].evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `direction` | training_plan.phases[].evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | ## `policy_plan.IQLPhaseConfig` | Baseline field | New owner / behavior | Acceptance coverage | | --- | --- | --- | -| `monitor` | training_plan.phases[].monitor | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `direction` | training_plan.phases[].direction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `monitor` | training_plan.phases[].evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `direction` | training_plan.phases[].evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `name` | training_plan.phases[].name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `updates` | training_plan.phases[].updates | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | @@ -421,13 +421,13 @@ This inventory records every directly declared field in the baseline run, policy | --- | --- | --- | | `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `preferences` | training_plan.phases[].preferences | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `validation_preferences` | training_plan.phases[].validation_preferences | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `validation_preferences` | training_plan.phases[].evaluation.sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `updates` | training_plan.phases[].updates | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `beta` | training_plan.phases[].beta | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `scoring_chunk_size` | training_plan.phases[].scoring_chunk_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `reference` | training_plan.phases[].reference | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `monitor` | training_plan.phases[].monitor | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `direction` | training_plan.phases[].direction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `monitor` | training_plan.phases[].evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `direction` | training_plan.phases[].evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | ## `policy_plan.GRPOEvaluationConfig` @@ -504,7 +504,7 @@ This inventory records every directly declared field in the baseline run, policy | --- | --- | --- | | `environment` | rollout.environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `trajectory_path` | rollout.trajectory_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed_index` | rollout.seed_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed_index` | rollout.seed_row_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `max_episode_steps` | rollout.max_episode_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | ## `policy_infer_config.PolicyInferenceConfig` @@ -559,7 +559,7 @@ This inventory records every directly declared field in the baseline run, policy | `environment` | rollout.environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `bindings` | rollout.bindings | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `trajectory_path` | rollout.trajectory_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed_index` | rollout.seed_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | +| `seed_index` | rollout.seed_window_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | | `max_episode_steps` | rollout.max_episode_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | ## `sequence_import_config.SequenceImportConfig` diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index c9b0d0a1..ad5dbc05 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -305,7 +305,7 @@ dropout remains supported for embeddings. ```yaml project_root: . -model_type: policy +model_type: generative model_path: models/policy/model.pt model_interface: actor data_path: observations.csv @@ -321,7 +321,7 @@ rollout: implementation_id: sequifier.categorical-control.v1 units: {} max_steps: 8 - seed_index: 0 + seed_row_index: 0 trajectory_path: outputs/trajectory.json ``` @@ -329,8 +329,11 @@ Rollout schemas are selected by the inference route; do not declare `rollout.typ Forecast rollout requires `bindings` and a version-1 environment. Policy rollout rejects bindings and requires the distinct version-3 environment protocol, including implementation identity and matching units. Both use `max_steps` and -optional `max_episode_steps`. `seed_index` counts prepared windows in forecast -rollout and raw input rows in policy rollout, each zero-based in input order. +optional `max_episode_steps`. Forecast `seed_window_index` counts prepared +windows; policy `seed_row_index` counts raw input rows, each zero-based in input +order. The legacy `seed_index` spelling is accepted within each route, but +providing both spellings is an error. Trajectory output metadata retains its +public `seed_index` key and existing units. `enforce_determinism` controls deterministic execution, independently of action selection and seed. A seeded stochastic policy can reproduce its sampled sequence @@ -338,3 +341,13 @@ under the same supported runtime, device and input ordering. Online evaluation uses `action_selection` with the same meaning; its execution determinism is the training run's `global_training.enforce_determinism`. See the [environment protocols](../rollout.md) and [policy guide](../policy-learning.md). + + +With `model_type: generative`, a policy-bound interface defaults to policy +inference; other interfaces retain forecast behavior and supported formats. +Select `model_interface` when ambiguous. Optional `task: policy` requires one +policy-bound PT artifact; `task: forecast` requests forecasts, including from a +policy-bound interface with a supported forecast contract. Legacy +`model_type: policy` forces policy inference and conflicts with `task: forecast`. +Embedding rejects `task`; each route retains its own option and rollout schema. +The CLI accepts `--task policy|forecast`. diff --git a/documentation/configs/train.md b/documentation/configs/train.md index d451d4e7..24123a84 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -462,6 +462,19 @@ Momentum cycling defaults to off for both cycle schedulers in plan mode; explicit `cycle_momentum: true` is unsupported. `base_learning_rate` is only valid with `CyclicLR`. +## Single-phase compatibility + +For one phase, `global_training.batch_size` and `accumulation_steps` move onto +that phase; `learning_rate`, `optimizer`, `scheduler`, `scheduler_step_on`, and +`gradient_clip` move to `optimization.model` for supervised training only. +Multi-phase plans reject these fields, and duplicate global/local declarations +fail even when equal. Policy optimization must select its role explicitly. +Legacy supervised defaults retain no gradient clipping, require a single +optimizer's learning rate, and resolve null accumulation to 1. Explicit modern +roles use the defaults below. Composition, CLI and search normalize identically; +fingerprints use resolved phases. CLI flags use the hyphenated field names; +`--optimizer` and `--scheduler` accept YAML mappings. + ## Optimization across phases Each phase owns `batch_size` (32 by default), `accumulation_steps` (1), and diff --git a/documentation/example_inputs/configs/infer-policy.yaml b/documentation/example_inputs/configs/infer-policy.yaml index 379b77c7..2be2220f 100644 --- a/documentation/example_inputs/configs/infer-policy.yaml +++ b/documentation/example_inputs/configs/infer-policy.yaml @@ -1,7 +1,7 @@ project_root: . model_path: models/policy/model.pt data_path: data/policy-observations.csv -model_type: policy +model_type: generative read_format: csv write_format: csv device: cpu diff --git a/documentation/example_inputs/configs/infer-ppo-toy.yaml b/documentation/example_inputs/configs/infer-ppo-toy.yaml index d25900cb..0f78ff9d 100644 --- a/documentation/example_inputs/configs/infer-ppo-toy.yaml +++ b/documentation/example_inputs/configs/infer-ppo-toy.yaml @@ -1,7 +1,7 @@ project_root: . model_path: models/online-ppo/model.pt data_path: documentation/example_inputs/ppo-observations.csv -model_type: policy +model_type: generative read_format: csv write_format: csv device: cpu diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index d01473c9..07b3cc55 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -1,13 +1,13 @@ project_root: . model_path: models/online-ppo/model.pt data_path: documentation/example_inputs/ppo-observations.csv -model_type: policy +model_type: generative read_format: csv write_format: csv device: cpu rollout: trajectory_path: outputs/ppo-rollout.json - seed_index: 0 + seed_row_index: 0 max_episode_steps: 8 max_steps: 8 environment: diff --git a/documentation/example_inputs/iql_toy_control.py b/documentation/example_inputs/iql_toy_control.py index 6592265e..e5e53983 100644 --- a/documentation/example_inputs/iql_toy_control.py +++ b/documentation/example_inputs/iql_toy_control.py @@ -129,7 +129,7 @@ def evaluate(paths): }, "max_steps": HORIZON, "trajectory_path": "unused.json", - "seed_index": 0, + "seed_row_index": 0, "max_episode_steps": HORIZON, } ) diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 9d240409..cadac5be 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -91,13 +91,16 @@ manifests, or scalar prepared histories. A row has current observations and its executed action; optional rewards/successors describe the resulting transition. `PolicyTableColumns` exposes explicit observation/static/action/reward/group/ID mappings. Temporal positions must be contiguous, zero-based at episode starts. -Fragments require a known preceding action. Reward-free sequences are ordinary +Fragments require a known preceding action. +`columns.action` conflicts with nonempty `columns.actions`; likewise for +`previous_action` and `previous_actions`. Empty plural maps (`{}`) are unused +and may coexist with a singular mapping, including structured actions. Reward-free sequences are ordinary `SequenceEpisode` objects with absent transition capabilities, rather than fabricated reward-bearing trajectories. -`policy_imitation` uses `sources`, optional disjoint `validation_sources`, and +`policy_imitation` uses `sources`, optional disjoint `evaluation.sources`, and `epochs`. It minimizes action negative log likelihood at each valid causal -decision. The monitor is `validation_action_nll`, direction `min`. BC requires no +decision. The fixed `evaluation.monitor` is `{metric: validation_action_nll, mode: min}`. BC requires no rewards. Return conditioning requires recorded rewards and either true task termination or an explicit finite return horizon, using the declared discount and scale. Unknown return tails are excluded. The input return label at decision @@ -109,6 +112,14 @@ Episodes load through bounded caches; windows are constructed on demand. Episode fitting and decision indexing are shared independently of imitation batches, IQL transitions, and preference pairs. +Offline validation uses `evaluation.sources: [{source: demonstrations.holdout}]`; +omitting sources preserves automatic splitting. `evaluation.monitor` fixes +`mode: min` and `metric: validation_action_nll` for imitation/IQL or +`validation_dpo_loss` for DPO. These fields are assertions, not tuning choices. +Legacy `validation_sources` (DPO: `validation_preferences`), `monitor`, and +`direction` normalize to this shape; combining any with `evaluation` fails. +Online evaluation retains its environment/episode options and nested monitor. + ## Offline IQL `iql` requires complete rewards, termination/truncation and bootstrap flags, @@ -165,8 +176,10 @@ Source identities, exact spans, fingerprints and membership persist in the run checkpoint. The manifest must state the preferences; reward is not interpreted as a preference. -`dpo` requires `preferences` and `updates`; optional `validation_preferences` -selects an explicit validation manifest. `beta` defaults to 0.1 and must be +`dpo` requires `preferences` and `updates`; optional `evaluation.sources` +selects explicit validation preference manifests using the same `[{source: dataset.part}]` +shape as other offline phases. Its fixed monitor is +`{metric: validation_dpo_loss, mode: min}`. `beta` defaults to 0.1 and must be finite positive. `scoring_chunk_size` defaults to 64 causal windows. For the summed FP32 action log likelihood `s`, it minimizes equally weighted pair losses: @@ -276,7 +289,7 @@ the live actor synchronously and restores every module's training mode and the training RNG state afterward. It requires `environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; `action_selection: deterministic | stochastic` defaults to deterministic. Default -`metric: mean_completed_return`, `direction: max` selects true completed returns. +`evaluation.monitor: {metric: mean_completed_return, mode: max}` selects true completed returns. `mean_completed_length` and `success_rate` are also supported. Censored episodes and unavailable metrics are explicit; best selection fails when no finite monitored value is available. The final iteration always evaluates. @@ -322,7 +335,7 @@ the actor. Reference and old-policy scores stay detached across optimization epochs. The reported `sampled_reference_penalty` is a sampled estimator under old-policy samples, not exact current-policy KL. -Evaluation requires `metric: score_mean`, `direction: max` and scores every +Evaluation defaults to `evaluation.monitor: {metric: score_mean, mode: max}` and scores every independent evaluation episode through its real boundary or declared horizon. The evaluation score uses the same fixed reward scale. Evaluation/checkpoint intervals count completed updates, with an explicit final evaluation. Reports @@ -414,7 +427,7 @@ inference can validate `PolicyInferenceConfig`, resolve it with `rl_api` exports phase schemas, policy bindings, action declarations, environment capabilities, sequence records and `load_policy`. -Use `model_type: policy` with a selected portable PT artifact for raw CSV/Parquet +Use `model_type: generative` with a selected portable PT artifact for raw CSV/Parquet observation rows. `enforce_determinism` requests deterministic execution; `action_selection: deterministic | stochastic` selects the policy mode or sampling. The seed is independent of both. Seeded sampling can reproduce a sequence under @@ -423,7 +436,7 @@ the same supported runtime, device and input order. Online evaluation uses Evaluation reports record that choice. Policy `rollout.environment` declares the version-3 environment, `max_steps` -sets the collection budget, and `seed_index` selects a zero-based raw input row. +sets the collection budget, and `seed_row_index` selects a zero-based raw input row. `trajectory_path` is required; `max_episode_steps` is optional. Policy rollout rejects forecast bindings and generation settings. It requires seeded observation reset; successors come from actual environment transitions. True termination may diff --git a/documentation/rollout.md b/documentation/rollout.md index c5d53801..69c7418f 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -19,7 +19,7 @@ device: cpu model_type: generative rollout: max_steps: 3 - seed_index: 0 + seed_window_index: 0 trajectory_path: outputs/trajectory.json environment: factory: your_application.adapters:make_environment @@ -36,7 +36,7 @@ Run the existing `sequifier infer --config-path configs/infer.yaml` command. PT and existing schema-bearing ONNX models use the same coordinator and their existing inference backends. This feature adds no export format. -`seed_index` is required and selects one prepared model window, using zero-based +`seed_window_index` is required and selects one prepared model window, using zero-based dataset/chunk order. Unlike legacy autoregression, selection happens before the first-subsequence-per-sequence filter. It uses the usual inference window selection rules, including `window_stride`. An out-of-range index fails. The @@ -164,11 +164,17 @@ GRPO also requires reproducible group starts. See the The inference route selects the rollout schema. Policy rejects forecast `bindings`; forecast rejects policy implementation/units declarations. Neither schema accepts a rollout discriminator. Both use `max_steps`, `trajectory_path`, -`seed_index`, and optional `max_episode_steps`. Forecast `seed_index` indexes -prepared windows; policy `seed_index` indexes raw input rows before sequence +and optional `max_episode_steps`. Forecast `seed_window_index` indexes +prepared windows; policy `seed_row_index` indexes raw input rows before sequence history preparation. Both indices are zero-based and out-of-range values fail before environment creation. Use `enforce_determinism` for execution and policy `action_selection` for mode versus sampling. A fixed seed can make stochastic actions reproducible under the same supported runtime and ordering. + + +Both routes accept legacy `rollout.seed_index` as an alias for their respective +index field and reject both names in one declaration. Serialized trajectories +preserve the public metadata key `seed_index` and its original route-specific +units; this configuration rename does not change the output format. diff --git a/release-notes/unreleased-phase-ownership.md b/release-notes/unreleased-phase-ownership.md index d851bc6e..aaf64f2e 100644 --- a/release-notes/unreleased-phase-ownership.md +++ b/release-notes/unreleased-phase-ownership.md @@ -1,8 +1,8 @@ # Phase-owned training and explicit inference routes - Move batching and role-keyed optimization into each training phase. Supervised - uses `optimization.model`; policy roles remain algorithm-specific. Remove global - optimization fields and the global reset switch. + uses `optimization.model`; policy roles remain algorithm-specific. Normalize + legacy single-phase global fields and remove the global reset switch. - Add phase `optimizer_state: reset | continue`, with reset as the default and validated compatible native continuation. Reject mixed-plan continuation and policy OneCycle continuation. Preserve supervised epoch, update and metric @@ -14,11 +14,17 @@ - Rename inference execution determinism to `enforce_determinism` and policy inference/evaluation selection to `action_selection`. Seeded sampling and deterministic execution remain separate controls. -- Use `rollout.environment` and select rollout schemas from `model_type`. +- Use `rollout.environment` and select rollout schemas from the artifact interface and `task`. Forecast version-1 bindings and policy version-3 environments remain distinct. - Document prepared-window versus raw-row `seed_index` units. + Document forecast `seed_window_index` versus policy `seed_row_index` units. - Update shipped examples, scaffolding, Python examples and consolidated docs. CPU smoke tests cover native supervised and all policy algorithms, mixed handoff/recovery, inference, rollout and configuration rejection. -This is a breaking authoring change; obsolete placements and names are rejected. +Legacy single-phase global fields normalize to phase-local settings. Offline +policy evaluation uses `evaluation.sources` and `evaluation.monitor: {metric, +mode}`, with old fields accepted as aliases. Generative inference dispatches +from the selected interface, with optional `task` and legacy `model_type: policy`. +Rollout indices are `seed_window_index` (forecast) and `seed_row_index` (policy); +`seed_index` remains an input alias and the trajectory metadata key. Populated +singular/plural action mappings conflict; empty plural maps remain unused. diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index cb59f9ab..a9e9e18a 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -1,18 +1,20 @@ import json import os import re -from typing import Any, Generic, Optional, TypeVar, Union +from typing import Any, Generic, Literal, Optional, TypeVar, Union import numpy as np from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator -from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.config.composition import ( load_composed_yaml_config, merge_config_fragments, ) from sequifier.config.image_config import MediaPath, bind_image_paths, image_views -from sequifier.config.inference_common import InferenceExecutionConfig +from sequifier.config.inference_common import ( + InferenceExecutionConfig, + inference_model_config, +) from sequifier.config.metadata import ( DatasetMetadata, extract_inline_metadata, @@ -397,29 +399,14 @@ def load_inferer_config( key: value for key, value in args_config.items() if key != "skip_metadata" } authored_values = merge_config_fragments((config_values, cli_values)) - if authored_values.get("model_type") == "policy": - from sequifier.config.policy_infer_config import PolicyInferenceConfig - - PolicyInferenceConfig.model_validate(authored_values) - if skip_metadata: - raise ValueError("skip_metadata is not supported for policy inference") - else: - forecast_values, _ = extract_inline_metadata(authored_values) - InferenceConfig.model_validate(forecast_values) artifacts = {} - artifact_paths = authored_values.get("model_path", []) - if isinstance(artifact_paths, str): - artifact_paths = [artifact_paths] - by_path = {} - for path in artifact_paths: - if path.lower().endswith(".pt"): - normalized = normalize_path(path, authored_values.get("project_root", ".")) - if normalized not in by_path: - by_path[normalized] = read_execution_artifact(normalized) - artifacts[path] = by_path[normalized] policy_config = policy_inference_config(authored_values, artifacts) if policy_config is not None: + if skip_metadata: + raise ValueError("skip_metadata is not supported for policy inference") return policy_config + forecast_values, _ = extract_inline_metadata(authored_values) + InferenceConfig.model_validate(forecast_values) authored_values, extracted_metadata_values = extract_inline_metadata( authored_values ) @@ -511,38 +498,7 @@ def load_inferer_config( for model_path in model_paths: if not isinstance(model_path, str): continue - if model_path.lower().endswith(".pt"): - model_config = artifacts[model_path].model_config.values - elif model_path.lower().endswith(".onnx"): - import onnx - - from sequifier.model.execution_schema import ( - EXECUTION_SCHEMA_KEY, - MODEL_CONFIG_KEY, - ) - - graph = onnx.load( - normalize_path(model_path, project_root), - load_external_data=False, - ) - properties = {item.key: item.value for item in graph.metadata_props} - model_config = ( - json.loads(properties[MODEL_CONFIG_KEY]) - if MODEL_CONFIG_KEY in properties - else None - ) - if ( - any( - len(value.type.tensor_type.shape.dim) == 3 - for value in graph.graph.input - ) - and EXECUTION_SCHEMA_KEY not in properties - ): - raise ValueError( - "Depth ONNX graphs require execution-schema metadata" - ) - else: - continue + model_config = inference_model_config(model_path, project_root, artifacts) if model_config is None: continue if ( @@ -920,6 +876,7 @@ class _InferenceConfigBase( metadata_config_path: Optional[str] = None model_path: Union[str, list[str]] model_type: str + task: Literal["forecast"] | None = None training_objective: _ObjectiveT data_path: _DataPathT training_config_path: Optional[str] = None @@ -953,6 +910,8 @@ class _InferenceConfigBase( @model_validator(mode="after") @beartype def validate_route_selection(self): + if self.model_type == "embedding" and self.task is not None: + raise ValueError("task is only supported for generative inference") for usage, value in ( ("dataset", self.dataset), ("part", self.part), diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index c516e62b..a7caeeda 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -1,6 +1,9 @@ """Execution options consumed by every inference route.""" -from pydantic import BaseModel, ConfigDict, Field +import json +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class InferenceExecutionConfig(BaseModel): @@ -14,3 +17,72 @@ class InferenceExecutionConfig(BaseModel): seed: int = 1010 enforce_determinism: bool = False inference_batch_size: int = Field(default=1, gt=0) + + +class InferenceRouteConfig(BaseModel): + """Validate artifact and interface selection before route-specific options.""" + + model_config = ConfigDict(extra="ignore") + project_root: str = "." + model_path: str | list[str] + model_type: Literal["generative", "embedding", "policy"] + model_interface: str | None = None + task: Literal["policy", "forecast"] | None = None + + @field_validator("model_path") + @classmethod + def validate_paths(cls, value): + paths = [value] if isinstance(value, str) else value + if not paths or any(not path.strip() for path in paths): + raise ValueError("model_path requires at least one nonempty artifact path") + return value + + @field_validator("model_interface") + @classmethod + def validate_interface(cls, value): + if value is not None and not value.isidentifier(): + raise ValueError("model_interface must be a valid identifier without '.'") + return value + + @model_validator(mode="after") + def validate_task(self): + if self.model_type == "policy" and self.task == "forecast": + raise ValueError( + "model_type: policy conflicts with task: forecast; " + "use model_type: generative" + ) + if self.model_type == "embedding" and self.task is not None: + raise ValueError("task is only supported for generative inference") + return self + + +def inference_model_config(path, project_root, artifacts): + """Read the interface contract once for PT artifacts, or from ONNX metadata.""" + from sequifier.artifacts.model_artifact import read_execution_artifact + from sequifier.helpers import normalize_path + + normalized = normalize_path(path, project_root) + if path.lower().endswith(".pt"): + if normalized not in artifacts: + artifacts[normalized] = artifacts.get(path) or read_execution_artifact( + normalized + ) + artifacts[path] = artifacts[normalized] + return artifacts[path].model_config.values + if not path.lower().endswith(".onnx"): + return None + import onnx + + from sequifier.model.execution_schema import EXECUTION_SCHEMA_KEY, MODEL_CONFIG_KEY + + graph = onnx.load(normalized, load_external_data=False) + properties = {item.key: item.value for item in graph.metadata_props} + if EXECUTION_SCHEMA_KEY not in properties and any( + len(value.type.tensor_type.shape.dim) == 3 for value in graph.graph.input + ): + raise ValueError("Depth ONNX graphs require execution-schema metadata") + return ( + json.loads(properties[MODEL_CONFIG_KEY]) + if MODEL_CONFIG_KEY in properties + else None + ) diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index a52eba3d..da96b219 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -4,10 +4,15 @@ from pydantic import Field, PrivateAttr, model_validator -from sequifier.artifacts.model_artifact import ModelArtifact, read_execution_artifact -from sequifier.config.inference_common import InferenceExecutionConfig +from sequifier.artifacts.model_artifact import ModelArtifact +from sequifier.config.inference_common import ( + InferenceExecutionConfig, + InferenceRouteConfig, + inference_model_config, +) from sequifier.config.policy_config import PolicyConfigBase from sequifier.config.policy_plan import OnlineEnvironmentConfig +from sequifier.config.rollout_config import normalize_seed_index from sequifier.helpers import normalize_path @@ -15,13 +20,20 @@ class PolicyRolloutConfig(PolicyConfigBase): environment: OnlineEnvironmentConfig max_steps: int = Field(gt=0) trajectory_path: str = Field(min_length=1) - seed_index: int = Field(ge=0) + seed_row_index: int = Field(ge=0) max_episode_steps: int | None = Field(default=None, gt=0) + @model_validator(mode="before") + @classmethod + def normalize_index(cls, values): + return normalize_seed_index(values, "seed_row_index") + class PolicyInferenceConfig(InferenceExecutionConfig): model_path: str - model_type: Literal["policy"] + model_type: Literal["generative"] + task: Literal["policy"] = "policy" + data_path: str output_path: str | None = None output_probabilities: bool = False @@ -45,6 +57,16 @@ class PolicyInferenceConfig(InferenceExecutionConfig): prediction_length: int | None = Field(default=None, gt=0) decoder_support: int | None = Field(default=None, gt=0) + @model_validator(mode="before") + @classmethod + def normalize_policy_alias(cls, values): + if isinstance(values, dict) and values.get("model_type") == "policy": + if values.get("task") == "forecast": + raise ValueError("model_type: policy conflicts with task: forecast") + values = dict(values) + values["model_type"] = "generative" + return values + @model_validator(mode="after") def validate_execution(self): if self.read_format not in {"csv", "parquet"} or self.write_format not in { @@ -65,12 +87,71 @@ class ResolvedPolicyInferenceConfig(PolicyInferenceConfig): _artifacts: dict[str, ModelArtifact] = PrivateAttr(default_factory=dict) +def _policy_inference_route( + route: InferenceRouteConfig, artifacts: dict[str, ModelArtifact] +) -> tuple[str, str] | None: + """Inspect artifact interfaces before applying either inference schema.""" + if route.model_type == "embedding": + return None + paths = ( + [route.model_path] if isinstance(route.model_path, str) else route.model_path + ) + bindings = [] + selected = route.model_interface + for path in paths: + model_config = inference_model_config(path, route.project_root, artifacts) + if model_config is None: + bindings.append(False) + continue + interfaces = model_config.get("interfaces") + if not isinstance(interfaces, dict) or not interfaces: + raise ValueError("model_config.interfaces must be a non-empty mapping") + name = route.model_interface + if name is None: + policy_names = [ + key + for key, interface in interfaces.items() + if interface.get("policy") is not None + ] + if route.model_type == "policy" and len(policy_names) == 1: + name = policy_names[0] + elif len(interfaces) > 1 and policy_names and route.task != "forecast": + raise ValueError( + "Artifact has multiple interfaces; select model_interface before choosing task" + ) + else: + name = next(iter(interfaces)) + if name not in interfaces: + raise ValueError(f"Unknown model_interface {name!r}") + bindings.append(interfaces[name].get("policy") is not None) + selected = name + task = "policy" if route.model_type == "policy" else route.task + if task is None: + if any(bindings) and not all(bindings): + raise ValueError( + "Selected artifacts mix policy and forecast interfaces; select task explicitly" + ) + task = "policy" if all(bindings) else "forecast" + if task == "forecast": + return None + if len(paths) != 1 or not all(bindings): + raise ValueError( + "task: policy requires one artifact with a selected policy-bound interface" + ) + if not paths[0].lower().endswith(".pt"): + raise ValueError( + "task: policy requires a portable .pt artifact; " + "use task: forecast for forecast inference" + ) + assert selected is not None + return paths[0], selected + + def policy_inference_config( values: dict, artifacts: dict[str, ModelArtifact] | None = None ) -> ResolvedPolicyInferenceConfig | None: """Dispatch from the persisted interface contract, without training YAML.""" - if values.get("model_type") != "policy": - return None + route = InferenceRouteConfig.model_validate(values) from sequifier.artifacts.model_config import resolved_config_from_model_config from sequifier.config.infer_config import ( _assert_metadata_matches_source, @@ -78,32 +159,14 @@ def policy_inference_config( ) from sequifier.config.metadata import load_dataset_metadata - request = PolicyInferenceConfig.model_validate(values) artifacts = {} if artifacts is None else artifacts - model_path = values.get("model_path") - paths = [model_path] if isinstance(model_path, str) else model_path or [] - selected = values.get("model_interface") - policy_routes = [] - for path in paths: - if not isinstance(path, str) or not path.lower().endswith(".pt"): - continue - if path not in artifacts: - artifacts[path] = read_execution_artifact( - normalize_path(path, values.get("project_root", ".")) - ) - artifact = artifacts[path] - for name, interface in artifact.model_config.values["interfaces"].items(): - if interface.get("policy") is not None and ( - selected is None or selected == name - ): - policy_routes.append(name) - if len(paths) != 1 or len(policy_routes) != 1: - raise ValueError( - "policy inference requires one artifact and one selected policy interface" - ) - selected = policy_routes[0] + selection = _policy_inference_route(route, artifacts) + if selection is None: + return None + model_path, selected = selection + request = PolicyInferenceConfig.model_validate(values) execution, _ = resolved_config_from_model_config( - artifacts[paths[0]].model_config.values, + artifacts[model_path].model_config.values, device=request.device, interface_name=selected, ) @@ -132,10 +195,10 @@ def policy_inference_config( metadata, "policy metadata", "storage_layout" - in artifacts[paths[0]].model_config.values["interfaces"][selected], + in artifacts[model_path].model_config.values["interfaces"][selected], ) authored = request.model_dump() - authored["model_path"] = paths[0] + authored["model_path"] = model_path authored["model_interface"] = selected config = ResolvedPolicyInferenceConfig.model_validate(authored) config._artifacts = artifacts diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 75d28252..2cb06945 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -10,6 +10,30 @@ from sequifier.config.sources import TrainingSourceSpecModel +class OfflineEvaluationMonitor(BaseModel): + model_config = ConfigDict(extra="forbid") + metric: Literal["validation_action_nll"] = "validation_action_nll" + mode: Literal["min"] = "min" + + +class DPOEvaluationMonitor(BaseModel): + model_config = ConfigDict(extra="forbid") + metric: Literal["validation_dpo_loss"] = "validation_dpo_loss" + mode: Literal["min"] = "min" + + +class OfflineEvaluationConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + sources: list[TrainingSourceSpecModel] | None = Field(default=None, min_length=1) + monitor: OfflineEvaluationMonitor = Field(default_factory=OfflineEvaluationMonitor) + + +class DPOEvaluationConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + sources: list[TrainingSourceSpecModel] | None = Field(default=None, min_length=1) + monitor: DPOEvaluationMonitor = Field(default_factory=DPOEvaluationMonitor) + + class PhaseConfig(BaseModel): model_config = ConfigDict(extra="forbid", allow_inf_nan=False) name: str @@ -22,6 +46,48 @@ class PhaseConfig(BaseModel): evaluation_interval: int = Field(default=1, gt=0) checkpoint_interval: int = Field(default=1, gt=0) + @model_validator(mode="before") + @classmethod + def normalize_evaluation(cls, values): + if not isinstance(values, dict): + return values + source_field = ( + "validation_preferences" + if "preferences" in cls.model_fields + else "validation_sources" + if "sources" in cls.model_fields + else None + ) + legacy = { + "validation_sources", + "validation_preferences", + "monitor", + "direction", + } & values.keys() + if "evaluation" in values and legacy: + raise ValueError( + f"Cannot combine {sorted(legacy)} with evaluation; " + "use evaluation.sources and evaluation.monitor.{metric,mode}" + ) + if not legacy: + return values + values = dict(values) + evaluation = {} + if source_field in values: + source = values.pop(source_field) + evaluation["sources"] = ( + [source] + if source_field == "validation_preferences" and source is not None + else source + ) + evaluation["monitor"] = { + new: values.pop(old) + for old, new in (("monitor", "metric"), ("direction", "mode")) + if old in values + } + values["evaluation"] = evaluation + return values + def optimizer_step_budget(self, training_size=None): raise NotImplementedError("built-in phase must declare its optimizer budget") @@ -53,16 +119,12 @@ class BatchedPhaseConfig(PhaseConfig): class OfflinePhaseConfig(PhaseConfig): sources: list[TrainingSourceSpecModel] = Field(min_length=1) - validation_sources: list[TrainingSourceSpecModel] | None = Field( - default=None, min_length=1 - ) + evaluation: OfflineEvaluationConfig = Field(default_factory=OfflineEvaluationConfig) class PolicyImitationPhaseConfig(BatchedPhaseConfig, OfflinePhaseConfig): algorithm: Literal["policy_imitation"] epochs: int = Field(gt=0) - monitor: Literal["validation_action_nll"] = "validation_action_nll" - direction: Literal["min"] = "min" def optimizer_step_budget(self, training_size=None): if training_size is None: @@ -73,8 +135,6 @@ def optimizer_step_budget(self, training_size=None): class IQLPhaseConfig(OfflinePhaseConfig): optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value", "critic") - monitor: Literal["validation_action_nll"] = "validation_action_nll" - direction: Literal["min"] = "min" algorithm: Literal["iql"] name: str = Field(default="offline_iql", min_length=1) updates: int = Field(gt=0) @@ -151,6 +211,8 @@ def validate_seeds(cls, values): class OnlineEvaluationMonitor(BaseModel): + model_config = ConfigDict(extra="forbid") + metric: Literal[ "mean_completed_return", "success_rate", "mean_completed_length", "score_mean" ] = "mean_completed_return" @@ -160,14 +222,6 @@ class OnlineEvaluationMonitor(BaseModel): class OnlineEvaluationConfig(EvaluationEpisodesConfig): monitor: OnlineEvaluationMonitor = Field(default_factory=OnlineEvaluationMonitor) - @property - def metric(self): - return self.monitor.metric - - @property - def direction(self): - return self.monitor.mode - class PPOPhaseConfig(PhaseConfig): optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value") @@ -231,13 +285,11 @@ def location(self): class DPOPhaseConfig(BatchedPhaseConfig): algorithm: Literal["dpo"] preferences: TrainingSourceSpecModel - validation_preferences: TrainingSourceSpecModel | None = None + evaluation: DPOEvaluationConfig = Field(default_factory=DPOEvaluationConfig) updates: int = Field(gt=0) beta: float = Field(default=0.1, gt=0) scoring_chunk_size: int = Field(default=64, gt=0) reference: ReferenceConfig = Field(default_factory=ReferenceConfig) - monitor: Literal["validation_dpo_loss"] = "validation_dpo_loss" - direction: Literal["min"] = "min" def optimizer_step_budget(self, training_size=None): return self.updates diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 8648d41d..a477c1e6 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -27,6 +27,18 @@ class PolicyTableColumns(PolicyConfigBase): group: NonemptyStr | None = None episode_id: NonemptyStr | None = None + @model_validator(mode="after") + def validate_action_mappings(self): + for singular, plural in ( + ("action", "actions"), + ("previous_action", "previous_actions"), + ): + if getattr(self, singular) is not None and getattr(self, plural): + raise ValueError( + f"columns.{singular} and columns.{plural} cannot both be declared" + ) + return self + class SequenceSourceConfig(PolicyConfigBase): project_root: str = "." diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index f5c651bc..0e38a635 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -5,6 +5,18 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +def normalize_seed_index(values, canonical): + """Keep the legacy spelling route-local and out of serialized config.""" + if isinstance(values, dict) and "seed_index" in values: + if canonical in values: + raise ValueError( + f"rollout.seed_index and rollout.{canonical} cannot both be declared" + ) + values = dict(values) + values[canonical] = values.pop("seed_index") + return values + + class InputBinding(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) @@ -42,9 +54,14 @@ class RolloutConfig(BaseModel): bindings: list[InputBinding] trajectory_path: str = Field(min_length=1) # Index over prepared windows in dataset/chunk order, before legacy seed filtering. - seed_index: int = Field(ge=0) + seed_window_index: int = Field(ge=0) max_episode_steps: int | None = Field(default=None, gt=0) + @model_validator(mode="before") + @classmethod + def normalize_index(cls, values): + return normalize_seed_index(values, "seed_window_index") + @model_validator(mode="after") def reject_duplicate_producers(self): names = [binding.input for binding in self.bindings] diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index ee2129ee..16f588fa 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -141,6 +141,65 @@ def _unique_columns(value: list[str], usage: str) -> list[str]: _TRAINING_PLAN_POLICY_FIELDS = frozenset({"curriculum_training", "curriculum_column"}) +_LEGACY_BATCH_FIELDS = frozenset({"batch_size", "accumulation_steps"}) +_LEGACY_OPTIMIZATION_FIELDS = frozenset( + {"learning_rate", "optimizer", "scheduler", "scheduler_step_on", "gradient_clip"} +) +_LEGACY_TRAINING_FIELDS = _LEGACY_BATCH_FIELDS | _LEGACY_OPTIMIZATION_FIELDS + + +def _normalize_legacy_training_fields(global_training, phases, *, search=False): + """Move authored legacy fields before validating phases or search values.""" + if not isinstance(global_training, dict): + return + fields = global_training.keys() & _LEGACY_TRAINING_FIELDS + if not fields: + return + replacements = ", ".join( + f"global_training.{key} -> training_plan.phases[0]." + + (key if key in _LEGACY_BATCH_FIELDS else f"optimization.model.{key}") + for key in sorted(fields) + ) + if not isinstance(phases, (list, dict)) or len(phases) != 1: + raise ValueError( + f"Legacy training fields require exactly one phase: {replacements}" + ) + phase = next(iter(phases.values())) if isinstance(phases, dict) else phases[0] + if not isinstance(phase, dict): + raise ValueError("Legacy training fields require a phase mapping") + supervised = phase.get("algorithm", "supervised") == "supervised" + optimization_fields = fields & _LEGACY_OPTIMIZATION_FIELDS + if optimization_fields and not supervised: + raise ValueError( + f"Legacy optimization fields require a supervised phase: {replacements}; " + "policy optimization must select optimization. explicitly" + ) + optimization = phase.get("optimization", {}) + if not isinstance(optimization, dict): + raise ValueError("Phase optimization must be a mapping") + role = optimization.get("model", {}) + if not isinstance(role, dict): + raise ValueError("optimization.model must be a mapping") + legacy_defaults = not search and supervised and "model" not in optimization + for key in fields: + target = phase if key in _LEGACY_BATCH_FIELDS else role + if key in target: + raise ValueError(f"Duplicate training declaration: {replacements}") + value = global_training.pop(key) + target[key] = 1 if key == "accumulation_steps" and value is None else value + if optimization_fields or legacy_defaults: + phase.setdefault("optimization", {})["model"] = role + if legacy_defaults: + role.setdefault("gradient_clip", None) + optimizer = role.get("optimizer", {}) + if role.get("learning_rate") is None and not ( + isinstance(optimizer, dict) and "groups" in optimizer + ): + raise ValueError( + "Legacy supervised training requires an explicit learning_rate" + ) + + @beartype def _normalize_dataset_part_surface(datasets: Any) -> None: if not isinstance(datasets, dict): @@ -271,6 +330,11 @@ def normalize_train_config_surface(values: Any) -> Any: }: phase.setdefault("sources", [{"source": implicit_source}]) + _normalize_legacy_training_fields( + normalized.get("global_training"), + training_plan.get("phases") if isinstance(training_plan, dict) else None, + ) + evaluation = normalized.get("evaluation") if isinstance(evaluation, bool): if not evaluation: @@ -383,6 +447,18 @@ def normalize_train_config_parameter_surface( "phases": {0: phase_override}, } + global_override = normalized.get("global_training") + if ( + isinstance(global_override, dict) + and global_override.keys() & _LEGACY_TRAINING_FIELDS + ): + base_phases = base_values.get("training_plan", {}).get("phases") + if not isinstance(base_phases, list) or len(base_phases) != 1: + _normalize_legacy_training_fields(global_override, base_phases, search=True) + plan = normalized.setdefault("training_plan", {}) + phases = plan.setdefault("phases", {0: {}}) + _normalize_legacy_training_fields(global_override, phases, search=True) + return normalized @@ -990,12 +1066,11 @@ def validate_relationships(self): raise ValueError("supervised best handoff requires evaluation.monitor") for phase in self.training_plan.phases: refs = list(getattr(phase, "sources", [])) + list( - getattr(phase, "validation_sources", None) or [] + getattr(getattr(phase, "evaluation", None), "sources", None) or [] ) - for key in ("preferences", "validation_preferences"): - ref = getattr(phase, key, None) - if ref is not None: - refs.append(ref) + preference = getattr(phase, "preferences", None) + if preference is not None: + refs.append(preference) for ref in refs: self._validate_source(ref.source, f"phase {phase.name!r}") if phase.algorithm != "supervised" and ( @@ -2327,6 +2402,21 @@ def load_train_config_with_source( for key in ("model_name", "seed", "device"): if key in args: raw[key] = args.pop(key) + from sequifier.config.composition import merge_config_fragments + + legacy_cli = { + key: args.pop(key) for key in list(args) if key in _LEGACY_TRAINING_FIELDS + } + overrides = { + key: args.pop(key) + for key in ("global_training", "training_plan") + if key in args + } + if legacy_cli: + overrides["global_training"] = merge_config_fragments( + (overrides.get("global_training", {}), legacy_cli) + ) + raw = merge_config_fragments((raw, overrides)) if args: raise ValueError(f"Unsupported training CLI overrides: {sorted(args)}") diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 68e7da37..1d8e469f 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -31,11 +31,42 @@ def source_parts(config, reference): ] +def _resolve_phase_sources(config, phase, available, *, strict=True): + """Expand named sources for both metadata fitting and final run resolution.""" + preference = phase.algorithm == "dpo" + fields = ( + [(phase, "preferences"), (phase.evaluation, "sources")] + if preference + else [(phase, "sources"), (phase.evaluation, "sources")] + ) + for owner, key in fields: + refs = getattr(owner, key, None) + if refs is None: + continue + single = key == "preferences" + expanded = [ + ref.model_copy(update={"source": full}) + for ref in ([refs] if single else refs) + for full, _ in source_parts(config, ref.source) + ] + if not expanded or any(ref.source not in available for ref in expanded): + if not strict: + return False + raise ValueError( + f"{phase.name}: select {'preference' if preference else 'sequence'} parts" + ) + if single and len(expanded) != 1: + raise ValueError(f"{phase.name}: select one training preference part") + setattr(owner, key, expanded[0] if single else expanded) + return True + + def phase_interface(config, phase): names = set() - for ref in list(getattr(phase, "sources", [])) + list( - getattr(phase, "validation_sources", None) or [] - ): + refs = list(getattr(phase, "sources", [])) + if phase.algorithm in {"policy_imitation", "iql"}: + refs.extend(phase.evaluation.sources or []) + for ref in refs: names.add(config.dataset_training[ref.source.partition(".")[0]].model_interface) explicit = getattr(phase, "model_interface", None) if explicit is not None: @@ -192,24 +223,13 @@ def _fit_input_metadata(config, name, binding, metadata, root): elif isinstance(part, PreferencePartSpecModel): preferences[ref] = str(root / part.manifest) phase = phase.model_copy(deep=True) - for key in ("sources", "validation_sources"): - refs = getattr(phase, key, None) - if refs is not None: - expanded = [ - ref.model_copy(update={"source": full}) - for ref in refs - for full, _ in source_parts(config, ref.source) - ] - if any(ref.source not in sources for ref in expanded): - return metadata # Prepared sources already own frozen preprocessing metadata. - setattr(phase, key, expanded) - for key in ("preferences", "validation_preferences"): - ref = getattr(phase, key, None) - if ref is not None: - parts = source_parts(config, ref.source) - if len(parts) != 1: - raise ValueError(f"{phase.name}: select one preference part") - setattr(phase, key, ref.model_copy(update={"source": parts[0][0]})) + if not _resolve_phase_sources( + config, + phase, + preferences if phase.algorithm == "dpo" else sources, + strict=False, + ): + return metadata # Prepared sources already own frozen preprocessing metadata. run = SimpleNamespace( validation_fraction=config.global_training.validation_fraction, seed=config.seed, @@ -462,28 +482,9 @@ def resolve_training_run(authored: SequifierConfig): raise ValueError( f"{phase.name}: interface {name!r} requires a policy binding" ) - for key in ("sources", "validation_sources"): - refs = getattr(phase, key, None) - if refs is None: - continue - expanded = [] - for ref in refs: - for full, _ in source_parts(config, ref.source): - if full not in sequences: - raise ValueError( - f"{phase.name}: {full} is not a sequence source" - ) - expanded.append(ref.model_copy(update={"source": full})) - setattr(phase, key, expanded) - if phase.algorithm == "dpo": - for key in ("preferences", "validation_preferences"): - ref = getattr(phase, key) - if ref is None: - continue - parts = source_parts(config, ref.source) - if len(parts) != 1 or parts[0][0] not in preferences: - raise ValueError(f"{phase.name}: select one preference part") - setattr(phase, key, ref.model_copy(update={"source": parts[0][0]})) + _resolve_phase_sources( + config, phase, preferences if phase.algorithm == "dpo" else sequences + ) for environment in ( getattr(phase, "environment", None), getattr(getattr(phase, "evaluation", None), "environment", None), diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md index ddabae20..9f2fe0a5 100644 --- a/src/sequifier/consolidated-docs.md +++ b/src/sequifier/consolidated-docs.md @@ -1415,6 +1415,19 @@ Momentum cycling defaults to off for both cycle schedulers in plan mode; explicit `cycle_momentum: true` is unsupported. `base_learning_rate` is only valid with `CyclicLR`. +## Single-phase compatibility + +For one phase, `global_training.batch_size` and `accumulation_steps` move onto +that phase; `learning_rate`, `optimizer`, `scheduler`, `scheduler_step_on`, and +`gradient_clip` move to `optimization.model` for supervised training only. +Multi-phase plans reject these fields, and duplicate global/local declarations +fail even when equal. Policy optimization must select its role explicitly. +Legacy supervised defaults retain no gradient clipping, require a single +optimizer's learning rate, and resolve null accumulation to 1. Explicit modern +roles use the defaults below. Composition, CLI and search normalize identically; +fingerprints use resolved phases. CLI flags use the hyphenated field names; +`--optimizer` and `--scheduler` accept YAML mappings. + ## Optimization across phases Each phase owns `batch_size` (32 by default), `accumulation_steps` (1), and @@ -2163,7 +2176,7 @@ dropout remains supported for embeddings. ```yaml project_root: . -model_type: policy +model_type: generative model_path: models/policy/model.pt model_interface: actor data_path: observations.csv @@ -2179,7 +2192,7 @@ rollout: implementation_id: sequifier.categorical-control.v1 units: {} max_steps: 8 - seed_index: 0 + seed_row_index: 0 trajectory_path: outputs/trajectory.json ``` @@ -2187,8 +2200,11 @@ Rollout schemas are selected by the inference route; do not declare `rollout.typ Forecast rollout requires `bindings` and a version-1 environment. Policy rollout rejects bindings and requires the distinct version-3 environment protocol, including implementation identity and matching units. Both use `max_steps` and -optional `max_episode_steps`. `seed_index` counts prepared windows in forecast -rollout and raw input rows in policy rollout, each zero-based in input order. +optional `max_episode_steps`. Forecast `seed_window_index` counts prepared +windows; policy `seed_row_index` counts raw input rows, each zero-based in input +order. The legacy `seed_index` spelling is accepted within each route, but +providing both spellings is an error. Trajectory output metadata retains its +public `seed_index` key and existing units. `enforce_determinism` controls deterministic execution, independently of action selection and seed. A seeded stochastic policy can reproduce its sampled sequence @@ -2198,6 +2214,16 @@ training run's `global_training.enforce_determinism`. See the [environment protocols](../rollout.md) and [policy guide](../policy-learning.md). +With `model_type: generative`, a policy-bound interface defaults to policy +inference; other interfaces retain forecast behavior and supported formats. +Select `model_interface` when ambiguous. Optional `task: policy` requires one +policy-bound PT artifact; `task: forecast` requests forecasts, including from a +policy-bound interface with a supported forecast contract. Legacy +`model_type: policy` forces policy inference and conflicts with `task: forecast`. +Embedding rejects `task`; each route retains its own option and rollout schema. +The CLI accepts `--task policy|forecast`. + + # Visualize Training Command Guide The `sequifier visualize-training` command reads the structured metric files generated during training and hyperparameter search to create interactive Plotly HTML visualizations of the training and validation losses. It supports viewing a single model's progress or comparing multiple models side-by-side. @@ -2609,13 +2635,16 @@ manifests, or scalar prepared histories. A row has current observations and its executed action; optional rewards/successors describe the resulting transition. `PolicyTableColumns` exposes explicit observation/static/action/reward/group/ID mappings. Temporal positions must be contiguous, zero-based at episode starts. -Fragments require a known preceding action. Reward-free sequences are ordinary +Fragments require a known preceding action. +`columns.action` conflicts with nonempty `columns.actions`; likewise for +`previous_action` and `previous_actions`. Empty plural maps (`{}`) are unused +and may coexist with a singular mapping, including structured actions. Reward-free sequences are ordinary `SequenceEpisode` objects with absent transition capabilities, rather than fabricated reward-bearing trajectories. -`policy_imitation` uses `sources`, optional disjoint `validation_sources`, and +`policy_imitation` uses `sources`, optional disjoint `evaluation.sources`, and `epochs`. It minimizes action negative log likelihood at each valid causal -decision. The monitor is `validation_action_nll`, direction `min`. BC requires no +decision. The fixed `evaluation.monitor` is `{metric: validation_action_nll, mode: min}`. BC requires no rewards. Return conditioning requires recorded rewards and either true task termination or an explicit finite return horizon, using the declared discount and scale. Unknown return tails are excluded. The input return label at decision @@ -2627,6 +2656,14 @@ Episodes load through bounded caches; windows are constructed on demand. Episode fitting and decision indexing are shared independently of imitation batches, IQL transitions, and preference pairs. +Offline validation uses `evaluation.sources: [{source: demonstrations.holdout}]`; +omitting sources preserves automatic splitting. `evaluation.monitor` fixes +`mode: min` and `metric: validation_action_nll` for imitation/IQL or +`validation_dpo_loss` for DPO. These fields are assertions, not tuning choices. +Legacy `validation_sources` (DPO: `validation_preferences`), `monitor`, and +`direction` normalize to this shape; combining any with `evaluation` fails. +Online evaluation retains its environment/episode options and nested monitor. + ## Offline IQL `iql` requires complete rewards, termination/truncation and bootstrap flags, @@ -2683,8 +2720,10 @@ Source identities, exact spans, fingerprints and membership persist in the run checkpoint. The manifest must state the preferences; reward is not interpreted as a preference. -`dpo` requires `preferences` and `updates`; optional `validation_preferences` -selects an explicit validation manifest. `beta` defaults to 0.1 and must be +`dpo` requires `preferences` and `updates`; optional `evaluation.sources` +selects explicit validation preference manifests using the same `[{source: dataset.part}]` +shape as other offline phases. Its fixed monitor is +`{metric: validation_dpo_loss, mode: min}`. `beta` defaults to 0.1 and must be finite positive. `scoring_chunk_size` defaults to 64 causal windows. For the summed FP32 action log likelihood `s`, it minimizes equally weighted pair losses: @@ -2794,7 +2833,7 @@ the live actor synchronously and restores every module's training mode and the training RNG state afterward. It requires `environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; `action_selection: deterministic | stochastic` defaults to deterministic. Default -`metric: mean_completed_return`, `direction: max` selects true completed returns. +`evaluation.monitor: {metric: mean_completed_return, mode: max}` selects true completed returns. `mean_completed_length` and `success_rate` are also supported. Censored episodes and unavailable metrics are explicit; best selection fails when no finite monitored value is available. The final iteration always evaluates. @@ -2840,7 +2879,7 @@ the actor. Reference and old-policy scores stay detached across optimization epochs. The reported `sampled_reference_penalty` is a sampled estimator under old-policy samples, not exact current-policy KL. -Evaluation requires `metric: score_mean`, `direction: max` and scores every +Evaluation defaults to `evaluation.monitor: {metric: score_mean, mode: max}` and scores every independent evaluation episode through its real boundary or declared horizon. The evaluation score uses the same fixed reward scale. Evaluation/checkpoint intervals count completed updates, with an explicit final evaluation. Reports @@ -2932,7 +2971,7 @@ inference can validate `PolicyInferenceConfig`, resolve it with `rl_api` exports phase schemas, policy bindings, action declarations, environment capabilities, sequence records and `load_policy`. -Use `model_type: policy` with a selected portable PT artifact for raw CSV/Parquet +Use `model_type: generative` with a selected portable PT artifact for raw CSV/Parquet observation rows. `enforce_determinism` requests deterministic execution; `action_selection: deterministic | stochastic` selects the policy mode or sampling. The seed is independent of both. Seeded sampling can reproduce a sequence under @@ -2941,7 +2980,7 @@ the same supported runtime, device and input order. Online evaluation uses Evaluation reports record that choice. Policy `rollout.environment` declares the version-3 environment, `max_steps` -sets the collection budget, and `seed_index` selects a zero-based raw input row. +sets the collection budget, and `seed_row_index` selects a zero-based raw input row. `trajectory_path` is required; `max_episode_steps` is optional. Policy rollout rejects forecast bindings and generation settings. It requires seeded observation reset; successors come from actual environment transitions. True termination may @@ -3184,7 +3223,7 @@ device: cpu model_type: generative rollout: max_steps: 3 - seed_index: 0 + seed_window_index: 0 trajectory_path: outputs/trajectory.json environment: factory: your_application.adapters:make_environment @@ -3201,7 +3240,7 @@ Run the existing `sequifier infer --config-path configs/infer.yaml` command. PT and existing schema-bearing ONNX models use the same coordinator and their existing inference backends. This feature adds no export format. -`seed_index` is required and selects one prepared model window, using zero-based +`seed_window_index` is required and selects one prepared model window, using zero-based dataset/chunk order. Unlike legacy autoregression, selection happens before the first-subsequence-per-sequence filter. It uses the usual inference window selection rules, including `window_stride`. An out-of-range index fails. The @@ -3329,11 +3368,17 @@ GRPO also requires reproducible group starts. See the The inference route selects the rollout schema. Policy rejects forecast `bindings`; forecast rejects policy implementation/units declarations. Neither schema accepts a rollout discriminator. Both use `max_steps`, `trajectory_path`, -`seed_index`, and optional `max_episode_steps`. Forecast `seed_index` indexes -prepared windows; policy `seed_index` indexes raw input rows before sequence +and optional `max_episode_steps`. Forecast `seed_window_index` indexes +prepared windows; policy `seed_row_index` indexes raw input rows before sequence history preparation. Both indices are zero-based and out-of-range values fail before environment creation. Use `enforce_determinism` for execution and policy `action_selection` for mode versus sampling. A fixed seed can make stochastic actions reproducible under the same supported runtime and ordering. + + +Both routes accept legacy `rollout.seed_index` as an alias for their respective +index field and reject both names in one declaration. Serialized trajectories +preserve the public metadata key `seed_index` and its original route-specific +units; this configuration rename does not change the output format. diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index 75362b87..260f862c 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -1201,7 +1201,7 @@ def infer_numeric_rollout(config, inferer, model_id, dataset, column_data_types) from sequifier.trajectory.store import save_trajectory rollout = config.rollout - remaining = rollout.seed_index + remaining = rollout.seed_window_index selected = None for data_id, data in enumerate(dataset): windowed = _windowed_inference_batch(config, data, column_data_types) @@ -1213,7 +1213,7 @@ def infer_numeric_rollout(config, inferer, model_id, dataset, column_data_types) break if selected is None: raise ValueError( - f"rollout seed_index {rollout.seed_index} is outside the prepared dataset windows" + f"rollout seed_window_index {rollout.seed_window_index} is outside the prepared dataset windows" ) data_id, windowed, row = selected metadata = config.dataset_metadata @@ -1296,7 +1296,7 @@ def predict(history): "enforce_determinism": config.enforce_determinism, "seed_origin": { "data_id": data_id, - "seed_index": rollout.seed_index, + "seed_index": rollout.seed_window_index, "sequence_id": int(windowed.sequence_ids[row]), "subsequence_id": int(windowed.subsequence_ids[row]), "model_start_position": int(windowed.model_start_positions[row]), diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index 5a8cb6cc..cd5fc545 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -189,14 +189,18 @@ def prepare_preferences(run, phase, config, catalog, cache): key = ( "preferences", phase.preferences.source, - phase.validation_preferences.source if phase.validation_preferences else None, + tuple(ref.source for ref in (phase.evaluation.sources or [])), ) if key in cache: return cache[key] pairs = load_preferences(run.preferences[phase.preferences.source]).pairs validation = ( - load_preferences(run.preferences[phase.validation_preferences.source]).pairs - if phase.validation_preferences + [ + pair + for ref in phase.evaluation.sources + for pair in load_preferences(run.preferences[ref.source]).pairs + ] + if phase.evaluation.sources else None ) all_pairs = pairs + (validation or []) diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index c35f9d71..c723091e 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -83,9 +83,9 @@ def desired_return(config, spec, row): def infer_rollout(policy, data, config): spec = policy.config rollout = config.rollout - if rollout.seed_index >= len(data): - raise ValueError("policy rollout seed_index is outside the input rows") - row = data.row(rollout.seed_index, named=True) + if rollout.seed_row_index >= len(data): + raise ValueError("policy rollout seed_row_index is outside the input rows") + row = data.row(rollout.seed_row_index, named=True) seed = SeedHistory( observations={ column: [0.0] * (spec.context_length - 1) + [row[column]] @@ -109,7 +109,7 @@ def infer_rollout(policy, data, config): { "model_path": config.model_path, "model_interface": config.model_interface, - "seed_index": rollout.seed_index, + "seed_index": rollout.seed_row_index, "enforce_determinism": config.enforce_determinism, "seed": config.seed, } diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index 11449ed0..e0bb88db 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -57,14 +57,14 @@ def prepare_offline(run, phase, config, catalog, cache): key = ( "offline", tuple(ref.source for ref in phase.sources), - tuple(ref.source for ref in (phase.validation_sources or [])), + tuple(ref.source for ref in (phase.evaluation.sources or [])), ) if key in cache: return cache[key] corpus = catalog.dataset training = corpus([ref.source for ref in phase.sources]) - if phase.validation_sources: - validation = corpus([ref.source for ref in phase.validation_sources]) + if phase.evaluation.sources: + validation = corpus([ref.source for ref in phase.evaluation.sources]) else: training, validation = split_by_episode( training, run.validation_fraction, run.seed, group_identical=True diff --git a/src/sequifier/rl/ppo_evaluation.py b/src/sequifier/rl/ppo_evaluation.py index cd4cc039..2964d7c7 100644 --- a/src/sequifier/rl/ppo_evaluation.py +++ b/src/sequifier/rl/ppo_evaluation.py @@ -144,13 +144,13 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): } if successes and all(success is not None for success in successes): result["success_rate"] = statistics.fmean(successes) - if evaluation.metric == "score_mean": + if evaluation.monitor.metric == "score_mean": if timed_out or len(episodes) != len(evaluation.seeds): raise ValueError("GRPO evaluation requires every finite-horizon score") result["score_mean"] = statistics.fmean( episode["return"] for episode in episodes ) - result["selection_metric"] = evaluation.metric - result["selection_value"] = result.get(evaluation.metric) + result["selection_metric"] = evaluation.monitor.metric + result["selection_value"] = result.get(evaluation.monitor.metric) result["selection_available"] = result["selection_value"] is not None return result diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 29faa73b..9ca06c2d 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -270,10 +270,10 @@ def __init__(self, context, *, modules=None, frozen=None): self.state = context.resume_state or {} self.transaction = self.state.get("transaction", 0) self.metrics = list(self.state.get("metrics", [])) - selection = getattr(context.phase, "evaluation", context.phase) - monitor = getattr(selection, "metric", getattr(selection, "monitor", None)) + selection = context.phase.evaluation.monitor + monitor = selection.metric self.selection = Selection( - context.phase, monitor, selection.direction, self.state.get("selection") + context.phase, monitor, selection.mode, self.state.get("selection") ) if self.state: require_equal( diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 19d933e7..e4d847b6 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -76,7 +76,7 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin "transaction": session.transaction, "epochs_completed": batches.epoch, "training_action_nll": total, - phase.monitor: score, + phase.evaluation.monitor.metric: score, } ) session.checkpoint(final, batch_order=batches.state_dict()) diff --git a/src/sequifier/sequifier.py b/src/sequifier/sequifier.py index 8c8da347..9eeda801 100644 --- a/src/sequifier/sequifier.py +++ b/src/sequifier/sequifier.py @@ -2,6 +2,7 @@ from typing import Any import numpy as np +import yaml from sequifier.docs import docs from sequifier.hyperparameter_search import hyperparameter_search @@ -119,6 +120,13 @@ def setup_parser() -> ArgumentParser: ) parser_train.add_argument("-mn", "--model-name", type=str) parser_train.add_argument("-s", "--seed", type=int) + parser_train.add_argument("--batch-size", type=int) + parser_train.add_argument("--accumulation-steps", type=int) + parser_train.add_argument("--learning-rate", type=float) + parser_train.add_argument("--gradient-clip", type=float) + parser_train.add_argument("--scheduler-step-on", choices=("epoch", "batch")) + parser_train.add_argument("--optimizer", type=yaml.safe_load) + parser_train.add_argument("--scheduler", type=yaml.safe_load) parser_infer.add_argument("-mp", "--model-path", type=str) parser_infer.add_argument("-s", "--seed", type=int) @@ -131,6 +139,7 @@ def setup_parser() -> ArgumentParser: parser_infer.add_argument( "--action-selection", choices=("deterministic", "stochastic") ) + parser_infer.add_argument("--task", choices=("policy", "forecast")) parser_infer.add_argument("--dataset", type=str) parser_infer.add_argument("--part", type=str) parser_infer.add_argument("--model-interface", type=str) diff --git a/src/sequifier/templates/infer-policy.yaml b/src/sequifier/templates/infer-policy.yaml index 6b1f44a5..f14b468a 100644 --- a/src/sequifier/templates/infer-policy.yaml +++ b/src/sequifier/templates/infer-policy.yaml @@ -1,7 +1,7 @@ project_root: . model_path: models/policy/model.pt data_path: PLEASE_FILL -model_type: policy +model_type: generative read_format: csv write_format: csv device: cpu From 6964c10a0f077f1b1726cdb04dc901866bca7ed4 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 11 Nov 2026 10:25:10 +0100 Subject: [PATCH 26/71] shorten --- .gitattributes | 1 - .gitignore | 3 + .pre-commit-config.yaml | 9 - MANIFEST.in | 1 + documentation/config-capability-inventory.md | 648 ---- documentation/configs/infer.md | 117 +- .../configs/import-policy-sequences.yaml | 13 +- .../configs/rollout-ppo-toy.yaml | 3 +- .../example_inputs/configs/train-bc-toy.yaml | 58 +- .../configs/train-iql-from-table.yaml | 75 +- .../example_inputs/configs/train-iql-toy.yaml | 62 +- .../configs/train-multiphase-control.yaml | 169 +- .../configs/train-policy-from-table.yaml | 65 +- .../configs/train-return-conditioned.yaml | 69 +- documentation/learning-architecture.md | 32 + documentation/policy-learning.md | 605 +-- ...blic-config-surface-implementation-plan.md | 368 -- documentation/rollout.md | 227 +- documentation/supervised-policy-learning.md | 244 +- .../supervised-policy-unification-plan.md | 236 -- hooks/consolidate_docs.py | 47 - setup.py | 21 + src/sequifier/artifacts/model_config.py | 41 +- src/sequifier/artifacts/model_export.py | 7 +- src/sequifier/artifacts/phase.py | 60 + src/sequifier/config/infer_config.py | 9 +- src/sequifier/config/inference_common.py | 44 + src/sequifier/config/policy_infer_config.py | 48 +- src/sequifier/config/training_resolution.py | 73 +- src/sequifier/consolidated-docs.md | 3384 ----------------- src/sequifier/docs.py | 27 +- src/sequifier/io/actions.py | 71 +- src/sequifier/model/execution.py | 86 + src/sequifier/model/factory.py | 11 +- src/sequifier/rl/batching.py | 24 + src/sequifier/rl/checkpoint.py | 3 +- src/sequifier/rl/dpo.py | 11 +- src/sequifier/rl/grpo_update.py | 28 +- src/sequifier/rl/iql.py | 25 +- src/sequifier/rl/joint_distribution.py | 26 +- src/sequifier/rl/learner.py | 4 +- src/sequifier/rl/ppo_update.py | 29 +- src/sequifier/rl/run_checkpoint.py | 5 +- src/sequifier/rl/run_state.py | 31 +- src/sequifier/rl/runtime.py | 70 +- src/sequifier/rl/training.py | 17 +- src/sequifier/rollout/coordinator.py | 49 +- src/sequifier/templates/train-dpo.yaml | 61 +- src/sequifier/templates/train-grpo.yaml | 80 +- src/sequifier/templates/train-iql.yaml | 58 +- .../train-policy-from-supervised.yaml | 30 +- src/sequifier/templates/train-policy.yaml | 65 +- src/sequifier/templates/train-ppo.yaml | 92 +- .../templates/train-supervised-to-policy.yaml | 108 +- src/sequifier/templates/train-supervised.yaml | 66 +- src/sequifier/training/coordinator.py | 89 +- src/sequifier/trajectory/recording.py | 48 + 57 files changed, 1272 insertions(+), 6681 deletions(-) delete mode 100644 .gitattributes create mode 100644 MANIFEST.in delete mode 100644 documentation/config-capability-inventory.md create mode 100644 documentation/learning-architecture.md delete mode 100644 documentation/public-config-surface-implementation-plan.md delete mode 100644 documentation/supervised-policy-unification-plan.md delete mode 100644 hooks/consolidate_docs.py create mode 100644 setup.py create mode 100644 src/sequifier/artifacts/phase.py delete mode 100644 src/sequifier/consolidated-docs.md create mode 100644 src/sequifier/model/execution.py create mode 100644 src/sequifier/rl/batching.py create mode 100644 src/sequifier/trajectory/recording.py diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 4f392e96..00000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -src/sequifier/consolidated-docs.md linguist-generated=true diff --git a/.gitignore b/.gitignore index c1667a2d..b5c6f865 100644 --- a/.gitignore +++ b/.gitignore @@ -181,3 +181,6 @@ cython_debug/ #.idea/ .aider* + +# Documentation is assembled into wheels by the build command. +src/sequifier/consolidated-docs.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 028352ec..fe7ebc04 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,12 +57,3 @@ repos: files: ^(pyproject\.toml|.github/workflows/docs.yml)$ pass_filenames: false additional_dependencies: [tomli] - - - repo: local - hooks: - - id: consolidate-docs - name: Consolidate configuration documentation - entry: python hooks/consolidate_docs.py - language: python - files: ^(README\.md|documentation/configs/(preprocess|train|infer|hyperparameter-search)\.md|documentation/(policy-learning|supervised-policy-learning|rollout)\.md|documentation/commands/visualize-training\.md|documentation/training/multi-gpu-training\.md|src/sequifier/consolidated-docs\.md)$ - pass_filenames: false diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..bc7ac30e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include documentation *.md diff --git a/documentation/config-capability-inventory.md b/documentation/config-capability-inventory.md deleted file mode 100644 index db93d1e8..00000000 --- a/documentation/config-capability-inventory.md +++ /dev/null @@ -1,648 +0,0 @@ -# Configuration capability inventory - -Capability baseline: `082225833e1d744e662d094008bcc44268ab868a`. Layout reference: `2d01d407`. - -The phase/route redesign supersedes baseline placements below. Training batching -and optimization belong to phases, inference validates the selected model route, -and rollout declarations use `environment`. See the current -[training](configs/train.md), [inference](configs/infer.md), and -[phase/route tests](../../sequifier-test/local/unit/test_phase_route_contracts.py). - -This inventory records every directly declared field in the baseline run, policy, inference, source, rollout and recovery models. Shared component behavior remains owned by the existing component models. Internal facts have resolved owners; duplicate routing spellings have one authored owner. Test references below are acceptance coverage, not support for old config syntax. - -## `train_config.GlobalTrainingSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `read_format` | global_training.read_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `training_objective` | global_training.training_objective | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `forward_attention` | global_training.forward_attention | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `context_length` | global_training.context_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `target_offset` | global_training.target_offset | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `window_stride` | global_training.window_stride | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `inference_batch_size` | global_training.inference_batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `batch_size` | training_plan.phases[].batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `accumulation_steps` | training_plan.phases[].accumulation_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `learning_rate` | training_plan.phases[].optimization..learning_rate | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `optimizer` | training_plan.phases[].optimization..optimizer | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `scheduler` | training_plan.phases[].optimization..scheduler | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `scheduler_step_on` | training_plan.phases[].optimization..scheduler_step_on | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `reset_optimization_on_phase` | training_plan.phases[].optimizer_state (reset by default; validated continuation) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `gradient_clip` | training_plan.phases[].optimization..gradient_clip | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `bert_spec` | global_training.bert_spec | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `next_occurrence_config` | global_training.next_occurrence_config | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `device_max_concat_length` | global_training.device_max_concat_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `log_interval` | global_training.log_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `early_stopping_epochs` | global_training.early_stopping_epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `save_interval_epochs` | global_training.save_interval_epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `save_latest_interval_minutes` | global_training.save_latest_interval_minutes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `save_interval_minutes` | global_training.save_interval_minutes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `save_interval_batches` | global_training.save_interval_batches | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `save_interval_val_loss` | global_training.save_interval_val_loss | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `calculate_validation_loss_on_initialization` | global_training.calculate_validation_loss_on_initialization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `resume` | global_training.resume | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `enforce_determinism` | global_training.enforce_determinism | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `distributed` | global_training.distributed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `load_full_data_to_ram` | global_training.load_full_data_to_ram | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_ram_gb` | global_training.max_ram_gb | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `world_size` | global_training.world_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `num_workers` | global_training.num_workers | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `backend` | global_training.backend | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `layer_type_dtypes` | global_training.layer_type_dtypes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `layer_autocast` | global_training.layer_autocast | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `data_parallelism` | global_training.data_parallelism | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `fsdp_cpu_offload` | global_training.fsdp_cpu_offload | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `torch_compile` | global_training.torch_compile | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `float32_matmul_precision` | global_training.float32_matmul_precision | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.ModelInterfaceSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `input_columns` | model.interfaces..input_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `policy` | policy_bindings. | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `prediction_distributions` | model.interfaces..prediction_distributions | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `static_conditioning` | model.interfaces..static_conditioning | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `preconditioning_image_encoder` | model.interfaces..preconditioning_image_encoder | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `target_columns` | model.interfaces..target_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `categorical_hashing` | model.interfaces..categorical_hashing | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `categorical_decoder_special_tokens` | model.interfaces..categorical_decoder_special_tokens | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `feature_layout` | model.interfaces..feature_layout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `ingestion` | model.interfaces..ingestion | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `decoder` | model.interfaces..decoder | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.ModelSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `backbone` | backbone | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `interfaces` | interfaces | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## Policy run fields (authored through `SequifierConfig`) - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_name` | model_name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `enforce_determinism` | global_training.enforce_determinism | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model` | model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `policy_binding` | policy_bindings. | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `execution_model` | resolved model execution | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `policy_interface` | dataset/phase.model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `context_length` | global_training.context_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `units` | policy_bindings..units | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `initialize_from` | initialize_from | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `normalization` | global_training.normalization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `sources` | dataset_training..parts. (sequence) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `preferences` | dataset_training..parts. (preferences) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `phases` | training_plan.phases | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `validation_fraction` | global_training.validation_fraction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `episode_cache_size` | global_training.episode_cache_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `resume` | global_training.resume | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_path` | model_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `report_path` | report_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `checkpoint_path` | global_training.resume.checkpoint_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `phase_directory` | phase_directory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.DatasetPartSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `metadata_config_path` | metadata_config_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `image_path` | image_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `preconditioning_image_path` | preconditioning_image_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `file_order` | file_order | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.DatasetFreezingSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `backbone` | backbone | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `ingestion` | ingestion | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `ingestion_adapter` | ingestion_adapter | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `decoder` | decoder | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.DatasetTrainingSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `model_interface` | model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `parts` | parts | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `criterion` | criterion | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `class_weights` | class_weights | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `loss_weights` | loss_weights | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `class_share_log_columns` | class_share_log_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `freeze` | freeze | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.TrainingSourceSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `weight` | weight | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `batches_per_selection` | batches_per_selection | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.TrainingPhaseSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `algorithm` | algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `name` | name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `epochs` | epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `mode` | mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `selection` | selection | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `sources` | sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.TrainingPlanModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `curriculum_training` | curriculum_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `curriculum_column` | curriculum_column | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `phases` | phases | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.EvaluationMonitorSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `metric` | metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `mode` | mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.EvaluationSpecModel` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `sources` | sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `monitor` | monitor | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `train_config.SequifierConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_name` | model_name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `global_training` | global_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model` | model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `dataset_training` | dataset_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `training_plan` | training_plan | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `evaluation` | evaluation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `export_generative_model` | export_generative_model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `export_embedding_model` | export_embedding_model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `embedding_layer_names` | embedding_layer_names | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `export_onnx` | export_onnx | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `export_pt` | export_pt | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `export_with_dropout` | export_with_dropout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## Mixed run fields (authored through `SequifierConfig`) - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `run_type` | removed dispatch hint; phase algorithms determine execution | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_name` | model_name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model` | model | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `initialize_from` | initialize_from | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `global_training` | global_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `dataset_training` | dataset_training | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `evaluation` | evaluation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `policy_interface` | dataset/phase.model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `policy_bindings` | policy_bindings | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `sources` | dataset_training..parts. (sequence) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `preferences` | dataset_training..parts. (preferences) | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `units` | policy_bindings..units | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `phases` | training_plan.phases | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `resume` | global_training.resume | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `validation_fraction` | global_training.validation_fraction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `episode_cache_size` | global_training.episode_cache_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `checkpoint_path` | global_training.resume.checkpoint_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_path` | model_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `report_path` | report_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `phase_directory` | phase_directory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `export_onnx` | export_onnx | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_config.ReturnDefinition` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `gamma` | gamma | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `scale` | scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `finite_horizon` | finite_horizon | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_config.ActionFieldConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `type` | type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `values` | values | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `scale` | scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `learn_scale` | learn_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `min_scale` | min_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_scale` | max_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `lower` | lower | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `upper` | upper | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `active_when` | active_when | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `inactive_value` | inactive_value | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `atoms` | atoms | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_config.PolicySchemaConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `alignment` | resolved from explicit model inputs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `observation_columns` | model input columns / resolved binding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `static_columns` | model static conditioning / resolved binding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `categorical_columns` | metadata types / resolved binding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `observation_n_classes` | metadata.n_classes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `static_n_classes` | metadata.n_classes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `id_maps` | metadata.id_maps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `categorical_encoding` | resolved fixed codec convention | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `action_column` | named actions key | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `action_values` | actions..values | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `actions` | policy_bindings..actions | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `action_order` | resolved decoder/activity dependency order | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `decision_position` | policy_bindings..decision_position | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `previous_action_inputs` | policy_bindings..previous_action_inputs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `initial_inputs` | policy_bindings..initial_inputs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `input_schema` | resolved ExecutionSchema | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `input_statistics` | metadata / resolved input statistics | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `input_encoding` | dataset part encoding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `return_conditioned` | policy_bindings..return_conditioned | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `initial_desired_return` | policy_bindings..initial_desired_return | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `returns` | policy_bindings..returns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_config.CategoricalPolicyConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `context_length` | resolved model context | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `interface_name` | resolved interface name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.PhaseConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `name` | training_plan.phases[].name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `handoff` | training_plan.phases[].handoff | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_interface` | training_plan.phases[].model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `freeze` | training_plan.phases[].freeze | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `representation` | resolved binding identity | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `optimization` | training_plan.phases[].optimization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `evaluation_interval` | training_plan.phases[].evaluation_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `checkpoint_interval` | training_plan.phases[].checkpoint_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.BatchedPhaseConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `batch_size` | training_plan.phases[].batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `accumulation_steps` | training_plan.phases[].accumulation_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.OfflinePhaseConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `sources` | training_plan.phases[].sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `validation_sources` | training_plan.phases[].evaluation.sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.PolicyImitationPhaseConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `epochs` | training_plan.phases[].epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `monitor` | training_plan.phases[].evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `direction` | training_plan.phases[].evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.IQLPhaseConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `monitor` | training_plan.phases[].evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `direction` | training_plan.phases[].evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `name` | training_plan.phases[].name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `updates` | training_plan.phases[].updates | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `batch_size` | training_plan.phases[].batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `gamma` | training_plan.phases[].gamma | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `expectile` | training_plan.phases[].expectile | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `inverse_temperature` | training_plan.phases[].inverse_temperature | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_weight` | training_plan.phases[].max_weight | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `target_update_rate` | training_plan.phases[].target_update_rate | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `evaluation_interval` | training_plan.phases[].evaluation_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `checkpoint_interval` | training_plan.phases[].checkpoint_interval | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.OnlineEnvironmentConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `factory` | factory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `config` | config | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `protocol_version` | fixed by adapter type; checked against implementation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `implementation_id` | implementation_id | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `units` | units | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.FrozenNormalizationConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `statistics` | statistics | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.EvaluationEpisodesConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `environment` | environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seeds` | seeds | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_episode_steps` | max_episode_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `timeout_seconds` | timeout_seconds | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `selection` | selection | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.OnlineEvaluationConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `metric` | evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `direction` | evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.PolicyResumeConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `policy` | global_training.resume.policy | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `mode` | global_training.resume.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.PPOPhaseConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `name` | training_plan.phases[].name | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `environment` | training_plan.phases[].environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `evaluation` | training_plan.phases[].evaluation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `normalization` | training_plan.phases[].normalization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed_policy` | training_plan.phases[].seed_policy | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `total_env_steps` | training_plan.phases[].total_env_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `rollout_length` | training_plan.phases[].rollout_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `allow_partial_rollout` | training_plan.phases[].allow_partial_rollout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `optimization_epochs` | training_plan.phases[].optimization_epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `minibatch_size` | training_plan.phases[].minibatch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `accumulation_steps` | training_plan.phases[].accumulation_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `gamma` | training_plan.phases[].gamma | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `gae_lambda` | training_plan.phases[].gae_lambda | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `clip_epsilon` | training_plan.phases[].clip_epsilon | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `value_clip` | training_plan.phases[].value_clip | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `value_coefficient` | training_plan.phases[].value_coefficient | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `entropy_coefficient` | training_plan.phases[].entropy_coefficient | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `target_kl` | training_plan.phases[].target_kl | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `normalize_advantages` | training_plan.phases[].normalize_advantages | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `reward_scale` | training_plan.phases[].reward_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `archive_directory` | training_plan.phases[].archive_directory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `archive_scores` | training_plan.phases[].archive_scores | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_buffer_bytes` | training_plan.phases[].max_buffer_bytes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.ReferenceConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `path` | path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.DPOPhaseConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `preferences` | training_plan.phases[].preferences | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `validation_preferences` | training_plan.phases[].evaluation.sources | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `updates` | training_plan.phases[].updates | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `beta` | training_plan.phases[].beta | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `scoring_chunk_size` | training_plan.phases[].scoring_chunk_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `reference` | training_plan.phases[].reference | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `monitor` | training_plan.phases[].evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `direction` | training_plan.phases[].evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.GRPOEvaluationConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `metric` | evaluation.monitor.metric | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `direction` | evaluation.monitor.mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_plan.GRPOPhaseConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `scoring_chunk_size` | training_plan.phases[].scoring_chunk_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `algorithm` | training_plan.phases[].algorithm | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `environment` | training_plan.phases[].environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `evaluation` | training_plan.phases[].evaluation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `normalization` | training_plan.phases[].normalization | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `reference` | training_plan.phases[].reference | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `total_groups` | training_plan.phases[].total_groups | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `groups_per_update` | training_plan.phases[].groups_per_update | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `group_size` | training_plan.phases[].group_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_steps` | training_plan.phases[].max_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_env_steps` | training_plan.phases[].max_env_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_buffer_bytes` | training_plan.phases[].max_buffer_bytes | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `optimization_epochs` | training_plan.phases[].optimization_epochs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `clip_epsilon` | training_plan.phases[].clip_epsilon | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `advantage_epsilon` | training_plan.phases[].advantage_epsilon | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `kl_coefficient` | training_plan.phases[].kl_coefficient | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `old_policy_target_kl` | training_plan.phases[].old_policy_target_kl | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `reward_scale` | training_plan.phases[].reward_scale | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `archive_directory` | training_plan.phases[].archive_directory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_train_config.PolicyTableColumns` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `sequence_id` | dataset part columns.sequence_id | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `item_position` | dataset part columns.item_position | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `observations` | dataset part columns.observations | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `static_inputs` | dataset part columns.static_inputs | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `action` | dataset part columns.action | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `actions` | dataset part columns.actions | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `previous_actions` | dataset part columns.previous_actions | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `reward` | dataset part columns.reward | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `terminated` | dataset part columns.terminated | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `truncated` | dataset part columns.truncated | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `previous_action` | dataset part columns.previous_action | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `successors` | dataset part columns.successors | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `bootstrap_allowed` | dataset part columns.bootstrap_allowed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `group` | dataset part columns.group | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `episode_id` | dataset part columns.episode_id | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_train_config.SequenceSourceConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `project_root` | resolved root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `units` | dataset_training..parts..units | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `paths` | dataset_training..parts..paths | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `manifest` | dataset_training..parts..manifest | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `format` | dataset_training..parts..format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `columns` | dataset_training..parts..columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `episode_end` | dataset_training..parts..episode_end | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `episode_namespace` | dataset_training..parts..episode_namespace | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `metadata_config_path` | dataset_training..parts..metadata_config_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `encoding` | dataset_training..parts..encoding | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `static_data_path` | dataset_training..parts..static_data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `split_index` | dataset_training..parts..split_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `action_semantics` | dataset_training..parts..action_semantics | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_infer_config.PolicyRolloutConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `environment` | rollout.environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `trajectory_path` | rollout.trajectory_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed_index` | rollout.seed_row_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_episode_steps` | rollout.max_episode_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_infer_config.PolicyInferenceConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_path` | model_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `execution_mode` | model_type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_type` | model_type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_interface` | model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `data_path` | data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `read_format` | read_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `write_format` | write_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `output_path` | output_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `output_probabilities` | output_probabilities | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `inference_batch_size` | inference_batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `enforce_determinism` | runtime deterministic and policy selection | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `desired_return` | desired_return | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `desired_return_column` | desired_return_column | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `previous_action_column` | previous_action_columns. | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `previous_action_columns` | previous_action_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `sequence_id_column` | sequence_id_column | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `item_position_column` | item_position_column | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `rollout` | rollout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `autoregressive` | explicit rollout for environment interaction | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `generation_steps` | rollout.max_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `rollout_config.InputBinding` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `input` | input | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `source` | source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `key` | key | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `rollout_config.AdapterConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `factory` | factory | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `config` | config | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `protocol_version` | fixed by adapter type; checked against implementation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `rollout_config.RolloutConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `mode` | rollout.type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `environment` | rollout.environment | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `bindings` | rollout.bindings | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `trajectory_path` | rollout.trajectory_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed_index` | rollout.seed_window_index | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `max_episode_steps` | rollout.max_episode_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `sequence_import_config.SequenceImportConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `operation` | sequence import operation | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `project_root` | sequence import project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `source` | sequence import source | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `policy` | policy_binding + source/model metadata | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `output_path` | sequence import output_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `episodes_per_shard` | sequence import episodes_per_shard | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `policy_optimization.OptimizationConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `optimizer` | optimizer | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `scheduler` | scheduler | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `learning_rate` | learning_rate | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `gradient_clip` | gradient_clip | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `infer_config._InferenceConfigBase` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `execution_mode` | execution_mode | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `project_root` | project_root | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `preprocessing_data_path` | preprocessing_data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `metadata_config_path` | metadata_config_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_path` | model_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_type` | model_type | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `training_objective` | training_objective | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `data_path` | data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `training_config_path` | training_config_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `dataset` | dataset | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `part` | part | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `model_interface` | model_interface | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `read_format` | read_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `write_format` | write_format | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `input_columns` | input_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `static_columns` | static_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `static_features_path` | static_features_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `image_path` | image_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `preconditioning_image_path` | preconditioning_image_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `target_columns` | target_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `column_data_types` | column_data_types | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `target_column_types` | target_column_types | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `enforce_determinism` | deterministic | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `output_probabilities` | output_probabilities | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `decode_categories` | decode_categories | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `seed` | seed | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `device` | device | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `context_length` | context_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `target_offset` | target_offset | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `window_stride` | window_stride | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `prediction_length` | prediction_length | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `decoder_support` | decoder_support | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `prediction_alignment_steps` | prediction_alignment_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `inference_batch_size` | inference_batch_size | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `sample_from_distribution_columns` | sample_from_distribution_columns | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `infer_with_dropout` | infer_with_dropout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `rollout` | rollout | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `autoregressive` | autoregressive | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `generation_steps` | generation_steps | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `infer_config.InferenceConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `data_path` | data_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `column_data_types` | column_data_types | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `target_column_types` | target_column_types | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## `components.ResumeConfig` - -| Baseline field | New owner / behavior | Acceptance coverage | -| --- | --- | --- | -| `policy` | global_training.resume.policy | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | -| `checkpoint_path` | global_training.resume.checkpoint_path | ../sequifier-test/local/unit/test_public_config.py; ../sequifier-test/local/integration/test_public_config_execution.py | - -## Operation and combination coverage - -In addition to field coverage, acceptance tests exercise same-design recovery, native supervised execution and initialization, mixed supervised/imitation/supervised handoff, single categorical and return-conditioned policies, IQL/PPO/DPO/GRPO updates, raw/manifest/prepared sources, explicit and automatic holdouts, custom inference outputs and probabilities, and both environment protocols. Model component, distribution, HPO and source-import cases are covered in their focused suites. - -Completion requires all referenced tests to pass and every supported baseline combination to retain an execution path. Rejection of already supported combinations is a regression, even if individual fields still parse. diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index ad5dbc05..41eaf4fc 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -269,85 +269,48 @@ ORT installation. Outputs stay indexed by outer item coordinates. ## Policy model interfaces -`model_type` selects the authored route before artifact resolution. `generative` -and `embedding` use forecast schemas; `policy` uses `PolicyInferenceConfig` -directly. Unknown and cross-route fields fail validation even when set to their -default values. CLI overrides and Python configuration loaders follow the same -selection rules. +With `model_type: generative`, the selected artifact interface determines the +default task. `task: policy` requires one policy-bound portable PT artifact; +`task: forecast` requests ordinary predictions, including from a policy-bound +interface with a supported forecast contract. Select `model_interface` when +ambiguous. Legacy `model_type: policy` forces policy execution and conflicts with +`task: forecast`. Embedding rejects `task`. The CLI accepts `--task policy|forecast`. -Policy inference requires one portable PT artifact and one unambiguous policy -interface. Supply raw CSV/Parquet observation rows; history alignment, vocabulary -and normalization come from that artifact. Policy supports the common model path, -interface, data path, formats, device, seed, `enforce_determinism`, batch size fields, plus: +Policy input is raw CSV/Parquet observation rows. History alignment, codecs and +normalization come from the artifact. Common fields cover model/interface/data +selection, read/write formats, device, seed, `enforce_determinism` and batch size. | Policy field | Default / meaning | | --- | --- | -| `output_path` | null; write policy predictions here instead of the generated output path | -| `action_selection` | `deterministic`; choose the policy mode. `stochastic` samples. | -| `desired_return` | null; fixed return budget for a return-conditioned policy | +| `output_path` | null; override the generated prediction destination | +| `action_selection` | `deterministic`; sequential mode/location choices, or `stochastic` sampling | +| `desired_return` | null; fixed budget for return-conditioned policies | | `desired_return_column` | `desired_return`; per-row budget when no fixed budget is supplied | -| `previous_action_columns` | {}; input-column mapping for recorded previous actions | -| `sequence_id_column`, `item_position_column` | `sequenceId`, `itemPosition`; order raw history rows | -| `output_probabilities` | false; include policy distribution outputs | -| `rollout` | null; execute the policy in a version-3 online environment | - -Policy assertions may specify `metadata_config_path`, `input_columns`, -`static_columns`, `target_columns`, `column_data_types`, `target_column_types`, -`training_objective`, `context_length`, `target_offset`, `prediction_length`, and -`decoder_support`. Every supplied assertion must match the selected artifact. -Forecast fields `sample_from_distribution_columns`, `infer_with_dropout`, -`decode_categories`, dataset/part selection, image path overrides, -`autoregressive`, and `generation_steps` are rejected on policy requests. -Policy-only return/action/history options and the policy-only `output_path` -are rejected on forecast requests. Embedding requests also reject generative -sampling, category decoding, probability output, autoregression and rollout fields; -dropout remains supported for embeddings. - -```yaml -project_root: . -model_type: generative -model_path: models/policy/model.pt -model_interface: actor -data_path: observations.csv -read_format: csv -device: cpu -seed: 1010 -enforce_determinism: true -action_selection: stochastic -rollout: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - units: {} - max_steps: 8 - seed_row_index: 0 - trajectory_path: outputs/trajectory.json -``` - -Rollout schemas are selected by the inference route; do not declare `rollout.type`. -Forecast rollout requires `bindings` and a version-1 environment. Policy rollout -rejects bindings and requires the distinct version-3 environment protocol, -including implementation identity and matching units. Both use `max_steps` and -optional `max_episode_steps`. Forecast `seed_window_index` counts prepared -windows; policy `seed_row_index` counts raw input rows, each zero-based in input -order. The legacy `seed_index` spelling is accepted within each route, but -providing both spellings is an error. Trajectory output metadata retains its -public `seed_index` key and existing units. - -`enforce_determinism` controls deterministic execution, independently of action -selection and seed. A seeded stochastic policy can reproduce its sampled sequence -under the same supported runtime, device and input ordering. Online evaluation -uses `action_selection` with the same meaning; its execution determinism is the -training run's `global_training.enforce_determinism`. See the -[environment protocols](../rollout.md) and [policy guide](../policy-learning.md). - - -With `model_type: generative`, a policy-bound interface defaults to policy -inference; other interfaces retain forecast behavior and supported formats. -Select `model_interface` when ambiguous. Optional `task: policy` requires one -policy-bound PT artifact; `task: forecast` requests forecasts, including from a -policy-bound interface with a supported forecast contract. Legacy -`model_type: policy` forces policy inference and conflicts with `task: forecast`. -Embedding rejects `task`; each route retains its own option and rollout schema. -The CLI accepts `--task policy|forecast`. +| `previous_action_columns` | {}; recorded lagged-action column mappings for fragments | +| `sequence_id_column`, `item_position_column` | `sequenceId`, `itemPosition`; raw history ordering | +| `output_probabilities` | false; include joint log probability and conditional categorical probabilities | +| `rollout` | null; execute in a version-3 online environment instead of writing prediction rows | + +Optional assertions `metadata_config_path`, `input_columns`, `static_columns`, +`target_columns`, `column_data_types`, `target_column_types`, `training_objective`, +`context_length`, `target_offset`, `prediction_length` and `decoder_support` must +match the selected artifact. Unknown/cross-route options fail even at default values. +Policy rejects external per-column sampling, dropout, category decoding, +dataset/part selection, image overrides, autoregression and `generation_steps`. +Forecast rejects policy-only output/return/action/history options; embeddings also +reject generative sampling, decoding, probabilities, autoregression and rollout. + +Policy rollout declares `environment: {factory: module:factory, config: {}, +implementation_id: ..., units: {}}`, required `max_steps`, `seed_row_index` and +`trajectory_path`, plus optional `max_episode_steps`. It rejects forecast `bindings`. +Forecast rollout uses the distinct version-1 environment, requires bindings and +selects `seed_window_index`. Neither accepts `rollout.type`; legacy `seed_index` +aliases the route-specific index, but combining spellings fails. Both indices are +zero-based and validated before environment creation. Trajectory metadata retains +its existing `seed_index` key. See the [rollout guide](../rollout.md) and +[policy template](../../src/sequifier/templates/infer-policy.yaml). + +Execution determinism, action selection and seed are independent. A seeded +stochastic policy can reproduce its actions under the same supported runtime, +device and input ordering. Online evaluation uses the same action-selection +meaning and the training run's execution-determinism setting. diff --git a/documentation/example_inputs/configs/import-policy-sequences.yaml b/documentation/example_inputs/configs/import-policy-sequences.yaml index 3df68100..3b99ed68 100644 --- a/documentation/example_inputs/configs/import-policy-sequences.yaml +++ b/documentation/example_inputs/configs/import-policy-sequences.yaml @@ -1,19 +1,12 @@ operation: import_sequences project_root: . source: - paths: - - data/demonstrations.csv + paths: [data/demonstrations.csv] columns: - observations: - position: position - velocity: velocity + observations: {position: position, velocity: velocity} static_inputs: {} output_path: data/imported-demonstrations episodes_per_shard: 64 policy_binding: actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index 07b3cc55..2f66632d 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -12,8 +12,7 @@ rollout: max_steps: 8 environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 + config: {horizon: 8} implementation_id: sequifier.categorical-control.v1 enforce_determinism: true action_selection: deterministic diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml index 47535a44..4ef969d5 100644 --- a/documentation/example_inputs/configs/train-bc-toy.yaml +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -8,58 +8,36 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: - n_heads: 4 - feed_forward: - dim: 128 + attention: {n_heads: 4} + feed_forward: {dim: 128} positional_encoding_scope: global interfaces: actor: - input_columns: - - position - - __policy_previous_action - target_columns: - - action + input_columns: [position, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - position: 2 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] -global_training: - context_length: 16 - validation_fraction: 0.2 + feature_embedding_dims: {position: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} +global_training: {context_length: 16, validation_fraction: 0.2} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} dataset_training: demonstrations: parts: - recorded: - type: sequence - paths: - - data/offline-transitions.csv + recorded: {type: sequence, paths: [data/offline-transitions.csv]} model_interface: actor training_plan: phases: - - name: behavior_cloning - algorithm: policy_imitation - sources: - - source: demonstrations.recorded - epochs: 30 - batch_size: 64 + - name: behavior_cloning + algorithm: policy_imitation + sources: + - {source: demonstrations.recorded} + epochs: 30 + batch_size: 64 diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index 342066e2..74abfb5b 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -8,56 +8,35 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: - n_heads: 4 - feed_forward: - dim: 128 + attention: {n_heads: 4} + feed_forward: {dim: 128} positional_encoding_scope: global interfaces: actor: - input_columns: - - position - - velocity - - __policy_previous_action - target_columns: - - action + input_columns: [position, velocity, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - position: 1 - velocity: 1 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {position: 1, velocity: 1, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 16 validation_fraction: 0.2 - resume: - policy: if_exists + resume: {policy: if_exists} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} dataset_training: demonstrations: parts: recorded: type: sequence - paths: - - data/offline-transitions.parquet + paths: [data/offline-transitions.parquet] columns: sequence_id: sequenceId item_position: itemPosition @@ -66,22 +45,20 @@ dataset_training: terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: - position: next_position - velocity: next_velocity + successors: {position: next_position, velocity: next_velocity} model_interface: actor training_plan: phases: - - name: offline_improvement - algorithm: iql - sources: - - source: demonstrations.recorded - updates: 10000 - batch_size: 64 - gamma: 0.99 - expectile: 0.7 - inverse_temperature: 3.0 - max_weight: 100.0 - target_update_rate: 0.005 - evaluation_interval: 100 - checkpoint_interval: 100 + - name: offline_improvement + algorithm: iql + sources: + - {source: demonstrations.recorded} + updates: 10000 + batch_size: 64 + gamma: 0.99 + expectile: 0.7 + inverse_temperature: 3.0 + max_weight: 100.0 + target_update_rate: 0.005 + evaluation_interval: 100 + checkpoint_interval: 100 diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml index 61d1d138..35bcb94c 100644 --- a/documentation/example_inputs/configs/train-iql-toy.yaml +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -9,69 +9,49 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: - n_heads: 4 - feed_forward: - dim: 128 + attention: {n_heads: 4} + feed_forward: {dim: 128} positional_encoding_scope: global interfaces: actor: - input_columns: - - position - - __policy_previous_action - target_columns: - - action + input_columns: [position, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - position: 2 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {position: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 16 validation_fraction: 0.2 - resume: - policy: if_exists + resume: {policy: if_exists} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} dataset_training: demonstrations: parts: recorded: type: sequence - paths: - - data/offline-transitions.csv + paths: [data/offline-transitions.csv] columns: reward: reward terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: - position: next_position + successors: {position: next_position} model_interface: actor training_plan: phases: - - name: offline_improvement - algorithm: iql - sources: - - source: demonstrations.recorded - updates: 10000 - batch_size: 64 - evaluation_interval: 100 - checkpoint_interval: 100 + - name: offline_improvement + algorithm: iql + sources: + - {source: demonstrations.recorded} + updates: 10000 + batch_size: 64 + evaluation_interval: 100 + checkpoint_interval: 100 diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index fa4b4447..546a5fa8 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -8,141 +8,104 @@ model: dim_model: 32 num_layers: 1 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 64 + attention: {n_heads: 4} + feed_forward: {dim: 64} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: - - cue - - __policy_previous_action - target_columns: - - action + input_columns: [cue, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - cue: 2 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 4 - normalization: - source: identity + normalization: {source: identity} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} dataset_training: decisions: parts: default: type: sequence - paths: - - data/control-decisions.csv + paths: [data/control-decisions.csv] columns: episode_id: episodeId reward: reward terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: - cue: next_cue + successors: {cue: next_cue} group: contextId model_interface: actor rankings: parts: - default: - type: preferences - manifest: data/control-preferences.json + default: {type: preferences, manifest: data/control-preferences.json} training_plan: phases: - - name: imitation - algorithm: policy_imitation - sources: - - source: decisions.default - epochs: 4 - - name: offline_improvement - algorithm: iql - sources: - - source: decisions.default - updates: 100 - - name: preferences - algorithm: dpo - preferences: - source: rankings.default - updates: 100 - beta: 0.1 - - name: group_improvement - algorithm: grpo - total_groups: 32 - groups_per_update: 2 - group_size: 4 - max_steps: 8 - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 - implementation_id: sequifier.categorical-control.v1 - evaluation: + - name: imitation + algorithm: policy_imitation + sources: + - {source: decisions.default} + epochs: 4 + - name: offline_improvement + algorithm: iql + sources: + - {source: decisions.default} + updates: 100 + - name: preferences + algorithm: dpo + preferences: {source: rankings.default} + updates: 100 + beta: 0.1 + - name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_update: 2 + group_size: 4 + max_steps: 8 environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 + config: {horizon: 8} implementation_id: sequifier.categorical-control.v1 - seeds: - - 17 - - 29 - - 43 - - 71 - max_episode_steps: 8 - timeout_seconds: 30 - monitor: - metric: score_mean - mode: max - - name: ppo_improvement - algorithm: ppo - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 - implementation_id: sequifier.categorical-control.v1 - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 + monitor: {metric: score_mean, mode: max} + - name: ppo_improvement + algorithm: ppo environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 + config: {horizon: 8} implementation_id: sequifier.categorical-control.v1 - seeds: - - 17 - - 29 - - 43 - - 71 - max_episode_steps: 8 - timeout_seconds: 30 + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml index 96d5aace..34d85593 100644 --- a/documentation/example_inputs/configs/train-policy-from-table.yaml +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -8,63 +8,38 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: - n_heads: 4 - feed_forward: - dim: 128 + attention: {n_heads: 4} + feed_forward: {dim: 128} positional_encoding_scope: global interfaces: actor: - input_columns: - - position - - velocity - - __policy_previous_action - target_columns: - - action + input_columns: [position, velocity, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - position: 1 - velocity: 1 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] -global_training: - context_length: 16 - validation_fraction: 0.2 + feature_embedding_dims: {position: 1, velocity: 1, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} +global_training: {context_length: 16, validation_fraction: 0.2} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} dataset_training: demonstrations: parts: - recorded: - type: sequence - paths: - - data/demonstrations.parquet + recorded: {type: sequence, paths: [data/demonstrations.parquet]} model_interface: actor training_plan: phases: - - name: behavior_cloning - algorithm: policy_imitation - epochs: 80 - sources: - - source: demonstrations.recorded - batch_size: 64 - optimization: - actor: - learning_rate: 0.005 + - name: behavior_cloning + algorithm: policy_imitation + epochs: 80 + sources: + - {source: demonstrations.recorded} + batch_size: 64 + optimization: + actor: {learning_rate: 0.005} diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index 44e45440..d259e484 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -8,73 +8,46 @@ model: dim_model: 32 num_layers: 2 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 128 - activation: swiglu + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: - - OBSERVATION_COLUMN - - __policy_previous_action - - __policy_return - target_columns: - - action + input_columns: [OBSERVATION_COLUMN, __policy_previous_action, __policy_return] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - OBSERVATION_COLUMN: 1 - __policy_previous_action: 6 - __policy_return: 1 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} model_path: models/policy/actor.pt report_path: outputs/policy/training-report.json global_training: context_length: 4 validation_fraction: 0.2 episode_cache_size: 8 - resume: - policy: never + resume: {policy: never} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: true - returns: - gamma: 1.0 - scale: 1.0 + returns: {gamma: 1.0, scale: 1.0} actions: - action: - type: categorical - values: - - 0.0 - - 1.0 + action: {type: categorical, values: [0.0, 1.0]} dataset_training: demonstrations: parts: - recorded: - type: sequence - paths: - - data/demonstrations.parquet + recorded: {type: sequence, paths: [data/demonstrations.parquet]} model_interface: actor training_plan: phases: - - name: offline_policy - algorithm: policy_imitation - epochs: 80 - sources: - - source: demonstrations.recorded - batch_size: 64 - optimization: - actor: - learning_rate: 0.005 + - name: offline_policy + algorithm: policy_imitation + epochs: 80 + sources: + - {source: demonstrations.recorded} + batch_size: 64 + optimization: + actor: {learning_rate: 0.005} diff --git a/documentation/learning-architecture.md b/documentation/learning-architecture.md new file mode 100644 index 00000000..fff96bdf --- /dev/null +++ b/documentation/learning-architecture.md @@ -0,0 +1,32 @@ +# Learning architecture + +All learning uses `SequifierConfig` and `training_plan.phases`; there is no second +policy architecture or run discriminator. See the [training reference](configs/train.md), +[policy guide](policy-learning.md), [artifact transfer guide](supervised-policy-learning.md), +and [inference reference](configs/infer.md) for the supported configuration surface. + +| Owner | Contract | +| --- | --- | +| Model execution | Named interfaces, forward attention, context geometry, codecs, normalization, input schemas, decoder order and prediction scales. These survive changes of learning objective. | +| Authored run | Model or initialization artifact, datasets, policy bindings, ordered phases, output paths and global execution settings. | +| Phase | Algorithm budget, batching, trainable roles, optimizer/scheduler settings, evaluation, freezing and handoff selection. | +| Sources | Prepared histories, raw sequences and preference manifests with explicit timing, episode boundaries and optional transition capabilities. | +| Native runner | Algorithm-specific sampling, objectives, critics, environment state and in-phase recovery. | +| Coordinator | Dependency validation, incoming/selected artifacts, phase boundaries, receipts and reports. | + +`ModelExecution` exposes execution facts without assigning them dataset ownership. +Compatibility adapters retain existing resolved training objects and artifact formats. +Policy bindings select existing targets; lagged actions and return conditioning are +explicit inputs. Categorical support, real scales/bounds/atoms and conditional activity +have the same meaning during training, collection, replay and inference. + +Historical decisions are reconstructed at their actual observation boundary. +Termination disables bootstrap; truncation preserves the true final successor; +a collection cutoff does not invent either boundary. Training/validation splits +keep connected episode, group and duplicate-content components together. + +Phase handoffs publish selected weights before committing progress. Checkpoints +retain optimizer/sampler/RNG state and validate representation, source, adapter and +reference identities. Mixed plans use native checkpoints inside an atomic run +envelope. Existing checkpoint formats, public configuration aliases, standalone +scaffold templates and portable model formats remain compatibility boundaries. diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index cadac5be..0db92ea7 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -1,449 +1,178 @@ # Policy learning -For supervised checkpoint attachment, mixed real/categorical heads, full model -input schemas, and mixed supervised/policy plans, see the -[unified learning guide](supervised-policy-learning.md). The configuration below -documents the compatible single-categorical authoring surface. - -`sequifier train` accepts one `SequifierConfig` with ordered -`training_plan.phases`. Supervised, imitation, IQL, PPO, DPO and GRPO use the same -model/interface and artifact contracts. A policy binding selects named real or -categorical action heads. See the [training guide](configs/train.md) for phase -ownership, scheduling budgets, optimizer groups and transition validation. -Policy execution currently uses a local CPU/CUDA/MPS process; standalone -supervised training also supports distributed execution. - -## Authoring a run - -Use the [policy template](../src/sequifier/templates/train-policy.yaml), -[PPO template](../src/sequifier/templates/train-ppo.yaml), or -[multiphase example](example_inputs/configs/train-multiphase-control.yaml). -The latter contains imitation → IQL → DPO → GRPO → PPO. Generate its data with: - -```shell -python documentation/example_inputs/policy_control_data.py data -sequifier train --config-path documentation/example_inputs/configs/train-multiphase-control.yaml -``` - -All sources live in `dataset_training..parts.`. Use `type: sequence` -for raw tables or exported sequence manifests, prepared metadata parts for -supervised data, and `type: preferences` with `manifest` for preference pairs. -Phase source references are objects such as `{source: decisions.default}`; -DPO `preferences` uses the same reference shape. Every data-bearing dataset -selects its model interface explicitly when more than one is available. - -Run-wide `context_length`, normalization, validation fraction (0.2), episode -cache size (8), determinism and resume settings live in `global_training`. -`resume` defaults to `{policy: never, mode: reset_environments}`; use `if_exists` -or `required` with a checkpoint path, and `exact` for supported online adapters. -`policy_binding` applies to a single model interface; use `policy_bindings` keyed -by interface name for multiple interfaces. Environment units must match the binding. -No optimizer defaults live in this run-wide block. - -Default outputs are `models//model.pt`, -`outputs/training//report.json`, and -`outputs/training//phases`. The resolved configuration records -role defaults and execution contracts. Fingerprints and checkpoints include -phase settings and validate source, reference and environment identities on resume. - -## Selection and optimization - -Every phase has a unique identifier `name` and `handoff: best | last` (default -`last`). Best selection retains the earliest finite best on ties. The final -update always evaluates. Missing finite evaluation for `best` fails explicitly. -`last` selects the most recently completed optimizer transaction. The final -export uses the last phase's selected actor, with metric, transaction and weight -hash recorded; metrics from different phases are never compared globally. - -Each phase configures trainable roles under `optimization`. Imitation, DPO and -GRPO have `actor`; PPO has `actor` and `value`; IQL has `actor`, `value` and -`critic` (the twin Q networks share that optimizer). Omitted roles use defaults. -For example: - -```yaml -optimization: - actor: {learning_rate: 0.005, optimizer: {name: AdamW}, gradient_clip: 1.0} -``` - -Every role defaults to `learning_rate: 0.0003`, `optimizer: {name: Adam}`, -`scheduler: {name: ConstantLR, factor: 1.0, total_iters: 1}`, and -`gradient_clip: 1.0`. Unknown roles fail validation. - -Imitation, DPO and GRPO share these batching and scheduling fields: - -| Field | Default | -| --- | --- | -| `batch_size`, `accumulation_steps` | 32, 1; DPO counts pairs, GRPO counts whole trajectories | -| `evaluation_interval`, `checkpoint_interval` | 1; optimizer updates for imitation/DPO, completed collection/optimization updates for GRPO | - -Optimizers support Adam, AdamW, SGD and RMSprop. Their constructor arguments -belong alongside `name` (or in `arguments`); `lr` and parameters belong to the -role configuration. Schedulers advance on completed optimizer steps; plateau/compound -schedulers are unsupported. `OneCycleLR.total_steps` must equal the authored -maximum optimizer-step budget. Accumulation weights by the actual number of -examples, including final partial batches. Network stochastic layers are disabled -while gradients remain enabled; likelihood ratios use reproducible scores. - -## Offline sequences and imitation - -Sources accept CSV/Parquet tables, JSON rollout logs, committed sequence -manifests, or scalar prepared histories. A row has current observations and its -executed action; optional rewards/successors describe the resulting transition. -`PolicyTableColumns` exposes explicit observation/static/action/reward/group/ID -mappings. Temporal positions must be contiguous, zero-based at episode starts. -Fragments require a known preceding action. -`columns.action` conflicts with nonempty `columns.actions`; likewise for -`previous_action` and `previous_actions`. Empty plural maps (`{}`) are unused -and may coexist with a singular mapping, including structured actions. Reward-free sequences are ordinary -`SequenceEpisode` objects with absent transition capabilities, rather than -fabricated reward-bearing trajectories. - -`policy_imitation` uses `sources`, optional disjoint `evaluation.sources`, and -`epochs`. It minimizes action negative log likelihood at each valid causal -decision. The fixed `evaluation.monitor` is `{metric: validation_action_nll, mode: min}`. BC requires no -rewards. Return conditioning requires recorded rewards and either true task -termination or an explicit finite return horizon, using the declared discount -and scale. Unknown return tails are excluded. The input return label at decision -`t` includes reward `r_t`; the current action is never an input at that decision. - -Episode/group/duplicate-content components remain wholly in training or -validation. Explicit validation sources undergo the same overlap checks. -Episodes load through bounded caches; windows are constructed on demand. -Episode fitting and decision indexing are shared independently of imitation -batches, IQL transitions, and preference pairs. - -Offline validation uses `evaluation.sources: [{source: demonstrations.holdout}]`; -omitting sources preserves automatic splitting. `evaluation.monitor` fixes -`mode: min` and `metric: validation_action_nll` for imitation/IQL or -`validation_dpo_loss` for DPO. These fields are assertions, not tuning choices. -Legacy `validation_sources` (DPO: `validation_preferences`), `monitor`, and -`direction` normalize to this shape; combining any with `evaluation` fails. -Online evaluation retains its environment/episode options and nested monitor. - -## Offline IQL - -`iql` requires complete rewards, termination/truncation and bootstrap flags, -with genuine successors for bootstrapped decisions. Nonterminal transitions -bootstrap; true terminal transitions do not. Never substitute a reset -observation for a successor. - -The phase has `updates` (required), `batch_size` (64), `gamma` (0.99), -`expectile` (0.7), `inverse_temperature` (3), `max_weight` (100), and -`target_update_rate` (0.005). Independent value and twin-Q networks use -expectile regression and one-step Bellman regression with Polyak target-Q -updates. Actor extraction minimizes action NLL weighted by +Use `sequifier train --config-path configs/train-policy.yaml`. All algorithms use +`training_plan.phases`; model interfaces and datasets follow the +[training reference](configs/train.md). Start from the shipped +[imitation](../src/sequifier/templates/train-policy.yaml), +[IQL](../src/sequifier/templates/train-iql.yaml), [PPO](../src/sequifier/templates/train-ppo.yaml), +[DPO](../src/sequifier/templates/train-dpo.yaml), or [GRPO](../src/sequifier/templates/train-grpo.yaml) +template. [Artifact transfer](supervised-policy-learning.md) covers joint real/categorical +actions and mixed supervised/policy plans. + +## Authoring and optimization + +`policy_binding` selects actions for one interface; `policy_bindings` maps named +interfaces. `global_training` owns context length, frozen normalization, validation +fraction, episode cache size, determinism and resume. Phases own batching, +`optimization.`, evaluation/checkpoint intervals and `handoff: last | best`. + +| Algorithm | Required budget/data | Trainable roles | +| --- | --- | --- | +| `policy_imitation` | `epochs`, `sources` | `actor` | +| `iql` | `updates`, transition `sources` | `actor`, `value`, `critic` | +| `dpo` | `updates`, `preferences` | `actor` | +| `ppo` | `total_env_steps`, `rollout_length`, `environment`, `evaluation` | `actor`, `value` | +| `grpo` | `total_groups`, `max_steps`, `environment`, `evaluation` | `actor` | + +Each role defaults to Adam, learning rate `0.0003`, gradient clip `1.0`, and +`ConstantLR` with factor `1.0`, total iterations `1`. Optimizers accept constructor +arguments; grouped optimizer plans set their own rates. Policy schedulers advance +on optimizer steps; plateau/compound schedulers are unsupported. `OneCycleLR.total_steps` +must match the maximum phase budget. Unknown roles fail validation. + +Imitation, DPO and GRPO default to `batch_size: 32`, `accumulation_steps: 1`, +`evaluation_interval: 1` and `checkpoint_interval: 1`. Accumulation uses actual +example counts, including partial batches. Network stochastic layers are disabled +while gradients remain enabled. Best handoff keeps the earliest finite best on ties; +last handoff keeps the latest transaction. Final updates always evaluate. +See [continuation rules](configs/train.md#optimization-across-phases) for +`optimizer_state: continue`; its default is `reset`. + +## Offline data + +Sequence parts accept CSV/Parquet tables, JSON trajectories, checksummed sequence +manifests and prepared histories. At row `t`, observations and lagged actions form +inputs; the recorded current action is the label, and reward/successor describe +its resulting transition. `columns` maps observations, static inputs, actions, +previous actions, reward, successors, boundary flags, group and episode IDs. +Singular `action`/`previous_action` conflict with nonempty plural maps. +Fragments require an explicit preceding action; file boundaries alone imply no +termination. Forecast trajectories require `action_semantics: executed_predictions`. + +Automatic splitting keeps transitive episode/group/duplicate-content components +together and requires at least two components. Explicit `evaluation.sources` +undergo the same overlap checks. Codecs and moments are fitted only on training +coverage. Indexed storage and bounded episode caches are shared across phases; +windows are constructed on demand. `import_sequences` exports canonical raw +Parquet shards and a checksummed manifest; see the +[import example](example_inputs/configs/import-policy-sequences.yaml). + +Imitation minimizes joint action NLL. Return conditioning requires rewards and +a known tail: true termination or an explicit finite return horizon. The label at +`t` includes `r_t`; unknown tails are excluded. Offline monitors are fixed to +`{metric: validation_action_nll, mode: min}` for imitation/IQL and +`{metric: validation_dpo_loss, mode: min}` for DPO. Legacy `validation_sources` +(DPO: `validation_preferences`), `monitor` and `direction` normalize to `evaluation`; +combining old and new spellings fails. + +IQL requires complete reward, terminated, truncated and bootstrap columns, plus +true successors for bootstrapped transitions. It samples uniformly with replacement, +fits expectile V and twin Q-functions, updates target Q-functions with Polyak +averaging, and weights actor NLL by `min(exp(inverse_temperature * (min(target_Q1,target_Q2)-V)), max_weight)`. -Each update samples transitions uniformly with replacement. Held-out unweighted -`validation_action_nll` is the minimizing selection monitor; Bellman residual, -values, Q and weight diagnostics are reported separately. - -The `optimization.actor`, `optimization.value` and `optimization.critic` roles -have independent optimizers and schedulers. All gradients are validated and -clipped once before any optimizer steps. Evaluation and checkpoint intervals -default to 100 updates. Critics are fresh even after another IQL phase and are -absent from the portable actor. +Defaults: `batch_size: 64`, `gamma: 0.99`, `expectile: 0.7`, +`inverse_temperature: 3`, `max_weight: 100`, `target_update_rate: 0.005`, +evaluation/checkpoint intervals `100`. Held-out action NLL selects the actor; +critic diagnostics are separate. Critics are fresh per phase and are not exported. ## DPO preferences -The inspectable manifest is separate from its sequence sources: - -```json -{ - "format": "sequifier.preference-pairs", - "version": 1, - "pairs": [{ - "pair_id": "pair-1", - "context_id": "initial-condition-1", - "chosen": {"source": "decisions", "episode_id": "good", "start": 0, "end": 8}, - "rejected": {"source": "decisions", "episode_id": "bad", "start": 0, "end": 6} - }] -} -``` - -Span bounds index the loaded episode/fragment rows, zero-based, start-inclusive -and end-exclusive. Both sides need -at least one executable action. Unequal lengths are permitted. Canonical -observations, static fields, prior-action histories and valid masks must match -at the branch point. Each subsequent score uses that alternative's actual -observations and prior actions. Duplicate pair IDs, identical/overlapping -alternatives, unsupported labels, non-finite features and incompatible initial -contexts fail before updates. - -Splitting joins shared context/group IDs, episode IDs and duplicate learning -content transitively; explicit validation manifests obey the same constraints. -At least two independent components are required for automatic splitting. -Source identities, exact spans, fingerprints and membership persist in the run -checkpoint. The manifest must state the preferences; reward is not interpreted -as a preference. - -`dpo` requires `preferences` and `updates`; optional `evaluation.sources` -selects explicit validation preference manifests using the same `[{source: dataset.part}]` -shape as other offline phases. Its fixed monitor is -`{metric: validation_dpo_loss, mode: min}`. `beta` defaults to 0.1 and must be -finite positive. `scoring_chunk_size` defaults to 64 causal windows. For the -summed FP32 action log likelihood `s`, it minimizes equally weighted pair losses: - -```text -margin = (s_actor(chosen)-s_actor(rejected)) - (s_ref(chosen)-s_ref(rejected)) -loss = -logsigmoid(beta * margin) -``` - -Padding and unselected context positions contribute zero. Losses are averaged -over pairs, without length normalization or an added supervised term. Reports -include lengths, margin and preference accuracy. Default selection minimizes -`validation_dpo_loss`. Reference scores are recomputed, avoiding stale caches. -A two-pass backward calculation uses the exact scalar margin derivative and -recomputes one chunk's differentiable likelihood at a time, bounding activation -memory without changing the summed sequence objective. - -DPO and GRPO each declare `reference: {source: phase_start}` (default) or -`{source: artifact, path: models/reference/actor.pt}`. A phase-start reference is -an independent weight snapshot before any phase update. Artifact references are -validated during preflight and reuse cached CPU weights; their files are checked -again before phase startup. References stay in evaluation mode with -gradients disabled. Full representations and actual weight hashes must match. -Embedded checkpoint reference weights survive removal of the original file; -a still-present artifact with changed weights is rejected on resume. +A preference manifest declares `format: sequifier.preference-pairs`, `version: 1` +and `pairs`. Each pair has `pair_id`, `context_id`, optional `group_id`, and +`chosen`/`rejected` spans `{source, episode_id, start, end}`. Bounds are zero-based, +start-inclusive/end-exclusive within loaded episode rows. Initial executable +histories must match. Duplicate IDs, overlapping/identical alternatives and +incompatible contexts fail before training. Rewards do not imply preferences. + +DPO minimizes `-logsigmoid(beta * ((s_actor_chosen-s_actor_rejected) - +(s_ref_chosen-s_ref_rejected)))`, where each score sums valid decision log likelihoods. +Pairs have equal weight, regardless of length. Defaults: `beta: 0.1`, +`scoring_chunk_size: 64`. Two-pass chunked backward preserves the scalar derivative +while bounding activation memory. Optional `evaluation.sources` selects held-out +preference manifests. + +DPO and GRPO default to `reference: {source: phase_start}`; an artifact reference +uses `{source: artifact, path: ...}`. References are independent frozen snapshots. +Embedded checkpoint references survive removal of their original files; +a still-present changed reference is rejected. ## Online adapters -PPO, GRPO, evaluation and policy rollout share the version-3 `OnlineEnvironmentAdapter` contract. Configure +PPO, GRPO, evaluation and policy rollout use the version-3 environment protocol: `environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. -Factories must create fresh independent adapters for evaluation. An adapter -explicitly declares: - -```python -from sequifier.rl_api import EnvironmentCapabilities, OnlineResetResult, OnlineStepResult - -class Adapter: - protocol_version = 3 - implementation_id = "my-simulator.v1" - capabilities = EnvironmentCapabilities( - exact_snapshot=True, - group_start="scenario", - reproducible_group_start=True, - group_start_randomness="independent", - ) - - def reset(self, seed: int) -> OnlineResetResult: ... - def step(self, action: dict) -> OnlineStepResult: ... - def close(self) -> None: ... - def snapshot_state(self) -> dict: ... - def restore_state(self, state: dict) -> None: ... - def create_group_start(self, seed: int) -> object: ... - def reset_group(self, token: object, rollout_seed: int) -> OnlineResetResult: ... -``` - -Reset returns the current observation, static inputs and units. Step returns -immediate scalar reward, actual successor observation, boundary flags, units -and optional boolean success. Only true termination may omit the successor. -`close()` runs in `finally`. `implementation_id` must match the authored config -and change whenever simulator/reset/RNG semantics change. - -GRPO requires `group_start: snapshot | scenario`, explicit methods, and an opaque -finite JSON-serializable token. Resetting a sample must reproduce the token's -initial condition while independently seeding future environment randomness. -`group_start_randomness` documents whether the token captures initial simulator -randomness or only the scenario. Group/sample indices and run/phase seeds derive -separate environment and action streams. Repeating a reset seed alone does not -establish this capability. Exact GRPO continuation additionally requires the -adapter's documented `reproducible_group_start` guarantee. The bundled -`CategoricalControlAdapter` demonstrates a deterministic scenario token. - -Policy inference from supplied observation rows additionally requires -`EnvironmentCapabilities(seeded_reset=True)` and -`reset_from_observation(initial: OnlineResetResult, seed: int) -> OnlineResetResult`. -It must initialize the simulator without executing an action and return exactly -the supplied observation, static inputs and units. This is distinct from the -ordinary `reset(seed)` used during training. The bundled control adapter -implements both. Forecast rollout bindings retain their separate version-1 -contract. - -## Online categorical PPO - -`ppo` requires `environment`, `evaluation`, `total_env_steps` and `rollout_length`. -Normalization is inherited from the run representation; a phase assertion is -optional. One persistent training environment collects compact sealed horizons. -True successors bootstrap fixed-length horizon/truncation boundaries; termination -disables bootstrap. FP32 GAE feeds the clipped policy objective, entropy bonus, -and separate value regression, with optional value clipping. Actor and value -optimizers share no parameters. Old collection scores are checked before update, -and collection generations are independent of phase/run indices. - +Factories receive `config` as keyword arguments. Implementation identity and units +must match the declaration. `OnlineEnvironmentAdapter`, `EnvironmentCapabilities`, +`OnlineResetResult` and `OnlineStepResult` are exported by `sequifier.rl_api`. + +Adapters implement `reset(seed)`, `step(named_domain_actions)` and `close()`; +policy rollout also requires `reset_from_observation(initial, seed)`. +Reset results provide observations/static inputs/units. Step results provide reward, +separate termination/truncation booleans, successor observation and optional success. +Never autoreset or substitute a reset observation for a final successor. A true +terminal policy step may omit its successor; other steps must retain it. + +Capabilities declare seeded reset, exact snapshot support and group-start support. +Exact PPO recovery requires `snapshot_state()`/`restore_state()` including simulator +RNG. GRPO requires `create_group_start(seed)` and `reset_group(token, rollout_seed)`; +all group members restore the same portable token with independent future streams. +Exact GRPO recovery requires reproducible group starts. Evaluation factories must +create independent adapters. The [control adapter](../src/sequifier/rl/control.py) +is a complete local example. + +## PPO and GRPO + +PPO seals one rollout generation, computes boundary-aware FP32 GAE, then optimizes +clipped actor and independent V objectives. Truncation bootstraps from the final +successor without carrying GAE across reset. Collection horizons retain live history. Defaults: `optimization_epochs: 4`, `minibatch_size: 64`, `accumulation_steps: 1`, -`gamma: 0.99`, `gae_lambda: 0.95`, `clip_epsilon: 0.2`, -`value_coefficient: 0.5`, `entropy_coefficient: 0.01`, -`normalize_advantages: true`, `reward_scale: 1`. Optional positive `target_kl` -stops optimization against the old collection policy; `value_clip` is optional. -The `optimization.actor` and `optimization.value` roles have independent -optimizers/schedulers and the shared learning-rate and clipping defaults. -`allow_partial_rollout: true` executes exactly the remaining action budget; -false requires divisibility. `max_buffer_bytes` defaults to 536870912. -`archive_directory` optionally writes separate versioned collection records; -`archive_scores` defaults to false. Archived records are not training input. - -`evaluation_interval` and `checkpoint_interval` default to one complete -collection/optimization iteration. Evaluation uses a separate adapter, -independent seeds and a whole-evaluation POSIX/main-thread deadline. It evaluates -the live actor synchronously and restores every module's training mode and the -training RNG state afterward. It requires -`environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; -`action_selection: deterministic | stochastic` defaults to deterministic. Default -`evaluation.monitor: {metric: mean_completed_return, mode: max}` selects true completed returns. -`mean_completed_length` and `success_rate` are also supported. Censored episodes -and unavailable metrics are explicit; best selection fails when no finite -monitored value is available. The final iteration always evaluates. - -## GRPO groups and objective - -`grpo` requires `environment`, `evaluation`, `total_groups` and `max_steps`. -Defaults: `group_size: 4` (at least 2), `groups_per_update: 1`, -`optimization_epochs: 4`, `advantage_epsilon: 1e-6`, `clip_epsilon: 0.2`, -`kl_coefficient: 0.04`, `reward_scale: 1`, `scoring_chunk_size: 64`. -`old_policy_target_kl` optionally limits old-policy drift independently of the -reference penalty. No value network, value optimizer, or GAE is allocated. - -Each group contains independent sequential trajectories from the same opaque -start token under one frozen old actor generation. All groups in an iteration -are sealed before any update. Scores sum observed rewards times the fixed -reward scale, through termination, truncation or the explicit `max_steps` -finite-horizon cutoff. Cutoffs do not invent termination or bootstrapped rewards. -Incomplete/failed/non-finite groups are discarded and never optimized. Errors -report issued environment actions, including actions discarded with the group; -there is no retry. Optional `max_env_steps` is checked before collection using -the conservative `group_size * max_steps` bound for every requested group. -Memory is bounded by `max_buffer_bytes` (default 536870912), checked before -collection and as frames are appended. Optional archives are separate from the -training buffer. - -The precise first variant is: - -```text -A_i = (R_i - mean(R)) / (std_population(R) + advantage_epsilon) -r_it = exp(log pi_actor(a_it|h_it) - log pi_old(a_it|h_it)) -d_it = log pi_ref(a_it|h_it) - log pi_actor(a_it|h_it) -K_it = exp(d_it) - d_it - 1 -loss = mean_trajectories(mean_valid_decisions( - -min(r_it*A_i, clamp(r_it,1-epsilon,1+epsilon)*A_i) + kl_coefficient*K_it)) -``` - -Each trajectory has equal weight regardless of length; group sizes are fixed. -Minibatches contain whole trajectories; accumulation uses actual trajectory -counts. FP32 ratios/objectives/gradients are checked before applying an update. -Equal group scores give zero advantages; the reference penalty can still update -the actor. Reference and old-policy scores stay detached across optimization -epochs. The reported `sampled_reference_penalty` is a sampled estimator under -old-policy samples, not exact current-policy KL. - -Evaluation defaults to `evaluation.monitor: {metric: score_mean, mode: max}` and scores every -independent evaluation episode through its real boundary or declared horizon. -The evaluation score uses the same fixed reward scale. Evaluation/checkpoint -intervals count completed updates, with an explicit final evaluation. Reports -include score spread, zero-variance groups, lengths, cutoff reasons, start -tokens, sampled penalty, clip fraction, generations, optimizer steps and issued -environment actions. Sealed buffers are consumed once; stale generations fail. - -## Checkpoints, resume, and artifacts - -Named sources share indexed episode storage across phases and partitions. Table -sources are parsed once into temporary sorted Parquet storage with direct episode -row-group access. The episode cache stays bounded; partition views retain metadata -and validated transition indices. Imitation shuffles episodes and then decisions -within each episode to preserve storage locality. - -Inputs are resolved once into a run-owned catalog. Source resolution returns -verified file checksums, including the checksums checked against sequence manifests; -preparation reuses those resolved sources without resolving or hashing them again. -Every phase boundary explicitly rehashes all input files, manifests, metadata, -static data and preference files to detect modifications. Output and archive path -conflicts are checked separately before output mutation and at phase boundaries. -Prepared data supplies its fitting view and frozen-representation validation; -algorithm-specific transition checks remain with the IQL dataset. - -The policy-training checkpoint is `sequifier_policy_run`, format version 6. -Version-5 checkpoints are read and upgraded on the next save. -Completed results determine the phase boundary; optional version-2 phase state -records the latest committed transaction. It contains the resolved plan, frozen -representation, source/partition and adapter identities, run RNG, completed -phase records pairing each result with its actor checksum and result fingerprint. -Actor/report paths come from the result provenance. Live phase state contains -role-keyed networks and optimizers/schedulers/step counts, sampler, selection, metrics and -algorithm-specific progress. DPO/GRPO references are embedded. Earlier selected -actors remain in their phase artifacts. Checkpoints older than version 5 are -not supported. - -Writes use temporary files, flush them, and atomically replace their destinations. -At a handoff, the selected `actor.pt` is published with its result metadata, then -the run checkpoint commits its checksum and result. The checkpoint is the -authoritative phase boundary. An interrupted, uncommitted export is replaced on -retry; a committed handoff advances exactly once. Completed phases never rerun -or reselect their actor. - -Retain phase actor artifacts alongside the checkpoint. Resume checks their -checksums and result identities before output mutation, without loading or -rewriting earlier models. Phase `report.json` files are derived from committed -results and regenerated on resume; they are not recovery dependencies. No -separate phase manifest is needed. Representation and data/adapter/reference -mismatches still fail before output mutation. - -`resume.mode: exact` requires validated PPO simulator/RNG snapshots or GRPO's -reproducible group-start capability and matching implementation identity. PPO -checks the restored snapshot exactly; GRPO derives subsequent group tokens and -independent streams from persisted counters. `reset_environments` explicitly -resets PPO environments or restarts GRPO with a new derived stream, preserving -completed updates and reporting a trajectory discontinuity. Neither mode reuses -partial groups or old on-policy samples. Offline continuation restores its -sampler/order and optimizer transaction exactly. - -Portable artifacts are ordinary actor-only `sequifier_model` bundles. They -carry normalization, codecs, architecture, policy and unit provenance, with no -critics, references or trainer dependency. The final artifact comes from the -final phase's selected actor. The run report records order/config fingerprints, -source/reference provenance, selections, clocks and continuation modes. - -## Python and inference - -The Python API starts from the same authored schema as YAML: - -```python -from sequifier.config.train_config import SequifierConfig -from sequifier.config.training_resolution import resolve_training_run -from sequifier.train import run_training -import yaml - -with open("configs/train-policy.yaml") as stream: - authored = SequifierConfig.model_validate(yaml.safe_load(stream)) -resolved = resolve_training_run(authored) -result = run_training(resolved) -``` - -`run_training` selects the supervised engine, native policy runner, or mixed-plan -coordinator from the resolved phase algorithms. `load_train_config(path, {}, False)` -combines YAML loading and resolution. `run_policy_training` takes an internal -resolved policy run, rather than a separate authored configuration. Direct Python -inference can validate `PolicyInferenceConfig`, resolve it with -`policy_inference_config(request.model_dump(exclude_unset=True))`, and call -`sequifier.rl.inference.infer_policy`; it uses the same schema as the YAML loader. -`rl_api` exports phase schemas, policy bindings, action declarations, environment -capabilities, sequence records and `load_policy`. - -Use `model_type: generative` with a selected portable PT artifact for raw CSV/Parquet -observation rows. `enforce_determinism` requests deterministic execution; -`action_selection: deterministic | stochastic` selects the policy mode or sampling. -The seed is independent of both. Seeded sampling can reproduce a sequence under -the same supported runtime, device and input order. Online evaluation uses -`action_selection`, defaulting to deterministic, with the run's execution settings. -Evaluation reports record that choice. - -Policy `rollout.environment` declares the version-3 environment, `max_steps` -sets the collection budget, and `seed_row_index` selects a zero-based raw input row. -`trajectory_path` is required; `max_episode_steps` is optional. Policy rollout -rejects forecast bindings and generation settings. It requires seeded observation -reset; successors come from actual environment transitions. True termination may -omit a successor; truncation and collection cutoffs retain it. Forecast rollout -instead selects a prepared model window and follows the separate version-1 -protocol. See [inference](configs/infer.md), [rollout](rollout.md), and the -[Python PPO workflow](example_inputs/ppo_toy_control.py). - -CPU tests cover the reference execution contract. CUDA and MPS reproducibility -also depends on backend/operators and matching device/RNG topology. +`gamma: 0.99`, `gae_lambda: 0.95`, `clip_epsilon: 0.2`, `value_coefficient: 0.5`, +`entropy_coefficient: 0.01`, `normalize_advantages: true`, `reward_scale: 1`. +Optional `value_clip` clips V changes; `target_kl` stops old-policy drift. +`allow_partial_rollout: true` executes the remaining budget exactly; false requires +divisibility. Optional `archive_directory` writes independent collection records; +`archive_scores` defaults to false. Archives are not update sources. + +GRPO seals complete same-start groups before updating an actor-only model. +Scores sum scaled rewards through termination, truncation or `max_steps`. +`A_i = (R_i-mean(R))/(population_std(R)+advantage_epsilon)`. +Its loss averages equally over trajectories, then over their decisions: +`-min(r*A, clip(r)*A) + kl_coefficient*(exp(d)-d-1)`, where +`r=exp(log_pi-log_pi_old)` and `d=log_pi_ref-log_pi`. +Defaults: `group_size: 4`, `groups_per_update: 1`, `optimization_epochs: 4`, +`advantage_epsilon: 1e-6`, `clip_epsilon: 0.2`, `kl_coefficient: 0.04`, +`reward_scale: 1`, `scoring_chunk_size: 64`. Optional `old_policy_target_kl` limits +old-policy drift separately. The sampled reference penalty is not exact current-policy KL. +`max_env_steps` uses a conservative complete-group bound. Failed/incomplete groups +are discarded without retry; errors report issued actions. Both online algorithms +default to `max_buffer_bytes: 536870912` and reject stale/reused buffers. + +Evaluation uses independent seeds/adapters, restores training modes and RNG, and +requires `environment`, `seeds`, `max_episode_steps`, `timeout_seconds` and the +main-thread POSIX timer. `action_selection` defaults to deterministic. PPO's default +monitor is `mean_completed_return`/`max`; `mean_completed_length` and `success_rate` +are supported. Censored episodes and unavailable metrics are explicit. GRPO uses +`score_mean`/`max`, including declared horizons and the fixed reward scale. +Online intervals count completed collection/update iterations. + +## Checkpoints and inference + +Policy run checkpoints use format 6 and read format 5. They retain representation, +plan/source/partition/adapter identities, selected phase results and checksums, +RNG and live phase networks/optimization/sampler/selection state. Keep committed +phase actors alongside the checkpoint; reports are regenerated from committed +results. Source files and output-path conflicts are rechecked at phase boundaries. +Atomic publication precedes checkpoint advancement, so completed phases never rerun. + +`resume.mode: exact` validates simulator snapshots or reproducible group starts. +`reset_environments` preserves updates but reports a trajectory discontinuity. +Offline resume restores sampler/order and optimizer transactions. Neither online +mode reuses partial groups or stale samples. Final portable artifacts contain +actor execution, codecs, normalization, policy and units, without critics or references. + +`load_train_config(path, {}, False)` loads/resolves the common schema; +`sequifier.train.run_training` dispatches its phases. Python APIs and environment +contracts are exported by `sequifier.rl_api` and `sequifier.training_api`. +For prediction, task routing, probability outputs and policy rollout, use the +[inference reference](configs/infer.md#policy-model-interfaces) and +[rollout guide](rollout.md). Deterministic execution, action selection and RNG seed +are independent settings; reproducibility also depends on runtime/device/order. diff --git a/documentation/public-config-surface-implementation-plan.md b/documentation/public-config-surface-implementation-plan.md deleted file mode 100644 index 5a65b18f..00000000 --- a/documentation/public-config-surface-implementation-plan.md +++ /dev/null @@ -1,368 +0,0 @@ -# Public configuration surface implementation plan - -## Design goal and reference - -Build an ergonomic policy-training and rollout extension of the public configuration at commit `2d01d407cd2134c8197b1a18f0034d01486f18a9`. Use that commit's ownership rules, singleton conveniences, source references, component specifications, and authored/resolved separation as the foundation. - -Use commit `082225833e1d744e662d094008bcc44268ab868a` as the capability baseline. Every supported operation and meaningful control at that commit must remain expressible and executable through the new design. The earlier reference commit guides layout, not the set of features to retain. Record and assess any subsequent capability additions before implementation finishes. - -Extend `SequifierConfig` and `ResolvedSequifierConfig` instead of introducing a parallel public run abstraction. Keep `training_plan` as the home of training phases, `dataset_training` as the home of offline data and dataset training policy, and `model` as the home of architecture. Keep the explicit inference selector `model_type`, extending its values to include `policy`. - -This is one clean public contract. The implementation removes the competing policy/mixed-run schemas and routing mechanisms. No configuration-version switch, conversion tool, deprecation period, alternate-format loader, import wrapper, or checkpoint-format upgrade is part of the work. - -The examples below describe the proposed design, not the schemas currently implemented. - -## 1. Reuse the reference commit's layout and abstractions - -The reference commit's train, infer, and hyperparameter-search guides and their corresponding configuration models establish these patterns: - -| Reference pattern | Apply it to the new surface | -| --- | --- | -| One shared backbone and named model interfaces | Policies select ordinary interfaces and existing target heads. No separate policy architecture declaration. | -| `dataset_training..parts.` | Put offline policy data in typed dataset parts. Reuse dataset/interface ownership and metadata resolution. | -| `sources: [{source: dataset.part}]` | Use the same reference object in supervised and offline policy phases and dataset evaluation. | -| `training_plan.phases` and direct single-phase fields | Add algorithm-specific phase variants in the same plan; support the same singleton convenience for policy phases. | -| Unique routes inferred; named routes required when ambiguous | Apply the rule to policy interfaces, sources, and bindings. Never select the first eligible mapping entry. | -| `model.interface`, `dataset.part`, and `evaluation: true` | Keep these intentional authoring conveniences and their deterministic expansion into named structures. | -| `ComponentSpec` and `OptimizerPlan` | Reuse flattened component arguments, parameter selectors, optimizer groups, and scheduler validation for policy optimization roles. | -| `global_training` owns runtime, optimization defaults, and recovery | Extend that owner rather than adding policy-only top-level runtime/recovery conventions. | -| Preprocessing metadata owns types, vocabularies, statistics, and storage facts | Resolve policy execution metadata from the same source of truth. | -| `SequifierConfig` / `ResolvedSequifierConfig`, `InferenceConfig` / `ResolvedInferenceConfig` | Validate authored choices first; perform resource loading and execution resolution separately. | -| Explicit required `model_type`, with `generative` and `embedding` operations | Add the `policy` operation; remove `execution_mode` and artifact-triggered operation selection. | -| Composed YAML and HPO trees use ordinary config paths | Extend the same composition, singleton expansion, and trial validation mechanisms. | - -The distinction between singleton and named authoring is useful ergonomics. Both forms expand to one schema and one execution meaning. Neither selects a different run class or runtime path. - -## 2. Training: extend `SequifierConfig` and `training_plan` - -### Run ownership - -Use the reference layout with a small number of deliberate extensions: - -| Location | Ownership | -| --- | --- | -| Top-level `project_root`, `model_name`, `device`, `seed`, export fields | Run identity, execution target, and outputs, as in the reference. | -| `model.backbone`, `model.interfaces` | Architecture, input/target columns, static/image/depth components, parameter initialization. | -| `initialize_from` | Optional full-model artifact initialization; distinct from resuming a run or reusing only a backbone repository. | -| `global_training` | Shared runtime settings, window defaults, supervised optimization defaults, determinism, precision, checkpointing, and resume. | -| Top-level `model_path`, `report_path`, `phase_directory` | Optional destinations for the final model, run report, and phase artifacts; retain configurable outputs alongside existing export fields. | -| `dataset_training` | Named offline datasets, interface references, typed parts, supervised loss settings, and dataset freezing. | -| `policy_bindings` | Interface-keyed task declarations: named actions, action distributions, and explicit policy input bindings. | -| `training_plan` | Ordered phases, phase budgets, and supervised curriculum policy. | -| `evaluation` | Shared dataset evaluation specification, with phase-local evaluation where the algorithm needs a different contract. | - -Do not add top-level `phases`, `run_type`, `policy_interface`, `execution_model`, `sources`, `preferences`, or a second `resume` block. Their responsibilities belong to the owners above. - -Extend `TrainingPlanModel.phases` to a discriminated union keyed by `algorithm`: `supervised`, `policy_imitation`, `iql`, `ppo`, `dpo`, and `grpo`. An omitted algorithm means `supervised`, keeping a simple supervised phase as concise as the reference. Insert that default before discriminated validation. - -Keep algorithm-specific budgets: supervised/imitation `epochs`, IQL/DPO `updates`, PPO `total_env_steps`, and GRPO `total_groups`. Do not make them fit an artificial universal duration field. Keep supervised `mode`, `selection`, weights, and batches per selection specific to supervised phases. - -Direct single-phase fields under `training_plan` work for every algorithm. Expand them into one phase named `train`; fill `mode: sequential` only for supervised phases. Reject direct phase fields combined with `training_plan.phases`. Policy phases never receive supervised-only defaults. - -### Global settings and phase optimization - -Retain `global_training` as one shared owner, including in policy-only runs. Extend its validation so users do not have to invent supervised settings for an online policy phase: - -- Require supervised objective/loss/data-loader settings only when selected supervised phases use them. -- Resolve window/model-contract facts from declared model metadata or `initialize_from`, checking explicit values as assertions. -- Keep policy-specific learning dynamics in their phase schema, including batch sizes that have algorithm-specific meanings. -- Reuse `ComponentSpec`, `OptimizerPlan`, parameter catalogs, and freezing selectors for each policy `optimization` role. A policy role uses its phase settings and documented algorithm defaults; do not silently inherit a supervised epoch scheduler as a policy update scheduler. -- Phase `optimizer_state: reset | continue` replaces the global reset switch. Continuing state requires matching algorithms, roles, parameter ownership, optimizer/scheduler definitions and a last handoff; mixed coordinator transitions are rejected. -- Keep `global_training.resume` using `ResumeConfig.policy` and `checkpoint_path`. Extend that block with an environment recovery `mode: exact | reset_environments`, defaulting to `reset_environments` when relevant. Validate snapshot capabilities for exact online recovery. -- Put automatic offline holdout sizing at `global_training.validation_fraction` and bounded episode caching at `global_training.episode_cache_size`. Keep both configurable; neither requires an offline preprocessing step. -- Retain phase-local `freeze`, `evaluation_interval`, and `checkpoint_interval`, alongside dataset-specific freezing. Resolve their combined parameter ownership with the existing catalogs/selectors and validate which roles may update each parameter. - -Retain every applicable algorithm-specific control, including PPO partial-rollout handling, normalization, reward scaling, clipping/KL/entropy/value settings, optimization epochs, minibatching and accumulation; IQL expectile, temperature/weight limits, target updates and discounting; DPO beta and reference selection; and GRPO grouping, reference/old-policy KL settings and collection limits. Keep `reference.source: phase_start | artifact` and `reference.path` for DPO/GRPO. Keep `scoring_chunk_size`, `max_buffer_bytes`, `max_env_steps`, `archive_directory`, and PPO `archive_scores` on the applicable phase schemas. Their units and effects are algorithm-specific and must be documented rather than dropped during consolidation. - -Use top-level `model_path`, `report_path`, and `phase_directory` overrides for run outputs and `global_training.resume.checkpoint_path` for the rolling run checkpoint destination, including when resume policy is `never`. Preserve timed/batch/epoch/latest checkpoint controls and supported export/dropout/embedding options. Resolve paths under `project_root` and validate input/output collisions. Document derived defaults once for the unified run; no output location should become unconfigurable merely because its former run class is removed. - -Keep supervised phase transitions on continuing model weights. Add `handoff: last | best` to phases, defaulting to `last`; `best` requires a usable evaluation monitor and restores the complete selected model/distribution state. Handoff and optimizer-state continuation are separate decisions and must be validated together. - -### One offline source system - -Extend dataset parts with a discriminated `type`, following the component pattern: - -| Part type | Authored data contract | Eligible uses | -| --- | --- | --- | -| `prepared` (default) | The reference `metadata_config_path`, media paths, and file ordering. | Supervised training; policy learning only when explicit action/episode mappings supply the needed task semantics. | -| `sequence` | Direct `paths`, a source `manifest`, or a metadata-selected split; optional schema/normalization metadata and typed column mappings. | Imitation and transition-based policy learning from tables, trajectories, or prepared data. | -| `preferences` | Preference manifest referencing sequence parts through the shared dataset catalog; metadata supplied by those parts or the resolved interface as needed. | DPO preference scoring. | - -Reuse `SequenceSourceConfig`, `PolicyTableColumns`, trajectory readers, and preference readers as implementation building blocks. Refactor their public placement into dataset parts instead of exposing parallel top-level source registries. Keep raw/model encoding declarations with the part's data contract. - -Sequence parts retain the current source controls: `paths`, `manifest`, `format`, `columns`, `encoding`, `metadata_config_path`, `split_index`, `static_data_path`, `episode_end`, `episode_namespace`, `units`, and `action_semantics`. Select exactly one data location: direct paths, a manifest, or `metadata_config_path` with `split_index`. Metadata may also accompany direct paths/manifests as a schema assertion; its presence alone must not replace their data location. Retain automatic format detection and explicit table/trajectory/prepared reading where currently supported. - -Raw table and trajectory inputs do not require a generated preprocessing metadata file when the model/artifact, source declarations, and frozen normalization supply the necessary contract. Model-encoded/prepared inputs still require the metadata needed to interpret their values. Missing necessary information is a precise resolution error, not a requirement to preprocess every raw source. Preference manifests resolve their sequence references against the same dataset/part catalog, with no independent top-level registry. - -Keep explicit `action_semantics: executed_predictions` for learning from forecast trajectories, separate from policy-action records. Retain episode identity namespaces, end-of-file completion declarations, previous-action/successor mappings, and bootstrap flags. These affect task meaning and leakage checks, not merely data-loading syntax. - -`dataset_training..model_interface` still owns the dataset's interface. A reference to a whole dataset selects all its parts in declaration order; `dataset.part` selects one. Parts in one dataset must agree on the selected interface's codecs, normalization, and input semantics. Phase validation rejects incompatible part types or missing task fields. - -Make `criterion` and supervised loss settings required when that dataset is selected by a supervised phase. Policy-only datasets do not need dummy criteria. Rewards, terminal/truncation flags, successors, action labels, and episode boundaries require explicit data semantics; a supervised window is not automatically an RL transition. - -Use `TrainingSourceSpecModel` references throughout: `sources: [{source: demonstrations.recorded}]`, not a different list-of-strings syntax for policy phases. DPO's `preferences` field uses the same reference object and requires a preference part. Validate whole-dataset references for that algorithm rather than silently skipping incompatible parts. - -Retain explicit offline `validation_sources` and DPO `validation_preferences` on phases, using the same source reference objects as their training counterparts. Resolve holdouts in this order: explicit phase validation references, declared metadata validation partitions where available, then a seeded automatic split using `global_training.validation_fraction`. Apply the automatic split only when no validation partition was supplied. Partition sequence data by episode and shared demonstration/group identity; split preference data by connected underlying episodes/spans rather than individual pairs. Detect overlap even when the same content is exposed through different source names. Keep explicit separate validation datasets and automatic splitting as first-class workflows. - -Dataset evaluation uses the same source references as training. Its selected sources/metrics must agree with the effective validation partitions; an evaluation declaration must not silently re-split or replace phase validation data. Extend `EvaluationMonitorSpecModel` with registered algorithm metrics while retaining the reference spelling `mode: min | max`. Online evaluation retains its environment, seeds, episode budget, timeout, and deterministic/stochastic `selection`; expose its monitor with the same `metric`/`mode` vocabulary. A phase-local evaluation specification replaces the shared evaluation for that phase. `evaluation: true` expands only when one dataset evaluation source can be inferred; it never invents online evaluation seeds or an environment. Each phase must have a compatible effective evaluation specification when its algorithm or handoff requires one. - -For example, a raw demonstration source and a separate holdout need no source metadata file when an initialization artifact supplies their model contract: - -```yaml -dataset_training: - demonstrations: - model_interface: control - parts: - train: {type: sequence, paths: [data/train.csv], format: table} - holdout: {type: sequence, paths: [data/holdout.csv], format: table} -training_plan: - algorithm: policy_imitation - sources: [{source: demonstrations.train}] - validation_sources: [{source: demonstrations.holdout}] - epochs: 3 -``` - -This fragment assumes matching column names and disjoint episode/group identities. Omitting `validation_sources` permits the configured automatic holdout when the source has no declared validation partition. - -### Minimal policy authoring - -This proposed complete run declaration initializes from an ordinary model artifact, infers its unique interface, and reuses the reference singleton conventions: - -```yaml -project_root: . -model_name: controller-imitation -device: cpu -initialize_from: models/controller.pt - -global_training: - resume: {policy: never} - -policy_binding: - actions: - action: {type: categorical, values: [left, right]} - -dataset: - part: - type: sequence - metadata_config_path: configs/metadata_configs/demonstrations.json - -training_plan: - algorithm: policy_imitation - epochs: 3 - -evaluation: true -export_pt: true -``` - -`policy_binding` is the singleton spelling of `policy_bindings.`, following `model.interface`; it has exactly the same schema and meaning. Reject singular and plural declarations together, even if their contents match. A binding never lives under `model.interfaces..policy` in authored config. - -For mixed training, the surrounding model, global settings, and datasets keep their locations. The phase list can contain: - -```yaml -training_plan: - phases: - - name: supervised_warmup - epochs: 5 - mode: sequential - sources: [{source: control_data.train}] - - name: imitate - algorithm: policy_imitation - sources: [{source: demonstrations.recorded}] - epochs: 3 - - name: online_improvement - algorithm: ppo - model_interface: control - total_env_steps: 4096 - rollout_length: 128 - environment: - factory: my_control:make_environment - implementation_id: control-v1 - evaluation: - environment: - factory: my_control:make_environment - implementation_id: control-v1 - seeds: [17, 29] - max_episode_steps: 100 - timeout_seconds: 30 - monitor: {metric: mean_completed_return, mode: max} -``` - -This second example is a fragment; it assumes the referenced datasets and binding for interface `control` are declared. Offline phases derive their interface from their dataset references. An explicit phase `model_interface` is an assertion when a source already fixes it, and selects the route for online phases. Infer it only when one eligible interface exists. A policy phase operates on one interface; reject offline references spanning several interfaces. - -## 3. Policy bindings: task choices separated from execution facts - -Introduce a small authored `PolicyBinding` and a separate `ResolvedPolicyBinding`. Use named `actions` for a single action and for several actions alike. Remove the `action_column`/`action_values` alternative and the runtime distinction selected by `is_joint`. - -An authored binding contains: - -- Named action distributions/domains, using `ActionFieldConfig` and existing categorical/real type vocabulary. -- `decision_position` when selecting a particular decoder output is necessary. -- `previous_action_inputs` and `initial_inputs` when ordinary declared model inputs require them. -- Optional return conditioning, defaulting to disabled, and its return definition/initial desired return when enabled. -- Task `units`, checked against source and environment declarations and retained in execution/trajectory identities. - -Resolve action ordering from the selected interface's target/decoder contract. Validate that actions are a compatible subset of model targets, categorical support matches executable decoder IDs, real scales are finite and explicit, and conditional activity respects decoder dependencies. Leave unrelated prediction heads available to supervised phases. - -The model and preprocessing metadata supply observation/static membership, types, class counts, ID maps, normalization, context geometry, and decoder semantics. Do not expose `alignment`, `input_schema`, `input_statistics`, class maps, `execution_model`, or phase `representation` in authored config. Do not insert previous-action/return inputs or replace target heads implicitly. - -A fresh policy model uses the same `ModelSpecModel`, component initialization, and metadata-driven interface resolution as a fresh supervised model. For online-only models, add optional `global_training.metadata_config_path`: a string when the model interface is unique, or an interface-to-path map when named routes need different metadata. This reuses the reference's scalar-or-named-map pattern for media paths and supplies input metadata without a dummy offline dataset. If a selected dataset or initialization artifact already supplies the interface metadata, this declaration is unnecessary; simultaneous declarations must agree. Keep statistics/vocabularies in the metadata file. Reuse the explicit frozen-normalization contract for online data instead of deriving normalization from a changing rollout stream. - -Full-model artifact initialization resolves architecture, codecs, and existing weights once. Bindings attach to existing targets; explicit architecture or metadata assertions must agree. Reuse parameter initialization/catalog abstractions for declared new distribution parameters. Do not overload backbone repository settings with full-policy initialization. - -Artifact serialization retains resolved policy metadata for portable execution. Artifact readers and checkpoint validation consume the resolved type directly; they do not pass internal dictionaries through authored config validation. Use one ordinary model-input execution path for all named action configurations. - -Removing the separate scalar-action branch must retain single-categorical behavior cloning and return-conditioned training. Provide explicit model inputs and binding declarations for previous-action history, initial/start values, and desired-return features; the resolver must support their encoding and update rules without depending on that branch. Cover string/numeric category labels, class IDs, first-step history, sequence fragments, padded windows, return-to-go/remaining-return calculations, sampled/deterministic action selection, likelihood, gradients, and collection/replay. Existing mixed/real distributions, learned scales, bounds, conditional activity, inactive values, and point-mass atoms remain available through `ActionFieldConfig`. Branch deletion is gated on capability coverage in the unified execution path. - -## 4. Inference: extend the existing `model_type` - -Keep `model_type` required and explicit, just as in the reference commit. Extend its allowed values; remove `execution_mode` and artifact-based auto-selection. - -| `model_type` | Operation | -| --- | --- | -| `generative` | Ordinary prediction on the selected model interface, even when the artifact also has policy metadata. | -| `embedding` | Extract configured representations using the existing embedding path. | -| `policy` | Execute the selected interface's resolved policy binding. Fail if that capability is absent. | - -There is no operation default to discover from an artifact. Missing `model_type` is a normal required-field error. Interface inference still follows the unique-route rule; unsupported formats and ambiguous routes produce explicit errors. Adding policy metadata cannot change what `model_type: generative` does. - -Keep the reference field locations and defaults for `project_root`, `model_path`, `data_path`, `training_config_path`, `dataset`, `part`, `model_interface`, read/write formats, device, batching, sampling, and media inputs. Share common inference fields rather than maintaining independent policy-only defaults. Retain top-level `output_path` for an explicit policy prediction destination, along with output-format and probability controls and default derived paths. - -For `model_type: policy`, keep `desired_return`, `desired_return_column`, named `previous_action_columns`, `sequence_id_column`, and `item_position_column`. A single previous action uses a one-entry named map. Retain independent-row and ordered-sequence inference, fragment-start previous actions, and recorded action history. Validate missing fragment context and incompatible return conditioning rather than silently resetting history or discarding the desired return. - -Use policy `action_selection: deterministic | stochastic` for action choice, consistently with online evaluation; use `enforce_determinism` for runtime algorithm determinism. Preserve seed-controlled stochastic sampling, categorical probability outputs, and joint-action probability/log-probability outputs with their existing meanings. Generative sampling/dropout and embedding output options remain available. Consolidating defaults must not remove either action-selection mode or make stochastic sampling depend on disabling runtime determinism. - -Refactor `load_inferer_config` around the reference resolution order: - -1. Compose YAML and apply supported CLI overrides. -2. Validate the explicit operation and authored route/field shapes. -3. Load the selected training route and/or portable artifact through a shared cache keyed by normalized path, independent of the chosen operation. -4. Resolve interface and metadata; explicit values are assertions against those contracts. -5. Validate operation capabilities and build the resolved inference specification. - -Fix the PT cache issue as part of this loader: prediction must not depend on a policy-routing helper having populated `artifacts[model_path]`. Require the declared policy operation to resolve a usable policy interface; no fallback to generative prediction. Extend the existing inference models with a typed operation-specific specification and share resolution services instead of dispatching between unrelated public constructors. - -## 5. Rollout: explicit type and local collection budget - -Use a discriminated `rollout.type: forecast | policy`, following the reference's `type` convention for components. Both variants share `adapter`, `trajectory_path`, `seed_index`, positive `max_steps`, and optional positive `max_episode_steps`. - -```yaml -project_root: . -model_path: models/controller.pt -model_type: policy -data_path: data/initial_observations.csv -read_format: csv -device: cpu -rollout: - type: policy - adapter: - factory: my_control:make_environment - implementation_id: control-v1 - config: {} - seed_index: 0 - max_steps: 200 - trajectory_path: outputs/control-trajectory.json -``` - -`model_type` selects the inference operation; `rollout.type` declares the interaction contract. Validate `forecast` with `generative` and `policy` with `policy`; reject embeddings with a rollout. Do not make either discriminator silently override the other. - -Reuse a shared adapter-factory base with `factory` and `config`, plus distinct forecast/online schemas. Forecast adds input `bindings`; online policy adds implementation identity, units, and capabilities. The type fixes the protocol: forecast uses protocol 1, online policy uses protocol 3. Keep protocol validation internal; users need not repeat a fixed `protocol_version` in YAML. Use the same online adapter schema in rollout, training `environment`, and online evaluation. - -Keep the two runtime protocols distinct: forecast reset receives prepared history and steps return external inputs; online policy reset/step supplies observations and exposes snapshot/group-start capabilities. Share loading, resource cleanup, limit validation, and meaningful common vocabulary. - -Define the collection controls precisely: - -- `max_steps` limits transitions collected by this invocation. Reaching it alone is a collection cutoff. -- `max_episode_steps` is an optional environment time limit with truncation semantics. True termination remains distinct; preserve correct bootstrapping when limits coincide. -- Forecast `seed_index` selects a prepared window in dataset/chunk order. Policy `seed_index` selects an initial observation row. Document the seed contract under each explicit rollout type. - -An environment rollout requires no `autoregressive` or top-level `generation_steps`. Reject those fields when `rollout` is present, so there is only one collection budget. Standalone autoregressive prediction keeps the reference `autoregressive`/`generation_steps` controls. - -Pass the local budget through forecast/policy inference, collectors, trajectory metadata, and public rollout APIs. Keep the collected data formats faithful to their forecast or policy semantics. - -## 6. Resolution, execution, and public APIs - -Reuse the existing pipeline and extend its typed results: - -```text -compose YAML + supported CLI overrides - -> expand singleton conveniences - -> validate authored SequifierConfig / InferenceConfig - -> resolve model, metadata, sources, bindings, and capabilities - -> ResolvedSequifierConfig / ResolvedInferenceConfig - -> phase coordinator and native algorithm runners -``` - -Singleton expansion is deterministic, non-mutating, and structural. It infers only unique routes, rejects singular/plural mixtures, and never performs artifact reads or chooses an execution implementation. Resolve an artifact-backed unique interface after resource loading, with the same ambiguity rule. - -Retain `additional_config_paths`, project-root-relative paths, duplicate-field rejection for complementary fragments, list replacement for overlays, and explicit assertions against artifact/metadata contracts. Extend discriminator replacement to phase `algorithm`, dataset-part `type`, rollout `type`, and operation-specific inference fields, so stale fields cannot survive a variant change. - -Extend HPO's same-path parameter tree and singleton expansion using the actual authored model. Validate each materialized trial; do not create another mirrored schema. Make algorithm/search capability limits explicit before trials start. - -Use `LoadedTrainConfig`, `ResolvedTrainingPhase`, `SelectedInterfaceConfig`, dataset build views, and `ExecutionSchema` as foundations. Extract neutral shared definitions where necessary to avoid circular imports; do not add a new public run-envelope class merely to carry internal runner arguments. - -Refactor the phase coordinator to consume the resolved `training_plan`. Preflight all source/interface/environment and execution-mode requirements before any optimizer update or output publication. Reuse native supervised and policy runners behind internal typed phase specifications; remove public `PolicyRunConfig`/`LearningRunConfig` dispatch. Retain supported supervised distributed execution, component initialization, exports, integrations, and optimizer plans as product capabilities. Unsupported mixed execution combinations fail with the relevant phase/capability named. - -Preflight restrictions may describe combinations the capability baseline does not support; they cannot be used to remove a combination that already works. Retain direct Python operations for model/policy loading, attachment, training, evaluation, collection, trajectory import/export, preference scoring, and replay/inspection through the new shared types. Configuration cleanup must not make these operations CLI-only. Keep the sequence-import preprocessing operation, sharding controls, source readers, and forecast/policy trajectory save/load/validation facilities; refactor their authored contracts to reuse the same source and binding types. - -Use one recovery contract owned by `global_training.resume`. Current-design checkpoints record phase position, committed optimizer clocks, RNG, role optimizers/schedulers/scalers, source/model/binding identities, selected handoff state, and necessary environment state. Validate exact versus reset-environment recovery and prevent duplicate phase execution. Test recovery of runs created by this design; no checkpoint conversion or older-format support is required. - -Update CLI and Python entry points together. Expose the same authored and resolved types through the existing public API modules. Remove superseded config classes, alternate selectors, scalar-action branches, and source registries rather than wrapping them. - -## 7. Implementation sequence and acceptance gates - -Before implementation, create a field-by-field capability inventory against `082225833e1d744e662d094008bcc44268ab868a`. Inspect public config models, CLI commands/overrides, Python API exports, readers, runners, and examples. For each functional option, record its new authored location or derived execution owner, supported combinations, and an executable acceptance test. An internal fact may become resolved state; a user-controllable behavior needs a new authored home. Retired duplicate spellings are not separate capabilities. The matrix below establishes minimum coverage and must be expanded if the inventory finds additional functions. - -| Current capability family | New owner / required expression | Completion evidence | -| --- | --- | --- | -| Raw tables, trajectories, source manifests, prepared splits, media/static inputs | Typed dataset parts with direct locations and optional/required metadata according to encoding. | Raw-source training without a generated metadata file; manifest and prepared-split execution; static/media resolution. | -| Episode/transition semantics and forecast-derived actions | Part column mappings, end declarations, namespaces, units, encoding, and `action_semantics`. | Correct history, terminal/bootstrap semantics, episode identities, and forecast-trajectory import. | -| Explicit and automatic validation | Phase `validation_sources` / `validation_preferences`, metadata partitions, `global_training.validation_fraction`. | Separate holdout execution; deterministic episode/component splits; overlap rejection. | -| Bounded episode loading | `global_training.episode_cache_size` and existing data-loader/memory controls. | Cache limit respected without changing episode content or partitions. | -| All supervised objectives/components and training policies | Existing model/global/dataset fields, phase selection/interleaving, curriculum, losses, hashing, initialization/freezing. | Supported model/objective fixtures, loss/gradient checks, and multi-dataset execution. | -| Imitation, IQL, PPO, DPO, GRPO and their optimization roles | Typed phases retaining every applicable budget, objective parameter, optimizer/scheduler and freeze control. | Each algorithm executes a meaningful update with representative non-default controls. | -| References, scoring, archives, and collection memory/step caps | Phase reference/scoring/archival/limit fields. | Phase-start/artifact references, chunked score agreement, archive score round trip, enforced limits. | -| Single categorical, return-conditioned, joint and real policies | Named bindings, explicit model inputs, existing action distribution fields. | History/return encoding, sampling, likelihood/gradients, update, and replay tests. | -| Policy inference controls | Explicit `model_type`, `selection`, desired-return/history/sequence columns, `output_path`, probabilities. | Independent rows and fragments, deterministic/stochastic actions, custom output and probability checks. | -| Prediction, embeddings, dropout and supported artifact/device routes | Existing inference/export fields and explicit operation. | PT/ONNX and supported device/mode tests; prediction independent of policy metadata. | -| Forecast and policy environment interaction | Typed rollouts/adapters, units/capabilities, local budgets, seed and episode controls. | Both protocols, snapshot/group-start capabilities, truncation/cutoff/cleanup checks. | -| Output and checkpoint control | Run output paths/export settings; global resume/checkpoint destinations and intervals; phase intervals/archives. | Custom destinations used, snapshot timing/rolling checkpoint behavior, report/phase artifact checks. | -| Recovery and mixed-phase handoff | Global resume mode, full phase state and identities, phase handoff/optimizer policy. | Same-design stop/resume, exact/reset environments, supervised → policy → supervised with no duplicate updates. | -| Public CLI/Python workflows and sequence import | Shared authored/resolved models and existing operation entry points. | Equivalent CLI/Python execution; import/sharding, direct evaluation/collection and trajectory operations. | -| Composition, HPO, integrations and supported distribution | Reference composition/search trees, runtime services, integration APIs. | Supported overrides/trials/integrations/distribution remain executable through the unified contract. | - -Each inventory entry must link its acceptance test before its implementation slice is complete. The final release gate requires no uncovered or removed baseline capability. New field names, defaults, explicit selectors, and derived metadata ownership may differ; functional expressiveness and supported operations must remain. - -| Slice | Implementation and principal locations | Acceptance gate | -| --- | --- | --- | -| 1. Reference contract and authored models | Complete the capability inventory; extend `SequifierConfig`, `TrainingPlanModel`, inference models, singleton expansion, and neutral shared definitions in `config/`. Write schema/examples first. | Every inventoried control has an owner; supervised/policy singleton and named configs use one model; public schemas contain no execution metadata or competing selectors. | -| 2. Dataset parts and shared references | Extend dataset parts/resolution, source and preference readers, validation partitioning, cache controls, source references and evaluation models. | Shared references resolve each data type; direct raw paths/manifests work without unnecessary metadata; explicit/automatic holdouts and bounded caching execute. | -| 3. Binding and execution resolution | Add authored/resolved bindings; update attachment, execution schemas, artifact config/export and shared materialization. Remove scalar-action execution selection only after coverage. | Single-categorical/return-conditioned and joint/real policies retain input, likelihood, update and replay capabilities; attachment and new artifact round trips work. | -| 4. Training lifecycle | Update training/coordinator/algorithm runners, all phase controls, optimizer/freezing services, output/checkpoint controls, global resume and API exports. | Each algorithm and references/archives/limits work; policy-only and mixed plans execute; custom outputs, handoff, role ownership and recovery pass their inventory tests. | -| 5. Inference and rollout | Extend explicit `model_type`, shared artifact caching, complete policy inference options, typed rollout/adapter schemas and public APIs. | Custom outputs, probabilities, return/fragment history and both action-selection modes work; generative routing is stable; both rollout contracts execute. | -| 6. Tooling, docs, and cleanup | Update composition, HPO, imports/CLI/Python operations, templates, examples/guides and CI; delete superseded public/runtime branches after coverage. | Every inventory entry has a passing acceptance test; all baseline capabilities remain expressible/executable; examples and shared CLI/Python contracts validate. | - -Dependencies: establish the authored shape in slice 1 before implementing resolution. Dataset resolution and bindings converge before complete training integration. Inference/rollout can build on binding resolution while training lifecycle work proceeds. Include recovery identities and fixtures during slice 4, not after public APIs are finished. - -## 8. Validation and completion criteria - -Keep tests in the adjacent `../sequifier-test` repository: schema and validation tests under `local/unit`, execution tests under `local/integration`, and shared fixtures in `local/conftest.py`. Reuse its existing conventions and validate the intended behavior of the new design. - -| Area | Required checks | -| --- | --- | -| Ergonomics and structure | Supervised singleton example from the reference; singleton policy run; named mixed run; independent singleton conveniences; unique-route inference; singular/plural conflicts; no extra routing/version flags. | -| Authored/resolved boundary | JSON schemas exclude internal fields; authored structural validation performs no file/environment loading; unknown fields and invalid algorithm-specific budgets fail at useful paths. | -| Dataset resolution | Raw table/trajectory paths without preprocessing metadata when resolvable; manifests/prepared splits; part eligibility; encoding/static/media/units; explicit versus automatic validation and configurable fractions; episode/preference leakage; episode-cache bounds; evaluation source/monitor validation. | -| Policy execution | Single-categorical behavior cloning and return conditioning; categorical IDs/start values; fragment/padded history; real/joint distributions and atoms; decoder order; fresh/initialized models; sampling, likelihood, gradients, update and replay after artifact reload. | -| Training | Each algorithm's full control inventory, budgets and roles; reference selection; chunked scoring; partial rollouts; archives/scores and buffer/step limits; dataset/phase freezing; interleaving/curriculum; handoff and optimization lifecycle. | -| Runtime capabilities | Supervised distribution/integrations and image/depth/static inputs; clear preflight errors for unsupported combinations; no source-shape-based runner selection. | -| Inference | Explicit operations and shared PT cache; independent-row/sequence/fragment inputs; desired-return/history column overrides; deterministic/stochastic selection independent of runtime determinism; custom output paths, probability outputs, dropout/embeddings and supported artifact/device routes. | -| Rollout | Forecast/online protocol validation; seed preparation for each type; budget 1 and invalid limits; termination/truncation/cutoff/bootstrap; adapter close on failure; no redundant generation controls. | -| Recovery | Uninterrupted versus resumed updates; phase-boundary commits; exact/reset-environment recovery; changed source/binding/adapter identities rejected; no repeated completed phase. | -| Outputs and checkpoints | Custom model/report/phase destinations; checkpoint destination with resume enabled/disabled; phase and timed/batch/epoch/latest intervals; archive paths; export/dropout settings and input/output collision checks. | -| Tooling | Composition/variant replacement; singleton/named HPO paths; CLI/Python operation coverage; sequence import/sharding; direct collection/evaluation and trajectory/preference inspection; all inventory entries linked to tests. | - -Use deterministic tiny CPU models and toy adapters for integration tests. Include actual artifact-backed inference, one mixed phase run, and stop/resume checks. Test encoding, gradients/log probabilities, and observable outputs rather than relying on schema parsing alone. - -Run focused pytest suites, relevant training/inference/rollout checks, Ruff and type checks. Update `README.md`, `documentation/configs/{train,infer,hyperparameter-search}.md`, policy/supervised-policy/rollout guides, templates and example configs; regenerate `src/sequifier/consolidated-docs.md` with `hooks/consolidate_docs.py`. - -The design is complete when a user familiar with the reference commit can add policy data and phases using the same ownership, naming, metadata, source-reference, and singleton rules, and every capability inventoried from `082225833e1d744e662d094008bcc44268ab868a` remains expressible and passes its acceptance test. The additional concepts should express the task itself: action distributions, interaction contracts, and algorithm-specific budgets. They should not expose implementation paths or introduce another way to describe a run. diff --git a/documentation/rollout.md b/documentation/rollout.md index 69c7418f..836650d4 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -1,180 +1,73 @@ -# Local numeric forecast rollouts +# Local rollouts -For trajectory-backed categorical policy training and return-conditioned -execution through the online environment protocol, see [policy learning](policy-learning.md). +Forecast and policy rollout share trajectory recording but retain distinct adapter +protocols. Both execute one local episode/fragment, own adapter cleanup, and atomically +write `trajectory_path` after successful validation. They do not add an export format. -The first rollout slice collects one seed-initialized episode or episode fragment -on CPU. It supports real scalar features and real static conditioning, with -one-step `causal` or `final_value` forecast models (`target_offset: 1`). Model -predictions and a persistent local adapter jointly produce the next input row. -There is no learning algorithm in this path. +## Forecast configuration and protocol -## Inference configuration - -Add this to an otherwise valid inference configuration for a model with inputs -`x`, `d` and target `x`: +Forecast rollout supports CPU, real scalar temporal/static inputs, and one-step +`causal`/`final_value` models with `target_offset: 1`, using PT or schema-bearing ONNX: ```yaml -device: cpu model_type: generative rollout: - max_steps: 3 + max_steps: 100 seed_window_index: 0 trajectory_path: outputs/trajectory.json - environment: - factory: your_application.adapters:make_environment - config: {} # keyword arguments accepted by your factory + environment: {factory: application.adapters:make_environment, config: {}} bindings: - {input: x, source: prediction, key: x} - - {input: d, source: environment, key: d} -``` - -Replace the adapter factory with your application's importable `module:factory`. -It must implement the protocol below and provide the bound environment fields. - -Run the existing `sequifier infer --config-path configs/infer.yaml` command. -PT and existing schema-bearing ONNX models use the same coordinator and their -existing inference backends. This feature adds no export format. - -`seed_window_index` is required and selects one prepared model window, using zero-based -dataset/chunk order. Unlike legacy autoregression, selection happens before the -first-subsequence-per-sequence filter. It uses the usual inference window -selection rules, including `window_stride`. An out-of-range index fails. The -seed must have at least one valid position, end in a valid position, and contain -only left padding before its valid observations. Every feature and the attention -mask must have the same context length. Static values come from the existing -processed static feature table and stay constant. - -Each changing input must have exactly one binding. `prediction` keys name model -targets; `environment` keys name adapter outputs. Static inputs cannot have -bindings. Missing producers, duplicate input bindings, unknown predictions, and -extra or missing adapter fields fail clearly. Outputs that are only consumed by -the adapter do not need a feedback binding. - -The seed dataset uses the existing **preprocessed training representation**. -The adapter and stored trajectory use **original units**. The inference wrapper -decodes the seed and forecasts and encodes every successor using the fixed -training `mean`, `std`, and `normalize_real_columns` metadata. It uses the same -`std + 1e-9` transform as preprocessing, without fitting statistics on rollout -values. Float overflow, non-finite values, arrays in place of scalars, and -incompatible model inputs are rejected. Model padding stays zero with a false -attention mask. Features and attention masks slide together, appending a valid -successor position even for the last executed prediction. - -With `rollout` present, inference writes the trajectory to `trajectory_path` -(relative to `project_root` unless absolute). It replaces that file atomically -after successful collection and validation. This path collects one selected -seed with one model; it does not produce legacy flat prediction/probability -files. Omitting `rollout` preserves existing inference output and configuration -rules. - -## Adapter protocol - -Adapters declare `protocol_version = 1` and implement: - -```python -from sequifier.rollout_api import ResetRequest, StepRequest, StepResult - -def reset(self, request: ResetRequest) -> None: ... -def step(self, request: StepRequest) -> StepResult: ... -def close(self) -> None: ... -``` - -A `module:factory` receives `adapter.config` as keyword arguments and is called -once per collection. The adapter stays alive for all steps and closes on success -or failure, including reset, prediction, and result-validation failures. A -factory that fails during construction owns cleanup of any partial resources. -Python callers can pass a supplied adapter instance to `collect_rollout`. - -`reset` receives the episode ID, complete seed history, validity mask, static -inputs, and random seed. It initializes the environment from this history and -must not execute a transition or replace the seed. `step` receives the current -observation, all named forecasts, static inputs, and zero-based step ID. It -returns external fields, one finite scalar reward, and separate Python boolean -`terminated` and `truncated` flags. Reset and step requests contain independent -copies of coordinator data. Adapters must return the final pre-reset successor -and must not autoreset. - -Every prediction executes exactly once before collection can end. A boundary -stops further prediction and stepping. `generation_steps` is a collection -budget: exhausting it leaves `collection_cutoff: true` when the episode is -unfinished, without fabricating termination or truncation. Optional -`rollout.max_episode_steps` is an episode time limit; reaching it records -`truncated: true`. An adapter can also report its own truncation. Both boundary -flags may be true together; termination takes precedence for bootstrap. - -## Trajectory format and Python API - -The version-1 JSON artifact contains the domain seed history and mask, static -values, input bindings, prediction names, episode ID, and ordered transitions. -Each transition records the observation, predictions (the executed forecast -actions), external fields, reward, successor, step ID, and separate boundary -flags. `bootstrap_allowed` is false on termination and true otherwise, including -time-limit truncation with its recorded final successor. The artifact also -contains `total_return`, `collection_cutoff`, and inference provenance, including -the exact preprocessing statistics and seed origin. - -Seed history plus successors reconstruct every later history and attention mask; -they are not duplicated in each record. Artifacts record one episode fragment -and do not promise simulator snapshots or exact resumability. - -```python -from sequifier.rollout_api import load_trajectory, validate_trajectory - -trajectory = load_trajectory("outputs/trajectory.json") -validated = validate_trajectory(trajectory) -``` - -Saving and loading validate format/version, finite scalar data, seed shape and -mask, producer coverage, successor bindings, observation continuity, contiguous -episode/step IDs, total return, bootstrap semantics, and absence of transitions -after an episode boundary. The same validation is available for in-memory data. - -For direct Python collection, a predictor is a callable receiving a copied -`SeedHistory` in original units and returning a dictionary of scalar forecasts. -The collector owns adapter closure. `history_validator`, if provided, validates -the initial history and every successor (including a terminal successor) for -the predictor's representation; the inference integration uses the fixed codec -and existing `ExecutionSchema` here. - -```python -from sequifier.rollout_api import collect_rollout, save_trajectory - -# predictor, adapter, seed and config are provided by your application. -trajectory = collect_rollout( - predictor, adapter, seed, config, prediction_columns=["x"], max_steps=1000, -) -save_trajectory(trajectory, config.trajectory_path) + - {input: disturbance, source: environment, key: disturbance} ``` -Multimodal and categorical observations, reset-only initialization, vectorized -environments, remote transport, learning algorithms, and new export support are -outside this slice. - -## Policy environments and reproducibility - -Policy training, online evaluation and policy rollout use `environment` with -`factory`, `config`, `implementation_id`, and `units` (empty means unitless). -This is a version-3 online environment, distinct from the forecast version-1 -protocol above. Policy rollout requires seeded observation reset; training -requires ordinary reset. Exact resume requires snapshot/restore capabilities. -GRPO also requires reproducible group starts. See the -[policy protocol reference](policy-learning.md#online-adapters). - -The inference route selects the rollout schema. Policy rejects forecast -`bindings`; forecast rejects policy implementation/units declarations. Neither -schema accepts a rollout discriminator. Both use `max_steps`, `trajectory_path`, -and optional `max_episode_steps`. Forecast `seed_window_index` indexes -prepared windows; policy `seed_row_index` indexes raw input rows before sequence -history preparation. Both indices are zero-based and out-of-range values fail -before environment creation. - -Use `enforce_determinism` for execution and policy `action_selection` for mode -versus sampling. A fixed seed can make stochastic actions reproducible under -the same supported runtime and ordering. - - -Both routes accept legacy `rollout.seed_index` as an alias for their respective -index field and reject both names in one declaration. Serialized trajectories -preserve the public metadata key `seed_index` and its original route-specific -units; this configuration rename does not change the output format. +Run `sequifier infer --config-path configs/infer.yaml`. `seed_window_index` selects +one prepared window in dataset/chunk order, following normal stride/window rules +before first-subsequence filtering. Each temporal input needs exactly one producer; +static inputs cannot have bindings. Extra/missing environment fields fail validation. + +Forecast adapters declare `protocol_version = 1` and implement `reset(ResetRequest)`, +`step(StepRequest) -> StepResult`, and `close()`, exported by `sequifier.rollout_api`. +Factories receive `environment.config` as keyword arguments. Reset initializes from +the supplied history without executing a transition; step receives copied current +observations, forecasts and static values. It returns external inputs, a finite reward +and separate Python boolean `terminated`/`truncated` flags. Adapters must not autoreset. + +Adapters and trajectories use original units. Fixed training codecs decode seeds/ +forecasts and encode successors with the same `std + 1e-9` normalization; rollout +never fits statistics. Padding stays zero with a false mask; history and mask slide +together. Unsupported shapes, overflow and non-finite transformed inputs fail. + +## Policy rollout + +Policy rollout uses the [version-3 online adapter](policy-learning.md#online-adapters), +matching `implementation_id`/`units` and seeded `reset_from_observation`. It selects +raw CSV/Parquet rows with `seed_row_index`, rejects forecast bindings, and executes +named policy actions. A seed supplies exactly one current observation; model history, +codecs and normalization come from its artifact. See +[inference options](configs/infer.md#policy-model-interfaces) and the +[Python example](example_inputs/ppo_toy_control.py). + +Both seed indices are required, zero-based, and checked before environment creation. +Legacy `seed_index` aliases the route-specific field; supplying both names fails. +Trajectory metadata retains its public `seed_index` key. No `rollout.type` is accepted. + +## Boundaries and trajectory API + +Every issued prediction/action executes once. True termination disables bootstrap; +truncation retains the final pre-reset successor. `max_steps` is a collection budget: +exhaustion sets `collection_cutoff` without inventing a boundary. Optional +`max_episode_steps` imposes a truncation. Terminal policy steps may omit successors. +The collector closes adapters on success/failure; construction failures remain the +factory's cleanup responsibility. + +Trajectories record seed/mask/static values, bindings, ordered transitions, original +unit predictions/actions/rewards, successors, boundary flags, total return and provenance. +Validation checks continuity, producer coverage, IDs, totals, finite values and boundary +semantics. Policy records also retain joint execution metadata and transform latents. + +`sequifier.rollout_api` exposes `collect_rollout`, `save_trajectory`, `load_trajectory` +and `validate_trajectory`. Direct forecast predictors consume a copied `SeedHistory` +and return named forecasts; optional `history_validator` checks the initial and every +successor history. `sequifier.rl_api.collect_policy_rollout` provides policy collection. +Standalone trajectories do not imply simulator snapshots or exact resumability. diff --git a/documentation/supervised-policy-learning.md b/documentation/supervised-policy-learning.md index 312af8ce..1b4b5857 100644 --- a/documentation/supervised-policy-learning.md +++ b/documentation/supervised-policy-learning.md @@ -1,207 +1,59 @@ # Supervised models and policy learning -Policy learning uses the same model interfaces, input tensors, decoder branches, -categorical codecs, and portable artifacts as supervised learning. A policy -binding identifies the output heads that form an environment action. The -binding does not automatically add inputs or replace a pretrained decoder. - -## Post-train a supervised artifact - -An ordinary supervised PT model, or a supported supervised run checkpoint, can -initialize a policy run. Its interface does not need existing policy metadata. +A policy binding attaches a task to existing model targets. It preserves interfaces, +weights, codecs, normalization, forward attention and decoder geometry; it does +not automatically add inputs or replace a pretrained decoder. Start from an +ordinary supervised PT artifact or supported run checkpoint: ```yaml -project_root: . -model_name: posttrained initialize_from: models/supervised/model.pt policy_binding: actions: mode: {type: categorical, values: ["off", "on"]} price: {type: real, scale: 0.5} -dataset_training: - demonstrations: - parts: - recorded: {type: sequence, paths: [data/demonstrations.parquet], format: table} -training_plan: - phases: - - name: imitate - algorithm: policy_imitation - sources: [{source: demonstrations.recorded}] - epochs: 5 - batch_size: 64 - optimization: - actor: {learning_rate: 0.0003} -``` - -Use the actual interface and target names from the source artifact. Quote YAML -labels such as `"off"` and `"on"` if the YAML parser interprets them as booleans. -Omitting categorical `values` selects the artifact's entire decoder vocabulary, -including any executable special tokens. Explicit values select a subset and -are reported as an adaptation when that changes the original support. - -The artifact provides the architecture, context length, target positions, -normalization, and class maps. A separately supplied model must agree with it. -Original weights load strictly. Only explicitly introduced exploration parameters -are initialized. The run report records the transfer and its new/reused parameters. - -## Real and categorical heads - -Every real action head requires an explicit positive `scale`, measured in the -head's physical units. Its pretrained scalar output is the Gaussian location; -categorical heads retain their logits and persisted decoder-ID semantics. - -| Setting | Meaning | -| --- | --- | -| `scale` | Fixed Gaussian standard deviation by default. | -| `learn_scale: true` | Learn a global log scale, bounded by `min_scale` and `max_scale`. | -| `lower` / `upper` | Transform the Gaussian onto a positive, negative, or finite open interval using a softplus or logistic bijection. | -| `atoms: {0.0: 0.05}` | Assign fixed probability mass to an exact real value, alongside the continuous component. Probabilities must sum to less than one. | -| `active_when: {mode: [on]}` | Activate a field only for these values of an earlier action field. | -| `inactive_value: 0.0` | Deterministic value used when a field is inactive. | - -Bounds change the deterministic prediction function and are reported as an -adaptation. Samples are never clipped and then scored as unclipped Gaussians. -Collection retains transform latents for stable replay near numerical boundaries. -Externally supplied exact endpoints require a declared point mass. Inactive -fields are point masses with zero log-probability contribution, rather than -Gaussians with zero variance. - -The existing decoder determines factorization. Ordinary branches predict -independent fields; an autoregressive branch samples in its existing target -order, embedding each sampled prefix value. Recorded actions are scored with -their recorded prefix. Selecting a later autoregressive target requires selecting -all its predecessors. Composite branches and tied weights are retained. Hashed -categorical outputs are normalized over canonical executable candidates. - -Likelihoods sum over active action fields once per decision. Supervised loss -weights do not alter PPO ratios or DPO likelihoods. Independent categorical and -unbounded Gaussian entropy is exact; dependent/transformed entropy uses a -score-function estimator that includes the sampled prefix distribution. These -estimates can have higher variance. The GRPO sampled reference penalty retains -its existing interpretation. - -All five algorithms consume this joint action contract: imitation, IQL, PPO, -DPO, and GRPO. IQL projects real action fields and embeds categorical fields -before evaluating its scalar twin Q-functions. - -## Inputs, history, and output positions - -The full persisted execution schema describes scalar, static, depth, image, and -mask tensors. Online adapters can return nested JSON values, or array/tensor -payloads that can be converted to portable values. Supply images with the -model's existing crop, channel, frame-capacity, and byte conventions. Depth -arrays and their named masks must match the source execution schema. - -Prepared sources reuse depth masks and the existing image store. Table sources -can provide nested Parquet payloads. Use `columns.actions` to map logical action -names to table columns; `columns.previous_actions` maps preceding recorded actions. -Exported sequence shards preserve named action records and optional real latents. - -No previous-action input is inserted by default. To feed an existing model input -from a lagged action, declare, for example: - -```yaml -policy_binding: - previous_action_inputs: {last_quantity: quantity} - initial_inputs: {last_quantity: 0.0} - actions: - quantity: {type: real, scale: 0.5} -``` - -`initial_inputs` supplies the first decision's lagged value. The existing input's -codec and normalization are retained. Temporal padding, action activity, and -missing training labels are distinct. Complete relevant action tuples are -required for likelihood training; missing predecessors are not silently masked. - -`decision_position` defaults to `-1`. Other negative positions select existing -decoder positions while preserving support and prediction geometry. A model -with several prediction positions does not automatically execute an action block. -Only the selected current decision is executed by these runners. - -Forward attention is persisted separately from the learning objective. A -bidirectional encoder can consume an entirely observed history. Each historical -decision is reconstructed at its actual availability boundary, so later -observations cannot leak into its likelihood. An input unavailable at action time -must be supplied through an explicit task adaptation. - -Return conditioning is opt-in for attached models with a `__policy_return` -input. Offline conditioning needs a known reward tail. Online conditioning needs -`initial_desired_return`; subsequent budgets follow the configured return update. - -## Mixed training plans - -Supervised and policy phases use the same `training_plan.phases` structure. -See the full [supervised-to-policy template](../src/sequifier/templates/train-supervised-to-policy.yaml). -Supervised prepared parts and policy sequence parts both belong to -`dataset_training`; each phase references the appropriate parts. No additional -run discriminator is needed. `global_training` owns execution and model/data -contracts. Each phase owns batching and `optimization.`: - -```yaml -training_plan: - phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential - sources: [{source: observations.train}] - batch_size: 64 - accumulation_steps: 1 - optimization: - model: {optimizer: {name: Adam}, learning_rate: 0.001} - - name: imitation - algorithm: policy_imitation - epochs: 5 - sources: [{source: demonstrations.recorded}] - batch_size: 64 - accumulation_steps: 1 - optimization: - actor: {optimizer: {name: Adam}, learning_rate: 0.0003} ``` -The coordinator preflights model, source and environment contracts and commits -incoming artifacts before native phase execution. Phase-local checkpoints recover -active work; atomic receipts recover completed handoffs. Policy `model_interface` -selects an interface when source references are insufficient to identify it. -All interfaces remain in the model artifact; a shared backbone update can change -other interfaces' predictions. - -`optimizer_state` defaults to `reset`. Mixed-plan continuation is rejected because -artifact handoffs do not preserve optimization state. Native supervised-only and -compatible policy-only plans support explicit continuation under the -[training transition rules](configs/train.md#optimization-across-phases). -`handoff: last | best` selects phase weights; continuation requires the predecessor's -`last` handoff. Mixed plans execute locally. Exact online recovery requires the -adapter's snapshot capabilities. - -The Python API exposes `SequifierConfig`, `ResolvedSequifierConfig`, and -`run_learning_plan` from `sequifier.training_api`. The common entry point -`sequifier.train.run_training` dispatches a resolved plan to its appropriate -runner. `load_policy(..., binding=...)` attaches a task directly to a supervised -artifact; policy bindings never author a second model architecture. - -## Distribution-aware supervised losses and inference - -Existing MSE, MAE, cross-entropy, and hashed-component losses retain their -semantics. An interface may declare `prediction_distributions` for real targets -and select `GaussianNLLLoss` in `dataset_training.criterion`. This uses the same -real distribution implementation as policy likelihoods and stores learned scales -in the model artifact. - -Artifacts retain ordinary point-prediction execution. For an artifact containing -policy metadata, set `execution_mode: prediction` to select ordinary forecast -inference, or `execution_mode: policy` for action sampling. Joint policy -inference writes every executed field; `output_probabilities: true` also writes -the joint action log probability. Deterministic generation uses sequential -locations/argmax choices; it does not claim a global mode of an autoregressive -joint distribution. - -New model artifacts use format version 3; versions 1 and 2 remain readable. -Joint trajectories and PPO collection records use version 2. Existing -single-categorical configuration and inference APIs remain available as -compatibility surfaces. ONNX remains a prediction export, not a general training -checkpoint or an automatically exported stochastic sampler. - -This implementation has not been exercised with tests or training experiments -yet. Probability, architecture-parity, learning, and interruption acceptance -checks remain to be performed before release. +Add datasets/phases as in the [policy guide](policy-learning.md). A separately +declared model must agree with the artifact. Omitted categorical `values` select +the whole decoder vocabulary; an explicit subset is reported as an adaptation. +Real `scale` is required and measured in physical units. `learn_scale` enables a +global log scale bounded by `min_scale`/`max_scale`. `lower`/`upper` use bijections; +`atoms` assigns exact point masses (positive probabilities summing to less than one). +Bounds change the deterministic prediction and count as adaptation. Samples retain +transform latents; external endpoints need declared atoms. `active_when` references +earlier action fields; inactive fields use `inactive_value` with zero likelihood cost. + +Ordinary branches factor independently; autoregressive branches retain target +order and score recorded prefixes. Selecting a later autoregressive action requires +all preceding decoder targets. Composite branches and tied weights are retained. +Likelihoods sum active fields; supervised loss weights never redefine probabilities. +Independent categorical/unbounded Gaussian entropy is exact; dependent/transformed +entropy uses a score-function estimator. Deterministic selection follows sequential +argmax/locations, without claiming a global joint mode. + +Inputs retain the complete scalar/static/depth/image/mask execution schema. +Prepared sources reuse depth masks and image stores; tables may contain nested +Parquet payloads. Explicit `previous_action_inputs` maps existing input names to +actions; `initial_inputs` supplies the first lagged value using its persisted codec. +`decision_position` defaults to `-1` and selects one existing decoder position, +not an action block. Historical likelihoods use only observations available then. +Return conditioning requires an existing `__policy_return` input and known offline +tails or online `initial_desired_return`. + +Supervised and policy phases share `training_plan.phases`; see the +[complete mixed template](../src/sequifier/templates/train-supervised-to-policy.yaml). +Prepared and sequence/preference parts belong to `dataset_training`. Each phase owns +its batching and role optimization (`model` for supervised; policy roles in the guide). +All interfaces survive handoff, though shared backbone updates can affect inactive +interfaces. The coordinator preflights the plan, commits incoming artifacts, and uses +native checkpoints plus atomic completion receipts. Mixed plans execute locally and +reject optimizer continuation; compatible native plans follow the +[continuation rules](configs/train.md#optimization-across-phases). + +Existing supervised losses retain their meanings. Real `prediction_distributions` +and `GaussianNLLLoss` opt into the shared distribution implementation; learned scales +belong to the model artifact. `task: forecast` selects point prediction from a +policy-bound artifact; `task: policy` selects policy execution. See +[inference routing](configs/infer.md#policy-model-interfaces). +Model artifacts write version 3 and read versions 1/2/3; joint trajectories and PPO +collection records use version 2. ONNX remains a prediction export. diff --git a/documentation/supervised-policy-unification-plan.md b/documentation/supervised-policy-unification-plan.md deleted file mode 100644 index 953cc063..00000000 --- a/documentation/supervised-policy-unification-plan.md +++ /dev/null @@ -1,236 +0,0 @@ -# Supervised and policy learning unification plan - -Status: proposed implementation plan, based on the current source and the two supplied analyses. No training implementation or experimental validation is included in this change. - -## 1. Intended outcome - -Unify around one model execution contract, one portable model artifact, and one run plan with algorithm-specific phase runners. Supervised objectives and policy objectives should consume the same model and prediction semantics. Episodes, preferences, critics, and online collection remain explicit algorithm-specific concepts. - -The two completion requirements are: - -1. **Architecture parity:** every architecture currently admitted by supervised learning can participate as an actor in policy learning, including arbitrary combinations of the currently supported scalar real and categorical targets, ordinary and autoregressive decoder branches, multiple interfaces, structured/depth/image ingestion, static conditioning, hashing, and nontrivial decoder support and prediction positions. -2. **Supervised-to-policy transfer:** every reconstructable supervised Sequifier model artifact can initialize a policy phase without first being trained or exported as a policy. Existing compatible prediction weights and preprocessing are retained. Necessary stochastic semantics are supplied explicitly for deterministic real heads. - -“Every architecture” concerns the model's supported execution surface. It does not mean that a dataset automatically contains rewards, preferences, successor states, or observations that are available at action time. Every policy still needs an explicit task binding and the data or environment required by its algorithm. A model consuming information unavailable at decision time needs a declared input adaptation; there is no exact behavior-preserving transfer in that case. - -This plan covers trainable Sequifier artifacts and supported run-checkpoint extraction. ONNX inference artifacts alone do not contain the complete training contract and are not a general source for further training. Retain existing supervised export behavior; specify policy deployment export capabilities separately rather than claiming arbitrary stochastic sampling exports automatically. - -## 2. What the current code establishes - -The supplied analyses correctly identify shared model construction and tensors, with separate policy configuration and execution. The following details affect the proposed work: - -| Boundary | Current implementation | Consequence | -| --- | --- | --- | -| Actor | `rl/policy.py` wraps the ordinary network but synthesizes policy inputs, metadata, and one categorical target. | Replace categorical assumptions without introducing a second architecture resolver. | -| Configuration | `PolicyRunConfig.resolve` permits only the selected interface, one action target, scalar fields, and decoder `support=prediction_length=1`. | Typed actions alone will not deliver architecture parity. | -| Transfer | `rl/learner.py::_initialize_actor` calls `load_policy`, then validates identical policy representation before loading weights. | A shared artifact file format currently does not imply supervised-to-policy compatibility. | -| Real targets | `TargetDecoderBranch` and the autoregressive branch already emit scalar real predictions. | Reuse those predictions as location parameters; do not discard and relearn the heads. | -| Autoregression | `AutoregressiveTransformerDecoderBranch` already supports teacher forcing and greedy generation across ordered targets at one temporal position. | Add stochastic generation to this execution path and score actual prefixes consistently. | -| IQL | `HistoryCritic` already evaluates a supplied categorical action through `nn.Embedding` and emits a scalar Q-value. | Analysis 2 overstates this particular redesign: generalize the action encoder, not an all-actions Q-vector architecture. | -| Environment | `OnlineEnvironmentAdapter.step` already accepts a mapping of action names to domain scalars. | Its action container can carry multiple scalar fields; collection, validation, history, and storage still assume one field. Observation transport must expand for non-scalar model inputs. | -| Trajectories | Records already contain named predictions, but policy validation restricts them to one action head and nonpositive log probabilities. | Version the policy record contract; continuous log densities can be positive. | -| Execution metadata | `artifacts/model_config.py` reconstructs execution using training-objective fields; the factory derives attention behavior from an objective. | Separate persisted forward semantics from the objective used to update parameters. | - -Primary implementation references: [configuration](../src/sequifier/config/train_config.py), [actor](../src/sequifier/rl/policy.py), [distributions](../src/sequifier/rl/distributions.py), [decoders](../src/sequifier/model/decoders.py), [network](../src/sequifier/model/network.py), [critics](../src/sequifier/rl/models.py), [artifact restoration](../src/sequifier/artifacts/model_config.py), and [trajectory validation](../src/sequifier/trajectory/schema.py). - -## 3. The shared contract - -Use this separation: - -```text -Authored model + persisted execution metadata - | - Shared model resolver/factory - | - Named prediction parameters - | - Shared prediction/distribution layer - / \ - Supervised objective Policy task binding - | - Imitation / IQL / PPO / DPO / GRPO - -One run coordinator owns phase transitions, artifacts, and recovery. -Each runner owns its algorithm's batches, collection, and update rules. -``` - -### 3.1 Model execution stays independent of the learner - -Persist the forward contract: named inputs, execution schema, normalization and codecs, attention policy, context geometry, decoder branches, target order, output positions, target transforms, parameter dtypes, and parameter sharing. A phase chooses an objective; it does not implicitly replace those forward semantics. - -Derive legacy forward behavior from existing objective metadata during artifact migration. In particular, do not silently turn every imported model into the current `training_objective="policy"` execution configuration. A bidirectional encoder over an entirely observed history is usable as an actor. Reconstructing old actions with later observations in that encoder would leak information; evaluate each historical decision using its actual observation boundary. BERT-specific masking and next-occurrence losses remain supervised data/objective behavior, not mandatory policy behavior. - -Keep `ComposableTransformerNetwork`, `ModelOutput`, and the existing named tensor boundary. Preserve the public `logits` field for compatibility even though its real-valued entries are prediction parameters. Add optional distribution metadata and parameter access without an immediate naming migration. - -### 3.2 Prediction distributions are reusable model capabilities - -Resolve a prediction specification for each target from existing interface metadata plus optional distribution settings. Types and categorical maps remain authoritative in the existing model metadata; policy configuration references targets and adds execution semantics rather than redefining them. - -For ordinary categorical heads, use categorical logits over persisted decoder IDs. For real heads, the existing scalar output is the location of a Gaussian by default. Supply an explicit positive exploration scale when attaching a distribution to a legacy point predictor; there is no universally appropriate scale across physical units. Initially support fixed per-target scales. Add learned global or state-dependent scales as optional model modules, with positive floors and stable parameterization. They must be owned and serialized by the network, not hidden inside an optimizer or inference wrapper. - -Preserve MSE, MAE, and other existing supervised losses. Adding a policy distribution must not reinterpret all historical supervised training as likelihood training. Gaussian NLL and other distribution-aware supervised losses become additional objectives; imitation is a weighted joint NLL objective using the same prediction contract. Class weights and supervised loss weights are loss reductions, not action-probability definitions. PPO ratios and DPO sequence scores use the true joint likelihood rather than a weighted average of head scores. - -Real domains are explicit: - -- Unbounded real: Gaussian with identity physical transform. -- Positive or finitely bounded real: a declared invertible transform or a support-aware family, with stable inverse and log-Jacobian calculations. Never clip a Gaussian sample and score it as an unclipped Gaussian. -- Exact endpoint actions or deterministic conditional fields: a specified atom/mixture or conditional point mass. An open-interval squashing transform cannot represent exact endpoints. Do not silently move recorded actions off a boundary. - -Freeze model-to-physical transforms across collection, replay, actor, and reference. Persist units and whether a scale is specified in model or physical coordinates. Newly imposed bounds may be incompatible with the pretrained point function; disclose this as adaptation rather than claiming exact preservation. - -### 3.3 Policy binding describes a task, not another model - -Introduce a `PolicyBinding` associated with a named model interface. It specifies: - -- Which targets and output positions form one environment decision. -- Input producers: observed fields, static values, named structured/media payloads, explicitly lagged action fields, and optional desired returns. -- Categorical executable support, real domains, and physical transforms. -- Joint factorization, target order, conditional activity, and legality rules. -- How predictions or sampled actions are fed back into future observations. -- How a decision or explicitly executed action block maps to environment time and rewards. - -Default transfer behavior preserves the source input schema. Previous actions and desired returns are optional bindings; do not append them automatically. If a source already contains the prior target as an input, bind that existing input with the correct lag and codec. For newly authored policies, the current observation/previous-action convention remains a convenient explicit preset. - -Retain all model interfaces. Select a policy interface per phase and retain inactive interfaces and tied parameters in the artifact. Optimizers follow explicit trainable/frozen parameter selection. Updating a shared backbone can change predictions through other interfaces; freezing only their decoders does not preserve their outputs. Offer supervised replay or regularization when retaining those behaviors matters. - -### 3.4 Typed actions and joint probability - -Represent actions internally as named tensors with dtypes, event shape, and explicit validity/activity masks. Preserve field order in the resolved schema; never depend on unordered dictionary iteration for likelihood factorization. Support arbitrary numbers of the real and categorical scalar heads currently available to supervised learning. Preserve event axes in the contract so adding true vector heads later does not require another buffer redesign. - -Provide a distribution execution interface along these lines: - -```python -sample(inputs, generator) -> ActionSample -deterministic(inputs) -> ActionSample -log_prob(inputs, recorded_action, context) -> DecisionScores -entropy(inputs, estimator_config) -> EntropyEstimate -kl(inputs, reference, estimator_config) -> KLEstimate -``` - -`ActionSample` carries model-space values, executed values, activity/support context, joint log probability, and optional latent values for numerically stable transformed-density replay. `DecisionScores` exposes one log probability per decision and optional per-field diagnostics. Keep the batch/decision axes; reduce action event axes exactly once. Use explicit capabilities for exact versus estimated entropy and KL. - -For independent branches, joint log probability is the sum over active fields. For an autoregressive branch, it is the sum of conditional log probabilities given the actual earlier sampled or recorded fields. Composite decoders retain their existing branch factorization. Do not flatten their architecture into one independent output layer. - -### 3.5 Autoregressive and hashed targets are required for parity - -Extend the existing autoregressive decoder with distribution-driven stochastic generation. Feed each sampled real or categorical value through the same value embedding used for teacher forcing. Scoring a recorded tuple must teacher-force its true prefix. Using logits conditioned on greedy prefixes produces the wrong joint likelihood. Share transformer, projection, embeddings, tied weights, and target ordering with supervised execution. - -When a selected action field depends on a non-action predecessor in an autoregressive branch, require a valid observed predecessor binding, or include the predecessor in the sampled probabilistic decision. Do not omit its likelihood while pretending a marginal probability has been computed. A general marginal over unobserved predecessors is additional inference work. - -For hashed categorical targets, reuse the existing collision-checked codebook and candidate scores. Normalize over the legal canonical action candidates with `logsumexp`. The product of hash-component probabilities is generally not already normalized over that candidate subset. Sample and score the same canonical candidates; do not sample arbitrary hash tuples and resolve them to a different action. Exact chunked normalization can control memory without changing semantics; approximate retrieval is a separate, explicitly limited mode. Keep supervised component losses intact. - -For dependent distributions, joint entropy requires an expectation over prefixes. Summing conditional entropies at an arbitrary recorded prefix is not generally joint entropy. Specify and validate any sampling-based estimator and its gradient, including real sampled prefixes, before enabling an entropy bonus. Preserve the distinction between a sampled reference penalty and exact KL; do not relabel the existing GRPO penalty as universally exact. - -## 4. Supervised artifact ingestion - -Add a policy attachment path to the common artifact loader: - -1. Restore the complete source network and resolved execution contract using the ordinary artifact machinery, without requiring `interface.policy`. -2. Select the source interface and resolve explicit action/input bindings against it. The artifact is authoritative for architecture, codecs, target ordering, preprocessing, and existing weights. If a run also supplies these declarations, validate agreement instead of building an unrelated new model. -3. Attach distributions to existing target outputs. Reuse categorical heads and real location heads. Initialize only declared additions, such as scale parameters, with reproducible seeds. -4. Load all original model parameters and buffers strictly. Permit only an enumerated set of newly introduced keys; do not use broad `strict=False` loading to hide mismatches. Verify tied/shared parameter identity as well as tensor values. -5. Emit a transfer report listing reused and new parameters, bindings, transforms, support restrictions, precision conversions, and any changed execution semantics. Compare source and attached-model prediction parameters on fixed inputs before training. -6. Export a normal model artifact containing the optional binding and distribution contract. Ordinary forecast inference and policy inference select their respective views of the same network. - -Distinguish two transfer modes: - -| Mode | Guarantee | Examples | -| --- | --- | --- | -| Attach | Original prediction function is retained, within declared numerical tolerance. Stochastic sampling is newly defined where necessary. | Existing real output becomes a Gaussian location with fixed scale; existing logits parameterize a categorical policy. | -| Adapt | Compatible weights are retained, but function equivalence is not promised. Every changed module or semantic choice is reported. | Adding action-history inputs, replacing an unusable input, changing attention visibility, restricting executable categories, imposing incompatible bounds, or changing context geometry. | - -Exact categorical transfer requires the same executable decoder support. Filtering special tokens changes the distribution even if logits are unchanged. For unbounded Gaussian heads with the original physical transform, the deterministic location reproduces the old point prediction. For an autoregressive policy, deterministic sequential predictions reproduce the old greedy path when support and transforms are unchanged; this does not imply that the greedy tuple is the global joint mode. - -Preserve source normalization by default. Never refit it merely because new policy episodes were supplied. Real action normalization must be reversible and shared by likelihood evaluation, critic action encoding, execution, and history. Sampling and scoring must remain consistent when dtypes are converted; replace the present unconditional FP32-weights rejection with an explicit precision policy while retaining stable probability arithmetic and deterministic compatibility checks. - -Initialization is distinct from exact resume. Fresh policy phases create new optimizer, critic, reference, sampler, and collector state according to the phase plan. Exact resume requires the complete run checkpoint and matching task/data/environment/distribution identities. Existing extraction of model state from supported run checkpoints should feed the same initialization path. - -## 5. Input, data, and time semantics - -Build a shared input materializer from the supervised preprocessing, execution schema, and window resolver. It accepts named raw values or explicitly model-encoded tensors and applies each transform exactly once. Policy and supervised source adapters call it instead of maintaining separate normalization or scalar-only encoding rules. - -Generalize history and episode representations to carry named previous-action fields and the full input payload required by the selected interface. Reuse existing depth/image/static tensor and mask contracts. Extend environment observations, snapshots, datasets, collation, memory accounting, and archival records accordingly. A symbolic media reference is sufficient only if replay can recover the identical content and transforms; persist payload identities or immutable materializations. Do not push images or depth arrays through `DomainValue` scalar validators. - -Keep source adapters distinct for supervised windows, episodes/transitions, preferences, and online collection. Share codecs, feature materialization, masks, and batching helpers. Rewards and terminal semantics cannot be inferred from arbitrary supervised windows. Require an explicit source mapping for them and preserve episode-level train/validation separation. - -Separate three masks: temporal padding, observed training labels, and semantically active action fields. Missing labels are not inactive actions. A missing predecessor in an autoregressive tuple cannot simply be masked out while treating subsequent terms as a complete joint likelihood. For likelihood-based phases, require complete relevant actions unless a supported marginalization method is explicitly selected. - -Remove the assumption that every policy decision is the last position of a `support=1` decoder. Preserve the source decoder's receptive field, padding requirements, and output-position semantics. The default binding selects the declared current decision position and leaves other outputs available for supervised losses. Existing `prediction_length` identifies output positions; it must not automatically become a future action chunk. - -Action blocks are an explicit extension: define each slot's execution time, whether intermediate observations are consumed, joint likelihood, interruption semantics, and reward/bootstrap aggregation before supporting them. For IQL, a multi-step action block needs matching offline transitions and duration-aware discounting. Selecting one current action from a model that emits several positions does not require block execution. - -The same availability check applies to bidirectional models and overlapping offline windows: each scored decision sees only what the policy would actually receive then, including any supplied masked-query token, static payload, or return-conditioning value. Return-conditioned actors remain loadable, but non-imitation phases need an explicit rule supplying and updating that input; do not silently remove the trained return feature. - -## 6. Algorithm changes - -| Algorithm | Required work | Acceptance condition | -| --- | --- | --- | -| Imitation | Use shared joint NLL over typed actions, with explicit decision reductions and activity masks. Reuse the objective in supervised training when likelihood training is requested. | Independent mixed-head NLL matches a manual sum; autoregressive NLL matches recorded-prefix scoring. Existing MSE/CE training remains valid. | -| PPO | Replace direct `CategoricalActionDistribution` construction in collection/update with distribution execution. Store exact action values, old joint log probability, support/activity context, and required transform latents. Keep V state-only. | Re-scoring a frozen collected actor gives ratio 1 within tolerance for every supported factorization and domain. No sampling/scoring mismatch at bounds. | -| GRPO | Apply the same action and score contract throughout group collection, buffers, reference scoring, and replay. Preserve group-start capabilities and trajectory-versus-decision weighting. | Old and reference scores replay consistently; equal actor/reference produces zero reference penalty; mixed actions execute in the environment. | -| DPO | Sum decision joint log likelihoods on preferred/rejected spans for actor and frozen reference. Persist identical data transforms, binding, support, and conditional masks. | Actor/reference sequence log ratios match manual mixed categorical/density calculations; no category-only or nonpositive-score assumptions remain. | -| IQL | Replace the single categorical action embedding with a schema-driven action encoder: categorical embeddings, transformed real projections, and explicit activity indicators, fused with the state representation. Retain scalar twin Q(h,a), V(h), target networks, and weighted actor log likelihood. Generalize state inputs and pooling. | Every action field reaches Q with finite gradients; Bellman/expectile/actor updates work on mixed actions; actor loss uses the same likelihood as imitation. | - -Critics can continue to own independent copies of the supported input/backbone architecture initially. Choose pooling through a declared state-representation adapter rather than hard-coding the last unmasked slot for every layout. Reusing the actor encoder is a later efficiency choice with explicit gradient and optimizer ownership; it is not necessary for unification. - -Begin continuous imitation, DPO, and weighted IQL actor fitting with fixed scales. Unrestricted learned scale can distort preference or weighted-likelihood fitting, and variance can collapse or expand without improving the intended behavior. Introduce learned-scale modes with explicit bounds/regularization and diagnostic acceptance criteria. PPO/GRPO can support controlled learned exploration once probability replay is established. Reward improvement is a benchmark goal, not a universal mathematical guarantee. - -## 7. Unified configuration and lifecycle - -Introduce one canonical run model with model/artifact initialization, named typed sources, policy bindings, execution settings, recovery settings, and ordered discriminated phases. Preserve algorithm-specific settings and budgets: supervised/imitation epochs, offline optimizer updates, PPO environment steps, and GRPO groups. References, critics, and rollout state remain phase-local. - -Migrate existing supervised and policy YAML/API forms into that canonical representation with compatibility adapters. A legacy single-categorical policy expands into a one-field binding with its existing previous-action and optional return inputs. Preserve its categorical IDs, start token, transforms, phase defaults, and scoring behavior. Avoid simultaneous wholesale renaming of public configuration keys. - -The coordinator performs whole-plan validation, resolves the model once, preflights task/data/environment capabilities, executes runners, selects best/last handoff state, and owns the common checkpoint envelope. Shared services cover parameter catalogs/freezing, optimizer transactions, scheduling, precision, metrics, RNG, atomic artifacts, and callbacks. Adapt the existing supervised and policy services incrementally; do not force RL collection and multiple optimizer roles into an epoch loop. - -Use an explicit capability matrix for execution: model architecture parity must not be confused with immediate parity across DDP/FSDP, compilation, integrations, or every device/dtype combination. Preserve existing supervised execution modes while adding policy capabilities with targeted validation. Distribute offline learners before attempting distributed environment collection if distributed policy execution is in scope. - -Each phase declares trainable modules and optimizer roles, sources, evaluation and monitor direction, budget units, and handoff behavior. A supervised phase can interleave its current datasets as before. Sequential supervised → RL → supervised phases are part of this plan; arbitrary simultaneous PPO plus supervised replay requires an explicit auxiliary-objective/update policy and is a subsequent extension, not an implicit meaning of interleaving phases. - -Default to fresh optimizer state at algorithm boundaries. Any continuation must name a compatible parameter/state mapping. Fresh policy references are captured at the specified phase boundary or loaded from a compatible artifact. Best/last selection restores associated actor distribution parameters and transforms, not only backbone weights. - -The checkpoint envelope contains model and binding identity, phase index, committed optimizer clocks, per-role optimizer/scheduler/scaler state, references/targets, sampler/RNG state, source fingerprints, rollout state, environment snapshot identity, and collection generation. Exact resume is guaranteed only where the environment exposes the necessary snapshot capabilities; reset-environment recovery remains a distinct mode. Rejected/stale buffers must not be consumed after a handoff or distribution change. - -## 8. Delivery sequence and completion gates - -These are implementation slices, not duration estimates. Several are substantial changes and should be split further when the first prototype exposes integration costs. - -| Slice | Deliverable and principal locations | Completion gate | -| --- | --- | --- | -| 1. Contracts and baselines | Inventory the currently admitted model combinations; specify forward semantics, prediction distributions, action bindings, and transfer reports. Start in `config`, `model`, and `artifacts`. | Representative supervised and legacy-policy fixtures establish current behavior; every supervised feature has a parity acceptance case. | -| 2. Minimal end-to-end transfer | Shared artifact attachment and a thin policy runtime for an unchanged ordinary supervised interface, one categorical target or one unbounded real location with fixed scale. Touch `rl/checkpoint.py`, `learner.py`, `policy.py`, and distribution resolution. | A real-only supervised artifact with no policy metadata completes a policy-imitation and PPO update without replacing its input or location head. A categorical artifact does the same. | -| 3. Mixed actions throughout the pipeline | Named action tensors, independent multiple heads, reversible transforms, datasets, history, trajectory schema, collectors, buffers, inference, and scalar environment execution. Add joint NLL. | A categorical-plus-two-real actor completes imitation and PPO collection/update, persists, reloads, and reproduces collection scores. Legacy single-categorical behavior still matches. | -| 4. Existing algorithm migration | Generalize GRPO and DPO sequence scoring and IQL action encoding against the established action contract. | All five algorithms train mixed-action fixtures; no algorithm directly instantiates a categorical-only distribution. | -| 5. Decoder architecture parity | Add stochastic autoregressive generation, recorded-prefix scoring, branch composition, tied heads, hashed candidate normalization, and conditional activity. | Sampled and teacher-forced tuple scores agree, normalization checks pass, and original decoder sharing survives reload. | -| 6. Input and geometry parity | Reuse the full supervised materializer and execution schema across policy data, environment observations, history, critics, and recovery. Remove scalar-only, one-interface, and unit-support restrictions after their acceptance cases pass. | Structured/depth/image/static/composite input fixtures, hashed inputs, multiple interfaces, nontrivial support, prediction-position selection, and observed-history bidirectional models train as actors. | -| 7. Common run plan | Compatibility loaders and coordinator around existing runners; then consolidate optimizer/metrics/checkpoint services. Touch `train.py`, `training/engine.py`, `rl/learner.py`, runtime and recovery modules. | A single supervised → PPO → supervised plan works, and interrupted phase transitions resume without duplicate updates or reinitialization. | -| 8. Compatibility and release | Versioned artifacts, trajectories and environment payload capabilities; public API/templates/docs; targeted execution-mode validation. | Legacy artifacts/configurations/records load through tested adapters; unsupported recovery versions fail explicitly; all mandatory parity cases pass. | - -Dependencies: slices 2–4 establish a usable supervised-to-RL path before broad lifecycle work. Decoder and input parity both build on the shared contracts and mixed-action representation. Slice 7 can begin with the compatibility schema earlier, but it is not a prerequisite for exporting a supervised artifact and post-training it in a separate run. The release is not architecture-complete until slices 5 and 6 are finished; “multiple independent scalar heads” is an intermediate milestone. - -## 9. Validation required for completion - -Use small deterministic fixtures for probability and recovery invariants, then a few controlled learning tasks. Do not rely solely on whether training losses decrease. - -1. **Transfer invariants:** strict reuse of source parameters/buffers and sharing; fixed-input predictions match before updates; explicit scale additions; unchanged normalization and categorical maps; artifact save/reload preserves both inference views. Include a model with no policy metadata and a model extracted from a supported supervised run checkpoint. -2. **Probability correctness:** manual independent sums; tiny enumerated autoregressive categorical joint normalization; numerical continuous density/transform checks; positive log-density acceptance; Gaussian location/scale gradients; hashed candidate normalization; boundary and conditional-atom behavior; padding/activity masks and missing-label rejection. -3. **Collection/replay equivalence:** seeded sample reproducibility on supported devices; sampled-prefix versus recorded-prefix scores; PPO/GRPO initial ratio 1; reference equality; immutable support and preprocessing; action-history lag; full-precision archived action/latent round trips. -4. **Architecture matrix:** every currently supported ingestion and decoder family plus feature combinations that change execution, including static prefixes, depth/media masks, parameter tying, multiple interfaces, support > 1, and multiple prediction positions. Test full likelihood and a gradient update, not just network construction. Cover ordinary/tabular and non-scalar inputs in critic execution as well. -5. **Algorithm behavior:** mixed imitation improves held-out likelihood; a deterministic toy mixed-action control task demonstrates PPO/GRPO updates and executable actions; DPO improves preferred relative log ratios; IQL has finite meaningful Q/V and actor updates without an action-count vector assumption. Keep reward and likelihood metrics separate. -6. **Lifecycle and compatibility:** supervised → RL → supervised handoff; stop/resume at committed updates and phase boundaries; best/last selection including scales; stale-buffer rejection; source/environment identity changes; legacy categorical config/artifact/trajectory adapters; explicit limits on exact recovery of old online checkpoints. - -Baseline performance checks should measure autoregressive sampling cost, hashed candidate scoring cost, and buffer bytes for structured inputs. Correctness is mandatory; use those results to prioritize caching or packing without changing the probability contract. - -## 10. Recommended scope decisions - -Proceed with these defaults unless a task explicitly requires something else: - -- Preserve pretrained interfaces and preprocessing on attachment; do not inject new history features by default. -- Use existing real heads as locations with explicitly configured fixed scales first. Add learnable scales without changing existing point-prediction APIs. -- Deliver independent mixed heads as the first broad action-space milestone, followed by the existing autoregressive and composite architecture surface before claiming parity. -- Support current-decision output selection first while preserving larger decoder support/prediction geometry. Add true action-block execution under a separate temporal contract. -- Represent conditional activity and deterministic values in the action contract now; implement their explicit probability semantics before advertising dependent constrained actions. General user-defined constraint solvers and arbitrary rejection/projection sampling are deferred. -- Keep algorithms and source adapters distinct. Consolidate their shared execution and lifecycle services around the contracts they actually share. -- Require a usable task binding and rewards/preferences/interaction source. Artifact ingestion makes a model trainable; it cannot manufacture the learning task. - -The first implementation should prove an unchanged supervised real-valued actor can be attached, sampled, scored, and updated by PPO. That exercises both overarching requirements immediately and prevents the work from stopping at a more generic policy schema that still cannot ingest an ordinary supervised checkpoint. diff --git a/hooks/consolidate_docs.py b/hooks/consolidate_docs.py deleted file mode 100644 index da497ceb..00000000 --- a/hooks/consolidate_docs.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -import sys -from pathlib import Path - -FILES_TO_READ = [ - "README.md", - "documentation/configs/preprocess.md", - "documentation/configs/train.md", - "documentation/configs/infer.md", - "documentation/commands/visualize-training.md", - "documentation/configs/hyperparameter-search.md", - "documentation/training/multi-gpu-training.md", - "documentation/policy-learning.md", - "documentation/supervised-policy-learning.md", - "documentation/rollout.md", -] - -OUTPUT_FILE = "src/sequifier/consolidated-docs.md" - - -def main(): - consolidated_content = [] - - for filepath in FILES_TO_READ: - path = Path(filepath) - if not path.is_file(): - print(f"Error: Required file not found - {filepath}") - sys.exit(1) - - consolidated_content.append(path.read_text(encoding="utf-8")) - - final_content = "\n\n".join(consolidated_content) - - output_path = Path(OUTPUT_FILE) - current_content = "" - if output_path.is_file(): - current_content = output_path.read_text(encoding="utf-8") - - if current_content != final_content: - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(final_content, encoding="utf-8") - print(f"Hook updated {OUTPUT_FILE}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..e000c575 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +"""Build standalone package resources from their canonical sources.""" + +from pathlib import Path +from runpy import run_path + +from setuptools import setup +from setuptools.command.build_py import build_py + + +class BuildPy(build_py): + def run(self): + super().run() + root = Path(__file__).resolve().parent + documentation = run_path(str(root / "src/sequifier/docs.py")) + destination = Path(self.build_lib) / "sequifier/consolidated-docs.md" + destination.write_text( + documentation["source_documentation"](root), encoding="utf-8" + ) + + +setup(cmdclass={"build_py": BuildPy}) diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 99c86bf3..7caa996c 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -5,11 +5,9 @@ from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.train_config import ( BackboneComponentConfig, - DatasetFreezingSpecModel, GlobalTrainingSpecModel, ModelInterfaceSpecModel, ModelSpecModel, - ResolvedDatasetTrainingSpec, ResolvedModelInterface, ResolvedSequifierConfig, ) @@ -205,38 +203,9 @@ def resolved_config_from_model_config( def execution_config(model, interfaces, global_training, *, device="cpu", **options): - """Compose the same execution contract for loaded and newly resolved models.""" - return ResolvedSequifierConfig( - global_training=global_training, - model=model, - dataset_training={ - name: ResolvedDatasetTrainingSpec( - name=name, - model_interface=name, - interface=interface, - parts={}, - criterion={ - target: "CrossEntropyLoss" if kind == "categorical" else "MSELoss" - for target, kind in interface.target_column_types.items() - }, - class_share_log_columns=[], - freeze=DatasetFreezingSpecModel(), - ) - for name, interface in interfaces.items() - }, - **{ - "project_root": ".", - "model_name": "loaded-model", - "seed": 0, - "device": device, - "training_plan": [], - "evaluation_sources": [], - "evaluation_monitor": None, - "export_generative_model": True, - "export_embedding_model": False, - "export_onnx": False, - "export_pt": False, - "embedding_layer_names": ["backbone.final_norm"], - **options, - }, + """Adapt model execution to the existing public resolved training contract.""" + from sequifier.model.execution import ModelExecution + + return ModelExecution(model, interfaces, global_training).training_config( + device=device, **options ) diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index e9f45577..a84ab155 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -9,13 +9,12 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: if not hasattr(training_config, "dataset_training"): raise TypeError("Model export requires a canonical training config") + from sequifier.model.execution import ModelExecution from sequifier.model.execution_schema import ExecutionSchema + execution = ModelExecution.from_config(training_config) interfaces = {} - for dataset in training_config.dataset_training.values(): - interface = dataset.interface - if interface.name in interfaces: - continue + for interface in execution.interfaces.values(): values = interface.model_dump( mode="python", exclude={ diff --git a/src/sequifier/artifacts/phase.py b/src/sequifier/artifacts/phase.py new file mode 100644 index 00000000..a3414f23 --- /dev/null +++ b/src/sequifier/artifacts/phase.py @@ -0,0 +1,60 @@ +"""Paths, identity and publication records at committed phase boundaries.""" + +import json +from dataclasses import dataclass +from pathlib import Path + +from sequifier.fingerprints import fingerprint +from sequifier.io.atomic import write_json_atomic +from sequifier.trajectory.sources import file_checksum + + +@dataclass(frozen=True) +class PhaseArtifacts: + model: Path + + @classmethod + def for_phase(cls, directory, index, name, filename="actor.pt"): + return cls(Path(directory) / f"{index:03d}-{name}" / filename) + + @property + def report(self): + return self.model.with_name("report.json") + + @property + def receipt(self): + return self.model.with_name("completed.json") + + def describe(self): + return {"actor_path": str(self.model), "report_path": str(self.report)} + + def record(self, report): + return { + "sha256": file_checksum(self.model), + "result_fingerprint": fingerprint(report), + } + + def read_receipt(self, identity): + if not self.receipt.is_file(): + return None + receipt = json.loads(self.receipt.read_text()) + if receipt["identity"] != identity or receipt["sha256"] != file_checksum( + self.model + ): + raise ValueError("completed phase artifact or receipt changed") + return receipt + + def write_receipt(self, identity, report): + write_json_atomic( + self.receipt, + { + "identity": identity, + "sha256": file_checksum(self.model), + "report": report, + }, + ) + + +def verify_artifact(path, checksum, message): + if path and file_checksum(path) != checksum: + raise ValueError(message) diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index a9e9e18a..33de83c3 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -491,7 +491,7 @@ def load_inferer_config( selected_metadata = part.metadata selected_metadata_path = part.metadata_config_path - from sequifier.artifacts.model_config import resolved_config_from_model_config + from sequifier.config.inference_common import SelectedInferenceModel autoregressive_decoder_targets: set[str] = set() if isinstance(model_paths, list): @@ -507,11 +507,14 @@ def load_inferer_config( and not route_selected ): authored_values["training_config_path"] = None - training, interface_name = resolved_config_from_model_config( + selection = SelectedInferenceModel.select( + model_path, model_config, - device=str(authored_values.get("device", "cpu")), interface_name=authored_values.get("model_interface"), + task="forecast", ) + training = selection.resolve(str(authored_values.get("device", "cpu"))) + interface_name = selection.interface_name dataset = training.dataset_training[interface_name] decoder = dataset.interface.decoder branches = ( diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index a7caeeda..9ce3b4cd 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -1,6 +1,7 @@ """Execution options consumed by every inference route.""" import json +from dataclasses import dataclass from typing import Literal from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -56,6 +57,49 @@ def validate_task(self): return self +@dataclass(frozen=True) +class SelectedInferenceModel: + path: str + values: dict + interface_name: str + + @classmethod + def select( + cls, path, values, *, interface_name=None, model_type="generative", task=None + ): + interfaces = values.get("interfaces") + if not isinstance(interfaces, dict) or not interfaces: + raise ValueError("model_config.interfaces must be a non-empty mapping") + if interface_name is None: + policies = [ + name + for name, interface in interfaces.items() + if interface.get("policy") is not None + ] + if model_type == "policy" and len(policies) == 1: + interface_name = policies[0] + elif len(interfaces) > 1 and policies and task != "forecast": + raise ValueError( + "Artifact has multiple interfaces; select model_interface before choosing task" + ) + else: + interface_name = next(iter(interfaces)) + if interface_name not in interfaces: + raise ValueError(f"Unknown model_interface {interface_name!r}") + return cls(path, values, interface_name) + + @property + def policy_bound(self): + return self.values["interfaces"][self.interface_name].get("policy") is not None + + def resolve(self, device): + from sequifier.artifacts.model_config import resolved_config_from_model_config + + return resolved_config_from_model_config( + self.values, device=device, interface_name=self.interface_name + )[0] + + def inference_model_config(path, project_root, artifacts): """Read the interface contract once for PT artifacts, or from ONNX metadata.""" from sequifier.artifacts.model_artifact import read_execution_artifact diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index da96b219..829ee736 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -8,6 +8,7 @@ from sequifier.config.inference_common import ( InferenceExecutionConfig, InferenceRouteConfig, + SelectedInferenceModel, inference_model_config, ) from sequifier.config.policy_config import PolicyConfigBase @@ -89,7 +90,7 @@ class ResolvedPolicyInferenceConfig(PolicyInferenceConfig): def _policy_inference_route( route: InferenceRouteConfig, artifacts: dict[str, ModelArtifact] -) -> tuple[str, str] | None: +) -> SelectedInferenceModel | None: """Inspect artifact interfaces before applying either inference schema.""" if route.model_type == "embedding": return None @@ -97,34 +98,20 @@ def _policy_inference_route( [route.model_path] if isinstance(route.model_path, str) else route.model_path ) bindings = [] - selected = route.model_interface + selection = None for path in paths: model_config = inference_model_config(path, route.project_root, artifacts) if model_config is None: bindings.append(False) continue - interfaces = model_config.get("interfaces") - if not isinstance(interfaces, dict) or not interfaces: - raise ValueError("model_config.interfaces must be a non-empty mapping") - name = route.model_interface - if name is None: - policy_names = [ - key - for key, interface in interfaces.items() - if interface.get("policy") is not None - ] - if route.model_type == "policy" and len(policy_names) == 1: - name = policy_names[0] - elif len(interfaces) > 1 and policy_names and route.task != "forecast": - raise ValueError( - "Artifact has multiple interfaces; select model_interface before choosing task" - ) - else: - name = next(iter(interfaces)) - if name not in interfaces: - raise ValueError(f"Unknown model_interface {name!r}") - bindings.append(interfaces[name].get("policy") is not None) - selected = name + selection = SelectedInferenceModel.select( + path, + model_config, + interface_name=route.model_interface, + model_type=route.model_type, + task=route.task, + ) + bindings.append(selection.policy_bound) task = "policy" if route.model_type == "policy" else route.task if task is None: if any(bindings) and not all(bindings): @@ -143,8 +130,8 @@ def _policy_inference_route( "task: policy requires a portable .pt artifact; " "use task: forecast for forecast inference" ) - assert selected is not None - return paths[0], selected + assert selection is not None + return selection def policy_inference_config( @@ -152,7 +139,6 @@ def policy_inference_config( ) -> ResolvedPolicyInferenceConfig | None: """Dispatch from the persisted interface contract, without training YAML.""" route = InferenceRouteConfig.model_validate(values) - from sequifier.artifacts.model_config import resolved_config_from_model_config from sequifier.config.infer_config import ( _assert_metadata_matches_source, _execution_source, @@ -163,13 +149,9 @@ def policy_inference_config( selection = _policy_inference_route(route, artifacts) if selection is None: return None - model_path, selected = selection + model_path, selected = selection.path, selection.interface_name request = PolicyInferenceConfig.model_validate(values) - execution, _ = resolved_config_from_model_config( - artifacts[model_path].model_config.values, - device=request.device, - interface_name=selected, - ) + execution = selection.resolve(request.device) policy = execution.interface(selected).policy if policy is None: raise ValueError("selected interface has no policy") diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 1d8e469f..af067688 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -31,6 +31,41 @@ def source_parts(config, reference): ] +class TrainingSources: + """Named authored parts shared by metadata fitting and run resolution.""" + + def __init__(self, config, root): + self.root = root + self.parts = { + f"{dataset_name}.{part_name}": (dataset.model_interface, part) + for dataset_name, dataset in config.dataset_training.items() + for part_name, part in dataset.parts.items() + } + + def resolve(self, *, prepared=True): + sequences, preferences = {}, {} + for ref, (_, part) in self.parts.items(): + if isinstance(part, PreferencePartSpecModel): + path = self.root / part.manifest + preferences[ref] = str(path.resolve() if prepared else path) + continue + if isinstance(part, SequencePartSpecModel): + source = SequenceSourceConfig.model_validate( + part.model_dump(exclude={"type"}) + ) + elif prepared and isinstance(part, DatasetPartSpecModel): + source = SequenceSourceConfig( + metadata_config_path=part.metadata_config_path, + split_index=0, + format="prepared", + encoding="model", + ) + else: + continue + sequences[ref] = resolve_sequence_source(source, self.root) + return sequences, preferences + + def _resolve_phase_sources(config, phase, available, *, strict=True): """Expand named sources for both metadata fitting and final run resolution.""" preference = phase.algorithm == "dpo" @@ -209,19 +244,7 @@ def _fit_input_metadata(config, name, binding, metadata, root): }, input_schema={}, ) - sources, preferences = {}, {} - for dataset_name, dataset in config.dataset_training.items(): - for part_name, part in dataset.parts.items(): - ref = f"{dataset_name}.{part_name}" - if isinstance(part, SequencePartSpecModel): - sources[ref] = resolve_sequence_source( - SequenceSourceConfig.model_validate( - part.model_dump(exclude={"type"}) - ), - root, - ) - elif isinstance(part, PreferencePartSpecModel): - preferences[ref] = str(root / part.manifest) + sources, preferences = TrainingSources(config, root).resolve(prepared=False) phase = phase.model_copy(deep=True) if not _resolve_phase_sources( config, @@ -443,29 +466,7 @@ def resolve_training_run(authored: SequifierConfig): if interface.policy is not None: resolved.policy_bindings.setdefault(name, interface.policy) - sequences, preferences = {}, {} - for dataset_name, dataset in config.dataset_training.items(): - for part_name, part in dataset.parts.items(): - ref = f"{dataset_name}.{part_name}" - if isinstance(part, PreferencePartSpecModel): - preferences[ref] = str((root / part.manifest).resolve()) - elif isinstance(part, SequencePartSpecModel): - sequences[ref] = resolve_sequence_source( - SequenceSourceConfig.model_validate( - part.model_dump(exclude={"type"}) - ), - root, - ) - elif isinstance(part, DatasetPartSpecModel): - sequences[ref] = resolve_sequence_source( - SequenceSourceConfig( - metadata_config_path=part.metadata_config_path, - split_index=0, - format="prepared", - encoding="model", - ), - root, - ) + sequences, preferences = TrainingSources(config, root).resolve() for phase in config.training_plan.phases: phase.optimization = phase.optimization_configs() if phase.algorithm == "supervised": diff --git a/src/sequifier/consolidated-docs.md b/src/sequifier/consolidated-docs.md deleted file mode 100644 index 9f2fe0a5..00000000 --- a/src/sequifier/consolidated-docs.md +++ /dev/null @@ -1,3384 +0,0 @@ -

- Sequifier -

- -# What is sequifier? - -Sequifier is the short, efficient, scalable path from tabular sequences to your own transformer model. - -Its three core commands, `preprocess`, `train`, and `infer`, are configurable. -Preprocessing can use multiple CPU workers for eligible workloads, and training -supports multiple GPUs. - -They enable you to go from multivariate sequence data to a model that ingests such data, and emits either (a) this data, (b) a subset, (c) other variables or (d) embeddings. - -If input and target variables are the same, it supports full autoregressive inference. - -Through configuration, most modern architectural variants such as RoPE, GQA, SwiGLU, and RMSNorm are supported. The ambition is to follow the frontier when industrial-grade implementations become available. - -The process looks like this: - -

- Sequifier preprocessing, training, and inference workflow -

- -## Examples from the literature - -Multivariate tabular transformers have *many* applications. - -These independent references illustrate some of them; inclusion does not imply that the cited work used or is affiliated with sequifier. - -**Finance** - -- Transaction Foundation Models ([Overview](https://sistemalabs.com/blog/shaping-the-problem), [EWE-1](https://sistemalabs.com/blog/introducing-ewe-1)) -- Order Book Transformers ([TransLOB](https://arxiv.org/abs/2003.00130)) -- Volatility Forecasting ([paper](https://www.sciencedirect.com/science/article/abs/pii/S0952197624003816)) -- Macroeconomic Models ([BISTRO](https://www.bis.org/publications/bistro-general-purpose-oracle-macroeconomic-time-series)) - -**Health/Bio** - -- Health Records ([BEHRT](https://www.nature.com/articles/s41598-020-62922-y), [Med-BERT](https://www.nature.com/articles/s41746-021-00455-y)) -- Treatment Outcome Prediction ([G-Transformer](https://arxiv.org/abs/2406.05504)) -- Health Trajectory Prediction ([ETHOS](https://www.nature.com/articles/s41746-024-01235-0)) -- Glucose Forecasting ([GluForecast](https://arxiv.org/html/2606.18640v1)) -- ECG rhythm/arrhythmia monitoring ([Heart Language Model](https://www.nature.com/articles/s41598-024-84270-x)) - -**Cybersecurity** - -- Network Intrusion Detection ([paper](https://arxiv.org/pdf/2309.01070)) -- Encrypted Traffic Analysis ([Criss-Cross Traffic Transformer](https://ieeexplore.ieee.org/document/11396346)) -- Insider Threat Detection ([paper](https://www.nature.com/articles/s41598-025-12063-x)) -- In-Vehicle Intrusion Detection ([paper](https://www.sciencedirect.com/science/article/abs/pii/S0950705124007251)) -- IoT Anomaly Detection ([paper](https://www.sciopen.com/article/10.32604/cmc.2024.053765)) -- System Logs Anomaly Detection ([DeepEAD](https://ieeexplore.ieee.org/document/10279563)) - -**Industrial & IoT** - -- Remaining Useful Life Estimation ([paper](https://www.sciencedirect.com/science/article/abs/pii/S095219762400633X)) -- Fault Diagnosis ([paper](https://www.sciencedirect.com/science/article/abs/pii/S0957582026001485)) -- Anomaly Detection for Industrial Control Systems ([paper](https://onlinelibrary.wiley.com/doi/10.1155/2024/5459452)) -- Soft Sensing ([Debutanizer](https://www.sciencedirect.com/science/article/abs/pii/S1876107024003249)) -- Battery State-of-Health Estimation ([DS-transformer](https://www.nature.com/articles/s41598-026-52202-6)) -- Production Line Modelling ([paper](https://www.sciencedirect.com/science/article/pii/S2212827126007791?)) - -**Agriculture & Environment** - -- Soil Moisture Forecasting ([paper](https://link.springer.com/chapter/10.1007/978-3-032-19763-4_18)) -- Crop Water Demand ([paper](https://www.mdpi.com/2073-4395/16/16/1600)) -- Rainfall–runoff modelling ([paper](https://www.sciencedirect.com/science/article/abs/pii/S0022169422003560)) -- Drought Forecasting ([paper](https://www.sciencedirect.com/science/article/abs/pii/S1364815225000787)) -- Land Surface Dynamics ([paper](https://www.sciencedirect.com/science/article/abs/pii/S0022169425002446)) - -**Neuroscience** - -- Neural Population Dynamics ([Neural Data Transformer](https://pmc.ncbi.nlm.nih.gov/articles/PMC10541112/)) -- Neural + Motor Modelling ([Intracortical Motor Decoder](https://www.biorxiv.org/content/10.1101/2025.02.02.634313v1)) -- fMRI State Prediction ([paper](https://arxiv.org/abs/2412.19814)) -- Seizure Detection ([BIOT](https://arxiv.org/abs/2305.10351)) -- Magnetoencephalography Data ([MEG-GPT](https://arxiv.org/abs/2510.18080)) - -**Animal Communication** - -- Sperm Whales ([whale-gpt](https://github.com/0xideas/whale-gpt), [WhAM](https://github.com/Project-CETI/wham/)) -- Bengalese Finches ([paper](https://pmc.ncbi.nlm.nih.gov/articles/PMC8746767/)) -- Zebra Finches ([ZF-Aim](https://www.biorxiv.org/content/10.64898/2026.02.12.705387v1)) - -Sequifier aims to standardise model implementation across these fields, to make them comparable, transfer learnings between domains, and converge on optimal solutions faster. - -## Value Proposition - -For the individual researcher, sequifier cuts the development time of a model significantly: typically, some preprocessing and the subsequent model evaluation are specific to the modelling problem, but all the steps in between are taken care of. - -This enables: - -- rapid prototyping on a configurable architecture -- trusted implementation (you can't create bugs inadvertently) -- scaling preprocessing across cores and training across GPUs and nodes -- hyperparameter optimization using Optuna (Bayesian, Random, or Grid search) - -## The Seven Commands - -There are seven standalone commands within sequifier: `make`, `preprocess`, `train`, `infer`, `hyperparameter-search`, `visualize-training`, and `docs`. - -| Command | Purpose | -| --- | --- | -| `make` | Create a new sequifier project with config templates. | -| `preprocess` | Convert input data into fixed-length subsequences. | -| `train` | Train supervised models or ordered imitation, IQL, PPO, DPO and GRPO policy phases. | -| `infer` | Generate predictions, probabilities, embeddings, or policy actions and rollouts. | -| `hyperparameter-search` | Use Optuna to find optimal configurations across multiple training runs. | -| `visualize-training` | Generate interactive HTML plots from structured training metrics. | -| `docs` | Print the consolidated documentation, or the guide for `train`, `preprocess`, or `infer`. | - -Run `sequifier docs` for all documentation, or `sequifier docs train`, `sequifier docs preprocess`, or `sequifier docs infer` for one command guide. - -The processing and visualization commands have dedicated documentation pages: - -- [preprocess documentation](./documentation/configs/preprocess.md) -- [train documentation](./documentation/configs/train.md) -- [policy training](./documentation/policy-learning.md), including BC, IQL, DPO, GRPO and [online categorical PPO](./documentation/policy-learning.md#online-categorical-ppo) -- [infer documentation](./documentation/configs/infer.md) -- [hyperparameter-search documentation](./documentation/configs/hyperparameter-search.md) -- [visualize-training documentation](./documentation/commands/visualize-training.md) - -## Other Materials - -To get the full documentation, visit [sequifier.com](https://sequifier.com) - -For changes in each published version, see the [release notes](https://github.com/0xideas/sequifier/releases). - -## Structure of a Sequifier Project - -Sequifier is designed with a specific folder structure in mind: - -```text -YOUR_PROJECT_NAME/ -├── configs/ -│ ├── preprocess.yaml -│ ├── train.yaml -│ └── infer.yaml -├── data/ -│ └── (Place your CSV/Parquet files here) -├── models/ -├── checkpoints/ -├── outputs/ -│ ├── embeddings(?) -│ ├── predictions(?) -│ ├── probabilities(?) -│ └── visualization/ -├── logs/ -├── state/ (created when hyperparameter search runs) -└── scripts/ -``` - -The `sequifier` commands should typically be run in the project root. - -Within YOUR_PROJECT_NAME, you can also add other folders for additional steps, such as `notebooks` or `scripts` for pre- or postprocessing, and `analysis`, `visualizations` or `evals` for files you generate in other, manual steps. - -### Data Transformations in Sequifier - -The basic input data format is this: - -| sequenceId | itemPosition | column1 | column2 | ... | -| --- | --- | --- | --- | --- | -| 0 | 0 | "high" | 12.3 | ... | -| 0 | 1 | "high" | 10.2 | ... | -| ... | ... | ... | ... | ... | -| 1 | 0 | "medium" | 20.6 | ... | -| ... | ... | ... | ... | ... | - -The two columns "sequenceId" and "itemPosition" have to be present, and there must be one or more feature columns. - -`sequifier preprocess` splits sequences into subsequences, normalises real variables and maps categorical variables to integers/tokens. Each stored subsequence contains `window_length` positions; `max_target_offset` reserves positions within that window for future targets. - -| sequenceId | subsequenceId | startItemPosition | leftPadLength | inputCol | [window_length - 1] | [window_length - 2] | ... | 0 | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | -| 0 | 0 | 0 | 0 | column1 | "high" | "high" | ... | "low" | -| 0 | 0 | 0 | 0 | column2 | 12.3 | 10.2 | ... | 14.9 | -| ... | ... | ... | ... | ... | ... | ... | ... | ... | -| 1 | 0 | 15 | 0 | column1 | "medium" | "high" | ... | "medium" | -| 1 | 0 | 15 | 0 | column2 | 20.6 | 18.5 | ... | 21.6 | -| ... | ... | ... | ... | ... | ... | ... | ... | ... | - -Generative inference returns a row-oriented table with the predicted target -columns plus identifiers for the source sequence and model window: - -| sequenceId | subsequenceId | windowStartOffset | itemPosition | column1 | column2 | ... | -| --- | --- | --- | --- | --- | --- | --- | -| 0 | 0 | 0 | 963 | "medium" | 8.9 | ... | -| 0 | 0 | 0 | 964 | "low" | 6.3 | ... | -| ... | ... | ... | ... | ... | ... | ... | -| 1 | 4 | 0 | 732 | "medium" | 14.4 | ... | -| ... | ... | ... | ... | ... | ... | ... | - -### Complete Example of Training and Inferring a Transformer Model - -Once you have your data in the input format described above, you can train a transformer model in a couple of steps: - -1. **Install sequifier.** Create and activate an environment with Python >=3.10, then run: - - ```console - pip install sequifier - ``` - -2. **Create a project.** Generate a project folder with config templates in its `configs` subfolder: - - ```console - sequifier make YOUR_PROJECT_NAME - ``` - -3. **Add your data.** Change into the `YOUR_PROJECT_NAME` folder, create a `data` folder, add your data, and adapt `preprocessing_data_path` in `preprocess.yaml` to point to it. - -4. **Preprocess the data.** Run: - - ```console - sequifier preprocess - ``` - -5. **Reference the generated metadata.** Preprocessing outputs metadata at `configs/metadata_configs/[INPUT BASENAME].json`. For a single dataset and part, reference that file from `dataset.part.metadata_config_path` in `train.yaml`; named configurations use `dataset_training..parts..metadata_config_path`. Inference may still use `preprocessing_data_path` or `metadata_config_path`. - -6. **Configure and train the model.** Adapt `train.yaml` with your transformer hyperparameters, then run: - - ```console - sequifier train - ``` - -7. **Configure inference.** Point `model_path` in `infer.yaml` at the default ONNX export. Keep the scaffold's explicit contract, or replace it with `training_config_path` and `dataset`; see the [ONNX/PT trade-offs](./documentation/configs/infer.md#onnx-or-pt). - - **GPU ONNX inference:** `device: cuda` requires `onnxruntime-gpu` built for - your CUDA and cuDNN versions. The CPU-only `onnxruntime` package does not - provide `CUDAExecutionProvider`. - -8. **Run inference.** Run: - - ```console - sequifier infer - ``` - -9. **Find your predictions.** They are written to `[PROJECT ROOT]/outputs/predictions/[EXPORTED_MODEL_BASENAME]/part-000.[FORMAT]`, for example `outputs/predictions/your-model-best-3/part-000.csv`. - -## Key Features - -### Causal Modelling - -Causal transformers are the core architecture in sequifier. Their training objective is to 'predict the next token', or, in the multivariate case, the next value for each target variable. - -To train a causal transformer, set `training_objective: causal`. If the target value of interest is further in the future, set `target_offset` to a value higher than 1. - -`prediction_length` controls how many consecutive positions at the end of a -window are returned during inference. For example, with `target_offset: 1` and -`prediction_length: 3`, the three outputs predict the next token after each of -the last three input positions. This is not LLM-style multi-token prediction, -where one input position predicts several future tokens. During training, the -current loss calculation uses all valid decoded positions, regardless of -`prediction_length`. - -### Autoregressive Inference - -Autoregressive inference is allowed when the model is causal, all input variables are target variables, `target_offset` equals 1 and `prediction_length` equals 1. It is enabled by setting `autoregressive: true`, and `generation_steps` to the desired integer value. The number of generation_steps will be generated starting from the *first* complete subsequence in a sequence. - -It iteratively predicts future values, by returning predictions at step t-1 as input for generating a prediction at t. Predictions for categorical target variables can be made using argmax or sampling. - -For dependencies between multiple outputs at the same time step, the -`autoregressive_transformer` decoder models targets in explicit -`target_columns` order. It supports categorical and real targets, parallel -teacher forcing during training and validation, greedy generation during -inference, composite decoder branches, shared categorical tables, and optional -input/output weight tying. Probabilities from this decoder are conditioned on -the preceding greedy same-step predictions, so external per-column sampling is -rejected for its targets. - -### Causal Modelling Variants - -#### Final-value Causal Modelling - -In final-value causal models, the final value of each target variable within the subsequence is projected back in time as target. The idea is that the sequence of events leading up to the final value is a continuous accrual of evidence for an outcome, with the final value being the resolution. For example, the sequence of clicks through an online shop are in search of a product, and the product that is actually purchased at the end is the resolution. - -#### Next-occurrence Causal Modelling - -Next-occurrence causal modelling is a generalisation of final-value causal modelling: instead of taking the last value of each target variable as target, it takes the next value at a position where another categorical variable matches a criterion value as target. To illustrate this, final-value causal modelling is equivalent to next-occurrence causal modelling where the criterion variable is 'is_last', which is '0' up to the last position, where it is '1', and the criterion value is '1'. The values at the last position are projected 'back' across the subsequence, only now, we also have the option to use a different criterion variable, set it to '1' at multiple locations, and train the model to predict 'next relevant event', rather than just 'last event'. - -### Causal Embedding Model - -sequifier also supports the export of causal embeddings, instead of predictions. It requires the following settings: `export_embedding_model: true` in the training config and `model_type: embedding` in the inference config. Selected activations are restricted to the configured final `prediction_length` positions and concatenated in configuration order along the feature dimension. - -If you are interested in activations *other* than the last backbone layer, you can configure the exact layers you want to contribute to the export using `embedding_layer_names`. You can pass an ordered list, such as `[backbone.layers.1, decoder.branches.default.hidden_blocks.0]`, and the activations of these layers will be concatenated and output. - -Layer names follow the network hierarchy using zero-based indices: `backbone.layers.` selects a transformer block output, `backbone.final_norm` the normalized backbone output, and `decoder.branches..hidden_blocks.` an MLP decoder hidden-block output; the same scheme applies to BERT embedding models. - -Backbone selectors contribute `dim_model` activations. Decoder MLP hidden-block selectors contribute their configured hidden width and receive the same flattened `decoding_support * dim_model` windows used during training. The default, `embedding_layer_names: [backbone.final_norm]`, preserves the final normalised backbone representation. - -### BERT Model - -Sequifier also supports training and inference of BERT-style masked reconstruction models. - -Configuration: - -- Preprocessing: Set `max_target_offset: 0` for equal-width input and target windows. -- Training: Set `training_objective: bert`, configure `bert_spec`, and set decoder `prediction_length` equal to `context_length`. Enable generative and/or embedding export according to the desired inference. -- Inference: Set `model_type: generative` to reconstruct explicitly masked input, or `model_type: embedding` to output contextual representations. - -Technical Details: BERT-style models use bidirectional attention and learn by reconstructing positions sampled according to `bert_spec`. Inference does not apply random masking; inputs that should be reconstructed must be masked explicitly, for example using `mask_column` during preprocessing. Embedding inference returns one contextual representation for every valid position in the input window. - -### Structured Ingestion - -Structured ingestion allows the model to learn relationships for configured sets of input variables *before* they are passed to the transformer backbone. - -This enables more constrained representation learning within these subspaces, and provides a structural inductive bias and may reduce parameters, depending on configuration. It can be helpful to think of them as smaller submodules that learn local structure before passing the extracted information to the transformer for longer-range dependencies. - -The key modalities are self-attention, pooling, 1D, 2D and 3D convolutions, and adding learned or rotary axis embeddings. - -Image ingestion reads named image or video views from their original folders. -Each view uses a frame rate relative to `itemPosition` or timestamps aligned -through a configured time column; a trainable -ResNet or frozen pretrained backbone embeds each image. Views can enter a -composite branch or occupy cells in a structured layout. A separate single -image can condition the whole sequence through one prefix embedding. See the -[image source configuration](documentation/configs/preprocess.md#image-and-video-sources) -and [image ingestion configuration](documentation/configs/train.md#image-ingestion). - -#### Repeated child collections - -Named depth layouts represent a fixed-capacity collection of child rows at each -outer sequence position. A `depth_transformer` ingestion branch encodes the -valid children, pools them through a CLS token, and emits one vector per outer -position for the temporal transformer backbone. It can be combined with shallow -features in nested composite ingestion branches. - -The preprocessing metadata and PT payload carry each layout's capacity and -validity mask. Portable PT and ONNX models retain that execution contract. See -the [preprocessing guide](documentation/configs/preprocess.md#named-depth-layouts), -[training guide](documentation/configs/train.md#depth-encoders-and-nested-composites), -and [inference guide](documentation/configs/infer.md#portable-depth-models-and-dropout-modes). - -#### Temporal Convolution - -Separately, `temporal_conv` enables temporal convolutions on pass-through or embedded real or categorical variables. - -`temporal_conv` uses causal, left-padded convolutions by default (`causal: true`). -For causal models, keep the default `orientation: within_item_position` when -using `post_conv_norm: layer_norm` or `rmsnorm`. With -`orientation: within_column`, these norms operate across the full time axis, so -earlier positions can depend on later ones despite the causal convolution and -the transformer's attention mask. If `within_column` is needed in a causal -model, set `post_conv_norm: none`. - -#### Categorical multi-hash embeddings and targets - -An interface can hash canonical categorical IDs for input embeddings and target -supervision. Configure `categorical_hashing` on the model interface, keyed by -logical column name: - -```yaml -model: - interfaces: - default: - input_columns: [product_id] - target_columns: [product_id] - categorical_hashing: - product_id: - type: multi_hash - num_buckets: 4096 - num_hashes: 3 - seed: 42 - ingestion: - type: embedding - output_dim: 128 -``` - -The `categorical_hashing` field belongs to the interface. Its settings drive -both the input embedding and target heads for `product_id`; do not put a -`hashing` field under `ingestion`. See the -[training guide](documentation/configs/train.md#categorical-multi-hash-embeddings-and-targets) -for a two-column example and the required target loss configuration. - -`type: qr` uses quotient and remainder heads; `num_hashes` is always 2. Each -hash component gets an equal share of the logical target's `loss_weights` -entry. Class weights are unavailable for hash targets. The model resolves -component predictions against the allowed canonical IDs and reports the -logical column as usual. The resolver contributes no training loss. No hash -columns are needed in stored data. Preprocessing cardinality hashing remains -independent: a category collapsed there still decodes as `[hash_bucket:k]`. -Configure model hashing through the interface-level `categorical_hashing` field. - -In an `autoregressive_transformer` decoder branch, a hashed target must be the -last target in that branch's `target_columns` order. - -### Multi-Part Datasets - -Multi-part datasets let a model train on separately preprocessed data with -distinct metadata configs. Parts must share the schema, categorical mappings, -normalization policy, `max_target_offset`, per-split window widths, alignment -settings, dtypes, and file/folder storage form. Set `metadata_config_path` to -the first part's metadata when preprocessing later parts to reuse its fitted -category and normalization meanings. - -### Composable Configs - -Between different training runs and hyperparameter searches, a lot of configuration can get duplicated, and it becomes hard to follow what differentiates them and where they overlap. One approach to address this is to create different config fragments, and compose them into full training and hyperparameter tuning configurations using separate 'top-level' configs, and assemble the fragments by listing them as value in `additional_config_paths`. Fragments can contribute disjoint nested fields, while duplicate fields, recursive fragment inclusion, and repeated files are rejected; command-line overrides are applied after composition. - -### Distributed Training - -Sequifier supports distributed training using torch `DistributedDataParallel` and `FullyShardedDataParallel`. To make use of multi gpu support, the preprocessing step must write sharded output with `merge_output: false`. `write_format: pt` is the recommended file format; sharded `parquet` is also supported but currently considered beta for distributed training. - -For the full guide on how to configure a distributed run, check the [multi-GPU training guide](./documentation/training/multi-gpu-training.md). - -### System Requirements - -Sequifier currently runs on MacOS and Ubuntu. - -## Citation - -Please cite with: - -```bibtex -@software{sequifier_2025, - author = {Luithlen, Leon}, - title = {sequifier - transformers for multivariate sequence generation and representation learning}, - year = {2025}, - publisher = {GitHub}, - version = {v3.1.2.0}, - url = {https://github.com/0xideas/sequifier} -} - -``` - -Policy runs support ordered imitation, IQL, DPO, GRPO and PPO phases; see the [policy learning guide](documentation/policy-learning.md). - - -# Preprocess Command Guide - -The `sequifier preprocess` command transforms raw tabular data (CSV or Parquet) into the specific sequence format required for training transformer sequence models. It handles windowing, data splitting (train/validation/test), categorical encoding, and optional numerical standardization. - -## Usage - -```console -sequifier preprocess --config-path configs/preprocess.yaml -``` - -## CLI Overrides - -Values passed on the command line override the YAML before validation. - -| Flag | Overrides / Action | -| :--- | :--- | -| `-r`, `--randomize` | Generates a random `seed`. The seed affects `between_sequence` split assignment. | -| `-dp`, `--data-path` | Overrides `preprocessing_data_path`. | -| `-sc`, `--selected-columns` | Overrides `selected_columns` with a space-separated list. Use `None` to process all columns. | - -## Composable Configuration Files - -A preprocessing entry config may set `additional_config_paths` to one -non-empty string, a list of non-empty strings, or `null`. Relative paths -resolve against the entry config's `project_root`; absolute paths are used -directly. Fragments are direct only and cannot include further fragments. They -may share nested containers when their child fields are disjoint, but duplicate -fields are errors. CLI values override the completed file composition. - -## Configuration Fields - -The configuration is defined in a YAML file (e.g., `preprocess.yaml`). Below are the available fields, their requirements, and their functions. - -### 1\. File System & Input/Output - -| Field | Type | Mandatory | Default | Description | -| :--- | :--- | :--- | :--- | :--- | -| `project_root` | `str` | **Yes** | - | The root directory of your Sequifier project. Usually `.` | -| `additional_config_paths` | `str`, `list[str]`, or `null` | No | `null` | Direct complementary YAML fragments. Relative paths resolve against `project_root`; recursive composition and duplicate fields are rejected. | -| `preprocessing_data_path` | `str` | **Yes** | - | Path to the raw input file or folder. | -| `static_data_path` | `str` | No | `null` | Separate CSV or Parquet table with exactly one row per `sequenceId`. Preprocessing requires its ID set to match the sequence data in both directions and writes a processed static feature table. | -| `static_categorical_columns` | `list[str]` | No | `[]` | Static columns to treat as categorical when their source dtype would not make that intent clear. | -| `static_real_columns` | `list[str]` | No | `[]` | Static numeric columns to treat as real valued, including integer measurements. | -| `read_format` | `str` | No | `csv` | Format of input data (`csv`, `parquet`). | -| `write_format` | `str` | No | `parquet` | Format of output data (`csv`, `parquet`, `pt`). | -| `merge_output` | `bool` | No | `true` | Whether to merge split files into single files or keep them sharded. | -| `continue_preprocessing`| `bool` | No | `false` | If `true`, resumes from an existing preprocessing temp folder created by an interrupted run. | - - -> **Important Constraint on `write_format`:** -> -> * If `write_format` is **`pt`** (PyTorch tensors), `merge_output` must be **`false`**. -> * If `write_format` is **`parquet`**, `merge_output` can be **`false`** or **`true`**. -> * If `write_format` is **`csv`**, `merge_output` must be **`true`**. -> For distributed training, `merge_output` must be set to **`false`**. - -Static columns are read from `static_data_path`, not `selected_columns` or the -sequence rows. The static table may reside in the sequence input folder; it is -excluded from sequence-file discovery. Its `sequenceId` column must be an -integer without nulls or duplicates. Static feature columns must have finite, -non-null values and must not share names with temporal columns. Categorical -maps and real normalization are fitted using sequences represented in training -split 0, unless `normalize_on_all_data` is enabled. The processed feature -table has one row per sequence and is referenced by the generated metadata. - -### Image and video sources - -Install `sequifier[images]` to use image sources. Temporal images live in one -flat folder per named view. Untimed images are named -`{sequence_id}-{frame_index}` plus an image extension; an untimed video view has -one `{sequence_id}` video per sequence. Decoded video frames are numbered from -zero in decode order. A separate `preconditioning_image` -source contains one `{sequence_id}` image per sequence in each folder. Its -`path` can be one folder or a map of named view folders. - -```yaml -image_sources: - front: - path: data/front_images - type: images - alignment: {frames: 3, items: 2} - side: - path: data/side_videos - type: videos - alignment: {frames: 1, items: 2} -preconditioning_image: - path: - front: data/sequence_front_images - side: data/sequence_side_images -``` - -`alignment` defaults to `{frames: 1, items: 1, phase: 0}`. `frames` and -`items` are positive integers; `phase` is an integer from zero through -`items - 1`. Every sequence starts at `itemPosition: 0`, and image frame indices -start at zero. With `p = frames`, `q = items`, and `h = phase`: - -- For `p >= q`, item `i` receives frames from boundary `b(i)` through - `b(i+1) - 1`, where `b(i) = ceil((i*p+h)/q) - ceil(h/q)`. -- For `p < q`, item `i` uses frame `floor((i*p+h)/q)`, repeating the latest - image until the next frame becomes available. - -For irregular item times, set `time_alignment_column` at the top level of the -preprocessing config. Timed media filenames use one of these patterns for every -sequence ID in a view: - -- Videos: `{sequence_id}-{time}` (one video per sequence). The filename gives - the first frame's time; subsequent frame times come from video presentation - timestamps. -- Images with a known interval: frame zero is - `{sequence_id}-0-{time}-{frame_interval_ms}`; later frames are - `{sequence_id}-{frame_index}`. The positive interval is in milliseconds. -- Images with every frame timestamped: - `{sequence_id}-{frame_index}-{time}` for all frames. - -`time` is either `HHmmssSSS` (relative time) or `yyyyMMddHHmmssSSS` -(absolute time). The time column must use the same kind of value for all rows: -relative `HHmmssSSS` or absolute timestamps with millisecond precision, including -the 17-digit filename format. A `Date` value represents midnight on that date; -date-only ISO strings use the same convention. Relative media and item times start at -`000000000` for every sequence. Absolute values are compared as written; users -must keep their inputs consistent. Sequifier performs no timezone conversion. -Filename modes and time representations must be uniform across sequence IDs in -each view. Timed views require a positive per-view `max_frames_per_item` and do -not use `alignment` to determine frame counts. -If `max_frames_per_item` is set, it must be at least -`ceil(alignment.frames / alignment.items)`; untimed views otherwise lose frames. -Leave `alignment` at its default for timed views. - -At each item, images since the previous item time (including images exactly at -the current item time) are pooled. If more than `max_frames_per_item` arrived, -the most recent images are kept. If none arrived, the latest prior image is -repeated. The first media frame must be available at or before the first item. -Preprocessing writes an item-time index beside the processed data; this file -must remain available for training and inference. - -```yaml -time_alignment_column: captureTime -image_sources: - front: - path: data/front_images - type: images - max_frames_per_item: 4 -``` - -`time_alignment_column` is used for alignment and is excluded from model -features. It may be omitted from `selected_columns`. - -Preprocessing checks all sequence IDs, contiguous image indices, exact frame -counts for untimed views, and decodability against the raw item grid. Untimed -views may have different frame counts when their configured rates cover the -same grid. Metadata records view names, alignment, transforms, and a file-list -signature, but omits the original folder paths. Supply those paths again when -training or inferring; original media is neither modified nor copied. Loading -fails if the source file list, sizes, or modification times change after -preprocessing. `resize_size` (default 256) and -`crop_size` (default 224) can be set per source; images are resized by their -shorter edge and then center cropped during loading. -Preprocessing writes a versioned video index for each source video beside the -processed data and records its path in metadata. Each index contains frame -timestamps, keyframe seek positions, and frame times for timed alignment. -Workers load an index when they first need its video and reject it if the media -signature or stream time base changed. Videos without reliable seeking use a -temporary indexed cache of transformed frames. Older metadata without video -indexes remains readable through a worker-local scan. - -### 2\. Column Selection & Filtering - -| Field | Type | Mandatory | Default | Description | -| :--- | :--- | :--- | :--- | :--- | -| `time_alignment_column` | `str` | For timed media | `null` | Raw item-time column used to align timed images and video. Stored in a separate item-time index and excluded from model features. | -| `selected_columns` | `list[str]` | No | `null` | A specific list of columns to process. If `null`, all columns (except metadata) are processed. | -| `categorical_columns` | `list[str]` | No | `null` | Explicitly classify supported discrete processed columns as categorical instead of relying on dtype inference. Undeclared columns retain dtype-based inference. Columns cannot also appear in `real_columns`. When `column_data_types` is set, these columns must use an integer output dtype. | -| `real_columns` | `list[str]` | No | `null` | Explicitly classify processed numeric columns as real-valued. This is useful for integer-valued amounts, counts, and epoch timestamps that must retain ordinal meaning. Undeclared columns retain dtype-based inference. Columns cannot also appear in `categorical_columns`. When `column_data_types` is set, these columns must use a floating-point output dtype. | -| `cardinality_config` | `dict[str, object]` | No | `{}` | Per-categorical-column rules for retaining frequent values and optionally hashing the rest into buckets. See [Categorical cardinality and hashing](#4-categorical-cardinality-and-hashing). | -| `column_data_types` | `dict[str, str]` | No | `null` | Optional output dtype map for processed columns, such as `Float32`, `Float64`, `Int32`, or `Int64`. A `Date` or `Datetime...` dtype is accepted for the `value_cutoff` `split_column` or `time_alignment_column`. If set, every processed column must be included. Parquet uses one unified sequence dtype; `pt` writes each variable to its configured tensor dtype. | -| `normalize_real_columns` | `bool` | No | `true` | If `true`, Z-score normalizes real-valued columns. Set to `false` to preserve their original values. Statistics are still recorded in metadata. | -| `normalize_on_all_data` | `bool` | No | `false` | If `false`, numeric statistics and dynamic categorical vocabularies are fitted only on split 0; values seen only in later splits map to `[other]` or a configured hash bucket. Set to `true` to fit on all splits. | -| `max_rows` | `int` | No | `null` | Limits processing to the first N rows. Useful for rapid debugging. | -| `metadata_config_path` | `Optional[str]` | No | `null` | Use a preexisting metadata config for tokenizing discrete columns and, when enabled, standardizing real-valued columns. | -| `mask_column` | `Optional[str]` | No | `null` | Optional input column used as a row-level mask. If set, `metadata_config_path` must also be set, and it cannot also be `split_column`. | -| `curriculum_column` | `Optional[str \| list[str]]` | No | `null` | One or more optional integer input columns to preserve as per-subsequence curriculum metadata. | -| `use_precomputed_maps`| `list[str]` | No | `null` | If not `null`, enforces the use of precomputed maps for the variables in the list. | - -When `curriculum_column` is set, each named column must be integer-valued and -constant within every generated subsequence. A list preserves several columns -so that a later training run can select any one of them. The values are stored -as metadata, not as model features. Different subsequences of one `sequenceId` -may use different values. For depth input, the values must also agree across -all repeated child rows for an outer item; depth PT output carries the same -per-window metadata. Names must be unique and non-empty, must exist in every -input file, and cannot be `sequenceId`, `itemPosition`, the mask column, or a -depth feature/position column. Names beginning with -`__sequifier_curriculum_value_` are reserved. - -### 3\. Sequence Logic & Splitting - -| Field | Type | Mandatory | Default | Description | -| :--- | :--- | :--- | :--- | :--- | -| `window_length` | `int` | Conditional | - | One physical width for every split. Specify exactly one of `window_length` or `window_lengths`. | -| `window_lengths` | `list[int]` | Conditional | - | Physical widths in split order, with one entry per split. Required when `prediction_alignment_steps > 1`; widths from split 2 onward must equal the first width plus `prediction_alignment_steps - 1`. | -| `max_target_offset` | `int` | No | `1` | Number of future items retained after the model input window. Use `0` for BERT-style same-width inputs and targets; use `1` for causal next-item training. | -| `split_ratios` | `list[float]`| Conditional | `null` | Ordered split proportions for `within_sequence` and `between_sequence`. Must sum to 1.0 and must be omitted for `value_cutoff`. | -| `split_method` | `str` | No | `within_sequence` | How rows are assigned to splits: `within_sequence`, `between_sequence`, or `value_cutoff`. | -| `split_column` | `str` | Conditional | `null` | Required for `value_cutoff`. Names the integer, Date, or Datetime column compared with `split_values`; it cannot be a mask, curriculum, sequence ID, or item-position column. | -| `split_values` | `list[int \| timestamp]` | Conditional | `null` | Required for `value_cutoff`. Strictly increasing boundaries that create `len(split_values) + 1` splits. Values must all be integers or all be ISO timestamp values. | -| `window_stride` | `int` | No | First split width | Stored-window stride for distributed splits. | -| `prediction_aligned_splits` | `list[int]` | No | `[]` | Zero-based split indices whose prediction groups are anchored to each split end. All other splits use distributed placement. | -| `prediction_alignment_steps` | `int` | No | `1` | Number of consecutive prediction groups represented by each aligned stored window. | -| `prediction_length` | `int` | Conditional | `null` | Required for prediction-aligned splits; number of output positions per window. | -| `target_offset` | `int` | Conditional | `null` | Required for prediction-aligned splits; must equal `max_target_offset`. | -| `allow_sequence_splitting` | `bool` | No | `false` | If `false`, a single sequence is kept within one preprocessing batch. | - -All newly preprocessed windows store their absolute start position and split -target bounds, including distributed windows and depth-layout PT windows. Dataset -loaders require these fields and reject outputs created with an older payload -schema; re-run preprocessing to migrate such data. - -`value_cutoff` applies the same boundaries to every sequence. Values equal to a -boundary belong to the later split, and the split column must be non-decreasing -within each sequence. Integer columns require integer boundaries. Timestamp -columns support Polars `Date` and `Datetime` values, or ISO timestamp strings -that can be parsed as UTC; `Time` and `Duration` are not supported. A timestamp -split column may be retained in preprocessing output by including it in -`selected_columns` (or by leaving `selected_columns: null`), but timestamp -columns cannot be model inputs or targets in training or inference. Any other -typed temporal column is rejected. The name `__sequifier_split_value` is -reserved for preprocessing internals. - -To align validation or test predictions exactly to their split positions, configure -those split indices and the model's prediction view: - -```yaml -window_length: 129 -max_target_offset: 1 -window_stride: 128 -prediction_aligned_splits: [1, 2] -prediction_length: 2 -target_offset: 1 -``` - -Aligned splits place the last prediction group at the split end and step backward -by `prediction_length`. The first group may begin before the split; its earlier -predictions are masked from loss, metrics, and inference output. Inputs may use -preceding rows from the same sequence. Missing history at the sequence start is -left-padded. The stored windows never read beyond the split end. An aligned -split requires `target_offset == max_target_offset` and -`allow_sequence_splitting: false`. Training and inference must use matching -`target_offset` and `prediction_length`. Training can sample additional model -views with `global_training.window_stride`; aligned inference requires -`window_stride: null`. Empty splits produce no windows. - -Prediction-aligned inference supports generative and embedding output without -autoregressive generation. Aligned embeddings are labeled with the target row -predicted from each activation and use the target validity mask. For a causal -model with `target_offset: 1`, the embedding labeled as row N represents the -context before row N; it does not include row N's value. Embeddings from other -splits retain their input-position labels. - -Distributed splits retain isolated, evenly spread windows and use the scalar -preprocessing `window_stride`. The same placement rules apply to depth-layout -PT output. - -### 4\. Performance & System - -| Field | Type | Mandatory | Default | Description | -| :--- | :--- | :--- | :--- | :--- | -| `seed` | `int` | No | `1010` | Random seed for reproducibility. | -| `n_cores` | `int` | No | Max Cores | Maximum worker processes for eligible preprocessing stages. Depth preprocessing parallelizes a single Parquet file only when it has multiple row groups and `max_rows` is unset. | -| `batches_per_file` | `int` | No | `1024` | Buffer flush threshold per split. For flat input, counts sequence-window groups in PT and Parquet output (including merged output); a byte limit can flush earlier. For depth input, counts windows in PT output. | -| `process_by_file` | `bool` | No | `true` | Memory optimization. If `true`, processes one input file at a time. | - ------ - -## Key Trade-offs and Decisions - -### 1\. `write_format`: `parquet` vs. `pt` - - * **Choose `parquet` (default):** Unless you have a specific reason, use `parquet`. *Note: If you are doing distributed training, Parquet support is currently in **Beta**.* - * **Choose `pt`:** Use `pt` data loading if speed and CPU overhead are your primary bottlenecks, **or if you are running multi-GPU distributed training.** This format is the most stable choice for high-throughput scaling. - -### 2\. Stored windows and model windows - -Preprocessing stores windows of the configured width for each split. Distributed -splits use `window_stride`; aligned stored windows step by -`prediction_alignment_steps × prediction_length`. Training's `context_length` -is the model input width. Each selected split width must be at least -`context_length + max_target_offset`. Training's -`window_stride` is separate: `null` uses one right-aligned model view per stored -window, while a positive integer samples additional views *within* a longer -stored window. If the two widths are equal, `window_stride` adds no views. - -Aligned inference reads `prediction_alignment_steps` model views from each -stored window, moving right by `prediction_length` positions for each view. -Preprocessing requires room for all prediction groups after `max_target_offset`; -training and inference also check the width against the model's `context_length`. -To use the entire window, choose its width as -`context_length + target_offset + (prediction_alignment_steps - 1) × prediction_length`. -Earlier positions outside the split are masked from prediction output. -Set `window_lengths` when `prediction_alignment_steps > 1`. The first two -split widths can differ; every later split width must equal the first width -plus `prediction_alignment_steps - 1`, even when that split uses distributed -placement. - -For example, with `context_length: 128`, `target_offset: 1`, and three splits, -the following gives aligned splits room for three two-position prediction -groups: - -```yaml -window_lengths: [131, 133, 133] -max_target_offset: 1 -prediction_aligned_splits: [1, 2] -prediction_length: 2 -prediction_alignment_steps: 3 -target_offset: 1 -``` - -| Scenario | Suggested settings | Trade-off | -| --- | --- | --- | -| Many short or varied-length sequences | Store the minimum width (for example, `window_length: 129` for `context_length: 128`, `max_target_offset: 1`), use a preprocessing `window_stride` near 128, and set training `window_stride: null`. | Limits padding and stores roughly one copy of long sequences, but targets near window boundaries have less preceding context. | -| More overlap during training | Keep that width; reduce the preprocessing `window_stride` to a fraction of the training `context_length`. | Roughly 2× or 4× as many stored events for long sequences. | -| Long sequences, several model views per stored window | Use a longer stored width (for example, `window_length: 513`, `window_stride: 384`, `context_length: 128`, training config `window_stride: 128`). | About 1.3× stored events on long sequences; short sequences pad to 513, and more model views cost more compute. | -| Dense evaluation with nearly full preceding context at each window's right edge | Use `prediction_aligned_splits` for exact split coverage, or a small preprocessing `window_stride` for distributed evaluation. | More stored windows and more evaluation compute. | - -Self-attention lets the model use relationships between positions within a -window, so heavily overlapping stored windows are often unnecessary. Small -preprocessing strides increase storage and compute and may put very similar -windows in a training batch. Sequifier currently computes training loss over -all valid decoded positions of each model view, regardless of -`prediction_length`. With causal next-event targets, each decoded position -predicts its own next event. A smaller stride gives more targets a view with -substantial preceding context, even though adjacent inputs overlap. -`prediction_length` selects the final consecutive output positions during -inference; it does not make one position predict several future events. Short -sequences are left-padded to their split's stored width regardless of stride; -inspect the sequence-length distribution before choosing a long stored width. - -### 3\. Distributed and prediction-aligned placement - -Distributed placement adjusts starts to cover each split evenly and includes -the final available window. Prediction-aligned placement anchors stored windows -to the split end, then steps backward by -`prediction_alignment_steps × prediction_length`. Each stored window contains -`prediction_alignment_steps` consecutive prediction groups. When the split -length is not divisible by this spacing, the first window includes positions -before the split; those predictions are masked. - -### 4. Categorical cardinality and hashing - -Use `cardinality_config` to limit the number of distinct IDs created for a -categorical column. Each key is a column name. For example, keep the 10,000 most -frequent product IDs and hash every other product ID into 4,096 buckets: - -```yaml -categorical_columns: [product_id] -cardinality_config: - product_id: - top_k: 10000 - hashing: - num_buckets: 4096 - seed: 42 -``` - -`top_k` retains at most that many values, ranked by frequency. Alternatively, -`min_freq` retains every value occurring at least that many times. Both must be -positive integers and cannot be used together. Retained values keep individual -IDs; all remaining non-null values, including values first seen in later -splits, map to a hash bucket when `hashing` is configured. Without `hashing`, -non-retained values map to `[other]` instead. To hash all ordinary categorical -values, omit both retention settings: - -```yaml -cardinality_config: - product_id: - hashing: - num_buckets: 4096 -``` - -`num_buckets` is a positive integer. The optional `hashing.seed` defaults to -`0` and is separate from the top-level preprocessing `seed`. Hashing is stable -for strings, booleans, and integers. Multiple values can land in the same -bucket, so their original identities cannot be recovered from preprocessed -data; decoded bucket values appear as `[hash_bucket:k]`. Null values retain the -reserved `[unknown]` ID. - -By default, frequencies and retained values are fitted on split 0. Set -`normalize_on_all_data: true` to fit them on all data. A configured column must -be categorical, must appear in `selected_columns` when that list is set, and -cannot also appear in `use_precomputed_maps`. If `categorical_columns` is set, -include the configured column there. Preprocessing hashing is independent of -the model interface's `categorical_hashing` setting described in the -[training guide](train.md#categorical-multi-hash-embeddings-and-targets). - -### 5. Advanced: Static Vocabularies (Custom ID Maps) - -By default, Sequifier dynamically builds ID maps from the data found in the input file. However, in production systems, you often need a **fixed vocabulary** to ensure that ID "105" always maps to "Item_X", regardless of the daily training batch. - -To use a static vocabulary: -1. Create a folder `configs/id_maps/` in your project root. -2. Add JSON files named `{COLUMN_NAME}.json`. -3. The format must be a dictionary mapping ordinary data values to integers **starting at 3**. Reserved labels may be included only with their fixed IDs. - -> **Reserved Indices:** -> * **0**: Reserved for `[unknown]` (padding/missing). -> * **1**: Reserved for `[other]` (unseen values not in your map). -> * **2**: Reserved for `[mask]`. -> * **3+**: Your data. - -**Example `configs/id_maps/itemId.json`:** -```json -{ - "apple": 3, - "banana": 4, - "cherry": 5 -} -``` ------ - -## Outputs - -After running `preprocess`, the following are generated: - -1. **Data Files:** Located in `data/`. Depending on your configuration, these will be merged files such as `[NAME]-split0.parquet` (Training), `[NAME]-split1.parquet` (Validation), etc., or split folders such as `[NAME]-split0/` containing `.pt` or `.parquet` shards. -2. **Metadata Config:** Located in `configs/metadata_configs/[NAME].json`. - * **Crucial:** This file contains the integer mappings for categorical variables (`id_maps`), statistics for real variables (`selected_columns_statistics`), and whether those variables were normalized (`normalize_real_columns`). - * **Next Step:** Reference this file from `dataset.part.metadata_config_path` in a singleton training config, or from `dataset_training..parts..metadata_config_path` in a named training config. In inference, either `preprocessing_data_path` or `metadata_config_path` can locate the metadata and its split paths. - -## Named depth layouts - -Repeated rows can describe one outer item with a fixed-capacity child collection. -Depth behavior is explicit; a column named `subItemPosition` alone remains an -ordinary flat feature. - -```yaml -project_root: . -preprocessing_data_path: data/raw-items -read_format: parquet -write_format: pt -merge_output: false -selected_columns: [accountType, subitemType, subitemAmount, nextAction] -depth_layouts: - subitems: - position_column: subItemPosition - columns: [subitemType, subitemAmount] - context_length: 16 - position_base: 0 - allow_gaps: false -window_length: 129 -max_target_offset: 1 -split_ratios: [0.8, 0.1, 0.1] -window_stride: 128 -``` - -Every file must contain the depth position column, which is read automatically -and excluded from feature statistics and output types. Configured -`curriculum_column` values are also read automatically and cannot be used as -the depth position or a depth feature. Configure at most one raw layout, use PT output -without merging, and omit `mask_column`. Reused metadata must have the same -complete layout definition, output types, and normalization policy. Sequence -identifiers and item positions must have integer source types and fit signed -Int64; curriculum and depth positions must have integer source types. - -Within each source file, rows for one `(sequenceId, itemPosition)` must be -adjacent, and distinct item coordinates must increase in that order. An item -may occur in only one file. Files may cover interleaved coordinate ranges; -the adapter merges their ordered item streams without a temporary database. -For large file counts, the merge uses bounded fan-in and temporary sorted runs. -`max_rows` counts complete outer items in global coordinate order. Ordering -and cross-file uniqueness are checked across the full input, including items -beyond `max_rows`. Shallow features must agree across every child row before -casting or mapping. Shallow statistics count each item once; deep -statistics count occupied child slots. Both populations are selected before -split extraction. Materialization encodes each item once and uses bounded -windows and output batches; `batches_per_file` bounds the number of windows -accumulated per split. For a single Parquet file with multiple row groups and -no `max_rows`, this adapter uses `n_cores` to parallelize auditing, metadata -fitting, and window materialization. Other depth inputs use the sequential -merge path. - -Child positions map to physical slots by subtracting `position_base`. Without -`allow_gaps`, occupied slots must be a prefix starting at zero. Tail padding is -always allowed. With gaps enabled, physical slots remain unchanged. Outer item -positions must be continuous within each selected sequence. An item in this raw -format must have at least one child; null child rows do not encode emptiness. - -All PT files use version 5 of the `sequifier_tensor_batch` envelope. The payload -stores shallow tensors as `[N,W]`, deep tensors as `[N,W,D]`, boolean masks under -`metadata.depth_valid_masks.`, and the absolute window and split-boundary -positions needed to enforce split ownership. Optional signed Int64 curriculum -values are stored under `metadata.sample_positions`; their source names are -stored under `metadata.curriculum_columns`, and multiple preserved columns use -shape `[N,C]`. Public metadata records the model-facing -`tensor_payload_version` separately from this internal storage envelope. Readers -reject legacy tuple payloads and envelope versions 2 through 4; re-run -preprocessing to migrate them. -Categorical padding is the existing unknown-token ID (zero); real padding is -finite zero after normalization. Temporal padding has false depth masks. - -External tensor payloads may contain several layouts with different capacities -and independently empty collections. Use `StoredTensorBatch`, `save_pt_payload`, -and `load_pt_payload` in `sequifier.io.pt_payload`, supplying the complete layout -registry and categorical vocabulary sizes. All stored feature values, including -masked slots, must have legal categorical indices and finite real values. Empty -collections use an all-false mask. Missing masks and forbidden internal gaps are -errors. Selected interfaces compare only relevant layout feature membership and -layout properties, so unused stored layouts/features can be added independently. - -To consume a named layout during training, configure a `depth_transformer` -ingestion branch whose `layout` names this entry and whose `columns` are selected -features from it. See [the training guide](train.md#depth-encoders-and-nested-composites). - -## Import sequence demonstrations - -`operation: import_sequences` selects the explicit policy-data import operation. -It accepts a `source`, a model-independent `policy` data schema, an `output_path` -(new dataset directory) and optional `episodes_per_shard` (default 64). Sources -can be raw CSV/Parquet tables, existing scalar prepared histories with their -metadata, or JSON rollout recordings. Import preserves domain values and missing -transition information; it does not fit vocabularies/statistics or create padded -windows. Raw tables can also be used directly by policy training without this step. - -The operation atomically publishes Parquet shards and a checksummed manifest. -See [policy learning](../policy-learning.md#optional-committed-import) and -[import-policy-sequences.yaml](../example_inputs/configs/import-policy-sequences.yaml). - - -# Train Command Guide - -For trajectory-backed categorical BC, return-conditioned imitation, and offline -IQL and online PPO through the same commands, see [policy learning](../policy-learning.md). - -`sequifier train` trains one shared transformer backbone through one or more -named model interfaces. An interface is an ingestion module, its generated -adapter, and a decoder. Phases own batching and optimization; datasets own data and losses; `model` -owns architecture. - -```console -sequifier train --config-path configs/train.yaml -``` - -## Singleton configuration - -When a run has one model interface, one dataset, one part, and one phase, those -values can be authored directly without routing names or references: - -```yaml -project_root: . -model_name: event-model -device: cuda -seed: 1010 -global_training: - read_format: parquet - training_objective: causal - context_length: 128 - inference_batch_size: 256 -model: - backbone: - architecture: - dim_model: 128 - max_context_length: 512 - num_layers: 6 - attention: - type: mha - n_heads: 8 - n_kv_heads: 8 - output_projection: true - feed_forward: - dim: 512 - activation: swiglu - normalization: - type: rmsnorm - norm_first: true - position_encoding: - type: rope - theta: 10000 - dropout: 0.1 - shared_layer_groups: [] - interface: - input_columns: - - event - target_columns: - - event - ingestion: - type: embedding - output_dim: 128 - decoder: - type: linear - prediction_length: 1 - support: 1 -dataset: - part: - metadata_config_path: configs/metadata_configs/events.json - criterion: - event: CrossEntropyLoss -training_plan: - phases: - - epochs: 5 - mode: sequential - name: supervised - algorithm: supervised - batch_size: 64 - optimization: - model: - learning_rate: 0.0001 -evaluation: true -``` - -The concise form expands before validation: - -| Authored field | Canonical form | -| --- | --- | -| `model.interface` | `model.interfaces.default` | -| `dataset.part` | `dataset_training.default.parts.default` | -| `training_plan.epochs` | One sequential phase named `train` | -| `evaluation: true` | Evaluate the inferred single source | - -These shortcuts can be used independently. A unique interface and training -source are inferred; multiple interfaces or datasets require explicit names and -references. Singular and plural spellings cannot be combined. Because errors -are reported after expansion, they may use the canonical paths above. - -To condition every model window on features from a separate per-sequence -table, preprocess with `static_data_path` and configure the interface: - -```yaml -model: - interface: - input_columns: [event] - static_conditioning: - columns: [account_type, account_age] - feature_embedding_dims: {account_type: 16, account_age: 8} - target_columns: [event] - ingestion: {type: embedding, output_dim: 128} - decoder: {type: linear, prediction_length: 1, support: 1} -``` - -The static columns are separate from temporal `input_columns` and targets. -They are looked up by `sequenceId` for every sampled window, including windows -from the middle of a sequence. -`feature_embedding_dims` assigns a width to each static feature; real features -with width greater than one use a learned projection. Alternatively, -`embedding_size` distributes a total width automatically when all static -features are categorical or all are real. For mixed types, specify -`feature_embedding_dims`. If neither option is set, categorical widths use -the existing cardinality-based default and real features use one channel. -The concatenated features are projected to the backbone `dim_model`. -The model creates one prefix token at runtime -and removes its output before decoding. Set backbone `max_context_length` to -at least `global_training.context_length + 1`. - -## Canonical configuration - -```yaml -project_root: . -model_name: event-model -device: cuda -seed: 1010 -global_training: - read_format: parquet - training_objective: causal - context_length: 128 - target_offset: 1 - window_stride: 1 - inference_batch_size: 256 - save_interval_epochs: 1 -model: - backbone: - architecture: - dim_model: 128 - max_context_length: 512 - num_layers: 6 - attention: - type: mha - n_heads: 8 - n_kv_heads: 8 - output_projection: true - feed_forward: - dim: 512 - activation: swiglu - normalization: - type: rmsnorm - norm_first: true - position_encoding: - type: rope - theta: 10000 - dropout: 0.1 - shared_layer_groups: [] - interfaces: - event_prediction: - input_columns: - - event - target_columns: - - event - categorical_decoder_special_tokens: - event: - - other - ingestion: - type: embedding - output_dim: 128 - decoder: - type: linear - prediction_length: 1 - support: 1 -dataset_training: - events: - model_interface: event_prediction - parts: - original: - metadata_config_path: configs/metadata_configs/events.json - increment: - metadata_config_path: configs/metadata_configs/events-increment.json - criterion: - event: CrossEntropyLoss - loss_weights: - event: 1.0 - freeze: - backbone: - freeze: - - attention.qkv -training_plan: - phases: - - name: incremental_finetuning - epochs: 2 - mode: sequential - sources: - - source: events.increment - batch_size: 64 - accumulation_steps: 4 - optimization: - model: - learning_rate: 0.0001 - optimizer: - name: AdamW - weight_decay: 0.01 - scheduler: - name: StepLR - step_size: 1 - gamma: 0.99 - scheduler_step_on: epoch - gradient_clip: 1.0 - optimizer_state: reset - - name: complete_retraining - epochs: 5 - mode: interleaved - selection: round_robin - sources: - - source: events - batches_per_selection: 4 - batch_size: 64 - accumulation_steps: 4 - optimization: - model: - learning_rate: 0.0001 - optimizer: - name: AdamW - weight_decay: 0.01 - scheduler: - name: StepLR - step_size: 1 - gamma: 0.99 - scheduler_step_on: epoch - gradient_clip: 1.0 - optimizer_state: reset -evaluation: - sources: - - source: events -export_generative_model: true -export_embedding_model: false -export_onnx: true -export_pt: false -export_with_dropout: false -``` - -The historical flat training schema is not accepted. In particular, -`training_spec`, top-level dataset paths/columns, `model.ingestion`, -`model.decoder`, and architecture-owned freezing are not canonical fields. - -For a causal objective, `target_offset` sets the target position relative to -each input position. With `target_offset: 1`, every decoded position predicts -its next token. A decoder `prediction_length` greater than 1 selects several -consecutive positions for inference; it does not make one position predict -several future tokens as in LLM multi-token prediction. Training currently -computes loss over all valid decoded positions, regardless of -`prediction_length`. - -## Ownership and resolution - -- `global_training` owns objective, window, optimizer, precision, - distribution, compilation, checkpoint, and data-loader behavior. Phase - entries own `epochs`. -- `model` contains exactly one backbone and one or more named interfaces. - Different interface names create distinct ingestion and decoder weights; - repeated references to one name share those weights. -- `dataset_training` owns parts, criterion/weights, class-share logging, - freezing, and the interface reference. -- Preprocessing metadata owns split paths, data types, class counts, ID maps, - special-token IDs, normalization facts, and stored-window layout. - -For categorical hard-label targets, set a target's criterion to -`AccurateCrossEntropyLoss` to preserve small loss values and target-logit -gradients at high confidence. `CrossEntropyLoss` retains the existing PyTorch -behavior. For example: - -```yaml -dataset_training: - events: - criterion: {event: AccurateCrossEntropyLoss} -``` - -The choice is per target and is checked when resuming a run. Class -weights and `loss_weights` retain their usual meaning: the target class weight -multiplies each example's loss, then the target loss weight scales the masked -sum divided by the number of valid positions. The criterion also works with -hashed target heads, which divide the target loss weight among their heads; -class weights remain unsupported for hashed targets. - -Every part of a dataset must resolve to the same schema, categorical semantics, -normalization contract, per-split window widths, alignment settings, and -file/folder storage form. Training validates the model view against each -selected training or evaluation split width. A source -named `events` iterates all parts in declaration order; `events.increment` -iterates only that part. Only parts selected by `evaluation.sources` require a -validation split. - -### Categorical multi-hash embeddings and targets - -Add `categorical_hashing` to a model interface, keyed by categorical column -name. The setting applies to an input column's embedding and, if the column is -also a target, to its prediction heads. For example, add these fields to a -training config with a backbone and dataset part: - -```yaml -model: - interfaces: - default: - input_columns: [accountId, merchantId] - target_columns: [accountId] - categorical_hashing: - accountId: - type: multi_hash - num_buckets: 50000 - num_hashes: 4 - seed: 1010 - merchantId: - type: qr - num_buckets: 1000 - ingestion: - type: embedding - output_dim: 128 - decoder: - type: linear - prediction_length: 1 - support: 1 -dataset_training: - default: - model_interface: default - criterion: - accountId: CrossEntropyLoss - loss_weights: - accountId: 1.0 -``` - -Here `accountId` uses both hashed input embeddings and hashed target heads; -`merchantId` uses hashed input embeddings only. `multi_hash` creates -`num_hashes` independently seeded tables of `num_buckets` rows and adds their -outputs. `num_hashes` must be positive; `seed` defaults to 0. `qr` uses the -quotient and remainder of the category ID as two table indices and multiplies -their embeddings. Its hash count is always two. Do not add `hashing` under -`ingestion`: the embedding ingestion reads the interface setting, including -when used by `temporal_conv` with `base_ingestion: embedding`. - -When `feature_embedding_dims` is omitted for a categorical-only embedding -ingestion, each column receives at least two dimensions. The remaining -`output_dim` positions are split in proportion to the logarithm of each -column's cardinality, rounding down first and assigning leftover positions -by largest fractional remainder (ties follow column order). For -`multi_hash`, the effective cardinality is the smaller of the original -cardinality and `num_buckets` per table. Other categorical columns use their -original cardinality. The output width must be at least twice the number of -categorical columns. Explicit `feature_embedding_dims` still controls each -column's width directly. - -A hashed target requires `CrossEntropyLoss` or `AccurateCrossEntropyLoss`. Its -loss weight is divided equally across the hash heads; `class_weights` cannot be -used for that target. The -stored data still contains the original categorical IDs, without extra hash -columns. Hash codes for classes eligible for prediction must be unique; config -resolution reports a collision if the chosen bucket count and hash count do -not distinguish them. - -In an `autoregressive_transformer` decoder branch, a hashed target must be the -last target in that branch's `target_columns` order. Earlier targets use a -full-size feedback embedding inside the decoder. - -Folder dataset parts accept `file_order: shuffled` (the default) or -`file_order: name`. For curriculum training, these respectively reshuffle file -blocks each epoch or keep them in lexicographic path order; curriculum order is -local to each file. Without curriculum training, eager `shuffled` loading keeps -the ordinary global sample shuffle. The setting has no effect on single-file -parts. -`training_plan` also accepts `curriculum_training: false` (the default). When -set to `true`, `curriculum_column` must name one column available in the -preprocessed data. Training visits lower values first inside each physical file -and reshuffles equal-valued samples each epoch. Curriculum ordering applies only -to training loaders for parts selected by the training plan; validation keeps -its ordinary unshuffled order. Ordering is intentionally not synchronized -across files, loader workers, or distributed ranks. -`curriculum_column` defaults to `null` and cannot be set when curriculum -training is disabled. When disabled, curriculum values are ignored and the -ordinary sample shuffle is preserved. - -```yaml -dataset: - part: - metadata_config_path: configs/metadata_configs/events.json - file_order: name -training_plan: - curriculum_training: true - curriculum_column: difficulty - epochs: 5 -``` - -`loss_weights` scales each target's contribution to the training and reported -aggregate loss. A weight of `0.0` disables that target's backward-loss -component while retaining its output and per-target accounting. At least one -target in each dataset must have a positive weight. - -## Optimizer plans - -The single-optimizer form applies one optimizer to all -parameters and uses the role’s `learning_rate`. To select parameters or -use multiple optimizers, replace it with an ordered plan. Each group has its -own learning rate; use each group’s `learning_rate` in this form: - -```yaml -training_plan: - phases: - - name: supervised - algorithm: supervised - optimization: - model: - optimizer: - groups: - - id: hidden_matrices - select: - semantic_groups: - - attention.* - - feed_forward.* - component: backbone - parameter_kind: weight - ndim: 2 - optimizer: - name: Muon - momentum: 0.95 - learning_rate: 0.02 - - id: remainder - select: otherwise - optimizer: - name: AdamW - weight_decay: 0.01 - learning_rate: 0.0003 -``` - -Each supplied selector condition must match. `semantic_groups` matches the -parameter catalog's semantic names using shell-style `*` patterns; a parameter -matches the list if it matches any pattern. Other available conditions are -`component` (`ingestion`, `backbone`, or `decoder`), `parameter_kind` (`weight`, -`bias`, or `other`), and `ndim`. At least one condition is required. The final -group must be `otherwise`, which receives every parameter not selected above. -Routes must be disjoint, all groups must receive parameters, and each trainable -parameter is assigned once. Group IDs are available to integration directives. - -The plan also supports different settings for one optimizer, for example two -AdamW groups with different weight decay. The named optimizer must be available -in Sequifier's optimizer registry. Muon availability depends on the installed -PyTorch version. The configured scheduler applies to every group; training -metrics report the first group's learning rate. With `OneCycleLR`, each group's -`learning_rate` is its peak rate. Omit `scheduler.max_lr` in plan mode; -Sequifier supplies the group rates in plan order. `OneCycleLR` derives the -starting rates from those peaks and `div_factor`. - -With `CyclicLR`, each group's `learning_rate` is also its peak rate. Add a -`base_learning_rate` below that peak to every optimizer group, and omit -`scheduler.base_lr` and `scheduler.max_lr`. For the plan above, the two base -rates could be `0.002` and `0.00003`, respectively. Configure the scheduler -with `name: CyclicLR`, `step_size_up: 2000`, and `scheduler_step_on: batch`. -Momentum cycling defaults to off for both cycle schedulers in plan mode; -explicit `cycle_momentum: true` is unsupported. `base_learning_rate` is only -valid with `CyclicLR`. - -## Single-phase compatibility - -For one phase, `global_training.batch_size` and `accumulation_steps` move onto -that phase; `learning_rate`, `optimizer`, `scheduler`, `scheduler_step_on`, and -`gradient_clip` move to `optimization.model` for supervised training only. -Multi-phase plans reject these fields, and duplicate global/local declarations -fail even when equal. Policy optimization must select its role explicitly. -Legacy supervised defaults retain no gradient clipping, require a single -optimizer's learning rate, and resolve null accumulation to 1. Explicit modern -roles use the defaults below. Composition, CLI and search normalize identically; -fingerprints use resolved phases. CLI flags use the hyphenated field names; -`--optimizer` and `--scheduler` accept YAML mappings. - -## Optimization across phases - -Each phase owns `batch_size` (32 by default), `accumulation_steps` (1), and -`optimization.`. Supervised phases have the single role `model`, meaning -the trainable model parameters. Imitation, DPO and GRPO have `actor`; PPO also -has `value`; IQL has `actor`, `value`, and `critic`. Unknown roles fail validation. -Omitted roles use Adam, learning rate 0.0003 and gradient clip 1.0. Set -`gradient_clip: null` to disable clipping. Optimizer plans take their rates from -their groups. Supervised roles default to StepLR (`step_size: 1`, `gamma: 0.99`); -policy roles default to ConstantLR (`factor: 1`, `total_iters: 1`). - -```yaml -training_plan: - phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential - sources: [{source: observations.train}] - batch_size: 64 - accumulation_steps: 1 - optimizer_state: reset - optimization: - model: - optimizer: {name: Adam} - learning_rate: 0.001 - scheduler: {name: StepLR, arguments: {step_size: 1, gamma: 0.99}} - scheduler_step_on: epoch - gradient_clip: 1.0 -``` - -`optimizer_state: reset | continue` describes the transition **into** this phase; -its default is `reset`. Reset creates new optimizer, scheduler and scaler state. -Continue preserves these states from the immediately preceding phase and requires -that phase's `handoff: last`, identical algorithm, role configurations, parameter -ownership and freeze selections. The first phase cannot continue. Supervised -continuation requires the same dataset ownership; policy continuation requires -the same interface. Mixed plans reject continuation because their artifact -handoffs do not preserve optimizer state. Policy OneCycleLR continuation is -also rejected. Native checkpoints restore the active lifecycle in either mode. Supervised -`handoff: best` requires an evaluation monitor and selects the preceding phase’s -best weights before starting the next phase; its monitor state is phase-local. - -Supervised `scheduler_step_on: epoch` advances once after a completed phase epoch; -ReduceLROnPlateau receives the configured evaluation monitor at that boundary -and requires `evaluation.monitor`. `batch` advances once per applied optimizer -update, after accumulation, rather than once per microbatch. Policy roles only -accept `scheduler_step_on: optimizer_step`, their default. Policy plateau and -compound schedulers are unsupported. Constructor arguments are validated before -training; the shared optimizer shape does not imply shared scheduler capabilities. - -For epoch OneCycleLR, the resolved `total_steps` is the phase's epoch count, or -the sum of epochs in its contiguous continuation chain. An explicit value must -match. Batch OneCycleLR requires an explicit `total_steps`. Preflight counts loader -batches using the same sequential/interleaved source scheduler, phase seeds and -accumulation rules as training, including partial updates flushed at dataset -boundaries. The budget sums over a contiguous continuation chain. This calculation -does not consume samples or advance loader RNG. Early stopping and explicitly -skipped updates may consume less than the authored maximum. -Policy OneCycleLR requires the exact maximum authored optimizer-update budget: - -| Algorithm | Optimizer updates per role | -| --- | --- | -| Imitation | epochs × ceil(training examples / (batch_size × accumulation_steps)) | -| IQL / DPO | updates | -| PPO | optimization_epochs × sum(ceil(rollout size / (minibatch_size × accumulation_steps))) | -| GRPO | optimization_epochs × sum(ceil(group trajectories / (batch_size × accumulation_steps))) | - -PPO `rollout_length` counts collected transitions; `minibatch_size` counts -transitions in each optimization microbatch. They are distinct budgets. IQL -uses `batch_size: 64` by default and one update per sampled batch; it does not -accept accumulation. DPO counts preference pairs; GRPO counts trajectories. -PPO defaults to `minibatch_size: 64`; other batched algorithms default to 32. -Early stopping, skipped updates or KL limits can consume less than the maximum. - -The authored schema is separate from resolved training phases. Checkpoints and -plan fingerprints contain resolved roles, batching and scheduler budgets; -reproduction does not depend on global optimizer values. `global_training` -contains execution/distribution, resume, data/window and model contracts only. - -## Training plans - -A sequential phase exhausts each source in listed order. An interleaved phase -uses `round_robin` or `weighted_random`. Each selection consumes at most -`batches_per_selection`; every source is still exhausted once per phase epoch. -Weights affect order and burst frequency, not the amount of data consumed. - -Gradient accumulation may cross part or source boundaries only while the -dataset stays the same. A dataset transition flushes a partial window using its -actual microbatch count. Dataset-specific frozen gradients are removed before -the optimizer step, preventing momentum and weight decay from changing frozen -parameters. - -With several evaluation sources, configure an explicit monitor when -validation-based saving or early stopping is enabled: - -```yaml -evaluation: - sources: [{source: events}, {source: telemetry.main}] - monitor: {source: events, metric: loss, mode: min} -``` - -Checkpoint interval fields retain distinct behavior. `save_interval_epochs`, -`save_interval_batches`, and `save_interval_minutes` create persistent -epoch/batch snapshots and refresh the rolling `latest` checkpoint; -`save_latest_interval_minutes` refreshes only `latest`. -`save_interval_val_loss` controls whether timed or batch snapshots also run -validation and record its monitored loss. Every checkpoint is written after -flushing any partial gradient-accumulation window. - -## Composable YAML files - -An entry file may declare complementary fragments with -`additional_config_paths`. Relative paths resolve against the entry file's -`project_root`. Fragments can contribute disjoint children under containers -such as `global_training`, `model.interfaces`, and -`dataset_training`; duplicate fields are rejected. CLI overrides are -applied after composition and before metadata resolution. - -Singleton fragments may likewise contribute disjoint fields under -`model.interface` or `dataset`. Normalization occurs after fragment -composition, so all fragments in one training config must consistently use the -singular or named spelling at each level. - -The training command accepts `--model-name`, `--seed`, and `--skip-metadata` as -configuration overrides. Dataset paths, columns, metadata paths, and device -selection must use their canonical YAML locations. - -## Artifacts - -ONNX is exported by default; PT inference bundles are opt-in. ONNX favors a -portable deployment runtime, while PT embeds its execution contract and retains -PyTorch behavior. See the "ONNX or PT?" section of the inference guide for the -trade-offs. - -Single-dataset filenames use ``, while multi-dataset logs, metrics, and -ONNX files use `-`. Part names are metric-row fields, not -filename components. PT inference bundles and exact-resume checkpoints remain -run-wide. Generated filenames do not use a `sequifier-` prefix. - -The PT inference bundle contains `artifact_type`, `format_version`, -`model_state_dict`, an execution-only `model_config`, and export metadata -(trace-site names and provenance). Optimizers, paths, parts, training plans, -evaluation policy, and dataset bindings remain outside that bundle. -`export_with_dropout` affects ONNX export only: enabling it exports -the ONNX graph in training mode and disables constant folding so dropout remains -active. - -## Image ingestion - -Image views are declared during preprocessing and selected by name in an -ingestion branch. They can be the only temporal inputs (`input_columns: []`) or -can be merged with tabular branches. Each configured view is required for every -sequence. An image encoder returns one embedding per source image; its frame -pool reduces the images assigned to one `itemPosition` to one embedding. - -```yaml -model: - interfaces: - prediction: - input_columns: [temperature] - target_columns: [nextAction] - preconditioning_image_encoder: - type: frozen - model: resnet18 - weights: IMAGENET1K_V1 - ingestion: - type: composite - branches: - measurements: - type: embedding - columns: [temperature] - output_dim: 32 - cameras: - type: images - views: [front, side] - output_dim: 128 - encoder: {type: resnet, depth: 18} - frame_pool: attention - view_merge: concat - merge: {type: concat} - decoder: {type: linear, prediction_length: 1} -``` - -Bind the selected views to their original folders in each dataset part. The -preprocessing metadata deliberately omits folder paths. A single selected view -may use a string `image_path`; multiple views require a name-to-path map. -Preconditioning images use `preconditioning_image_path` in the same way: - -```yaml -dataset_training: - camera_data: - model_interface: prediction - parts: - main: - metadata_config_path: configs/metadata_configs/camera_data.json - image_path: {front: data/front_images, side: data/side_videos} - preconditioning_image_path: - front: data/sequence_front_images - side: data/sequence_side_images - criterion: {nextAction: CrossEntropyLoss} -``` - -Image view names and structured image cell names cannot be target columns; -the available decoder heads produce tabular targets. For BERT training, temporal -image frames at selected positions receive the configured masked, random, or -unchanged replacement. Masked frames become black and random frames receive -random pixel values; frame validity stays the same. Sequence preconditioning -images remain available as static context. - -The trainable `resnet` supports depths 18, 34, 50, 101, and 152, or a custom -`block` (`basic` or `bottleneck`) with four `layers` depths. Bottleneck models -also support `groups`, `width_per_group`, and `replace_stride_with_dilation`. -Trainable ResNets use per-image group normalization so later frames cannot -affect earlier image embeddings during causal training. -The `frozen` encoder supports TorchVision `resnet18` and `resnet50` with -an explicit weight version. Its backbone stays frozen and in evaluation mode; -its output projection trains. The default image size and normalization match -the V1 ImageNet weight transforms. ResNet-50 V2 requires `resize_size: 232` -on its source; model resolution checks this. `frame_pool` is `mean` (default) or -`attention`; `view_merge` is `concat` (default) or `mean`. - -Image embeddings may also occupy cells in an existing structured layout. The -cell names are part of `feature_layout`; `image_cells` maps those names to image -view names. The usual structured processing blocks then combine image and -tabular cells: - -```yaml -feature_layout: - sensors: - axes: {source: [camera, measurement]} - columns: - camera_cell: {source: camera} - temperature: {source: measurement} -ingestion: - type: structured - layout: sensors - output_dim: 128 - image_cells: {camera_cell: front} - image_encoder: {type: frozen, model: resnet50, weights: IMAGENET1K_V1} - processing_blocks: - - {type: axis_pool, axes: [source], mode: mean} -``` - -The preconditioning source is independent of temporal views. One or more -preconditioning views are encoded and merged into a single sequence prefix -embedding. When scalar static conditioning is also set, both are fused into -that one prefix position. - -## Depth encoders and nested composites - -Use `global_training.read_format: pt` for preprocessed depth inputs. Layout -capacities come from dataset metadata. The following singleton model fragment -combines one shallow branch with a depth encoder; its metadata comes from the -preprocessing example in [preprocess.md](preprocess.md#named-depth-layouts). - -At every outer time step, a `depth_transformer` branch embeds the valid child -slots from its named layout, applies a masked transformer across the child -dimension, and uses the CLS output as that collection's single temporal token. -The shared backbone then models relationships across outer time steps. - -```yaml -model: - backbone: - initialization_seed: 20260910 - architecture: - dim_model: 128 - max_context_length: 128 - num_layers: 4 - attention: {type: mha, n_heads: 8} - feed_forward: {dim: 512, activation: swiglu} - position_encoding: {type: rope} - interface: - input_columns: [accountType, subitemType, subitemAmount] - target_columns: [nextAction] - ingestion: - type: composite - branches: - item: - type: embedding - columns: [accountType] - output_dim: 32 - subitems: - type: depth_transformer - layout: subitems - columns: [subitemType, subitemAmount] - feature_embedding_dims: {subitemType: 48, subitemAmount: 16} - output_dim: 96 - dropout: 0.1 - initialization_seed: 12345 - initialization: - attention.qkv: - weight: {method: xavier_uniform, gain: 1.0} - free_parameter: - weight: {method: normal, mean: 0.0, std: 0.02} - architecture: - dim_model: 64 - num_layers: 2 - attention: {type: gqa, n_heads: 4, n_kv_heads: 2} - feed_forward: {dim: 256, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: learned} - dropout: 0.1 - merge: {type: concat} - decoder: {type: linear, prediction_length: 1} -``` - -For a `gated` or `attention` merge, `merge.softmax` accepts `standard` -(the default) or `max_excluding`. The latter uses the [max-excluding softmax](https://arxiv.org/abs/2610.31642) -to retain small gradients when one branch score dominates, at additional -compute cost. Set it independently for each composite, including nested ones. -`concat` and `sum` merges require `standard`. - -Depth encoders support learned, sinusoidal, or rotary positions; MHA, MQA, or -GQA; LayerNorm or RMSNorm; the shared feed-forward activations and layer-sharing -configuration. `architecture.dropout` controls depth position and transformer -sites. The ingestion-level `dropout` controls the pooled output. Mixed -categorical/real features require explicit feature widths; homogeneous features -can divide `architecture.dim_model` using the ordinary ingestion width rules. - -For categorical-only depth features, automatic allocation gives each column at -least two dimensions, so `architecture.dim_model` must be at least twice the -number of columns. To use smaller shares, set positive widths for every column -in `feature_embedding_dims`. For example, with two categorical columns, -`feature_embedding_dims: {first: 1, second: 1}` permits one dimension each. -Explicit widths may sum to a value different from `architecture.dim_model`; -the depth input projection handles the difference. The pooled projection handles -differences between `architecture.dim_model` and `output_dim`. - -CLS occupies position zero, and physical slot `s` occupies position `s+1`. -Empty collections have a learnable CLS-only representation. Depth columns -cannot be targets because there is no depth decoder head. BERT training masks -valid child slots at selected outer time steps, using the configured masked, -random, or unchanged replacement; padded child slots stay untouched. -Autoregressive depth inference is excluded. - -A composite branch may itself be a composite. Every nested composite requires -`output_dim`; an omitted root composite width retains the existing backbone -input width. Child widths resolve recursively before merge projections are -built. `allow_shared_columns` applies to the immediate children of the node -where it appears, including overlaps between their descendant features. -`allow_unused_input_columns` and `auxiliary_input_columns` belong at the root; -non-default child policies are rejected. Temporal per-feature positions occur -at leaf outputs only, and global temporal positions occur in the backbone. -Composite merges do not add another temporal position stage. - -## Ordered autoregressive target decoding - -Use `type: autoregressive_transformer` when outputs at the same temporal -position depend on one another. `target_columns` is an ordered contract: each -target is conditioned on the backbone state and all targets before it. Training -and validation use parallel teacher forcing; inference generates that prefix -greedily inside the decoder. Categorical and real targets can be mixed. - -```yaml -decoder: - type: autoregressive_transformer - prediction_length: 1 - support: 1 - target_columns: [patch_length, byte_0, byte_1, magnitude] - architecture: - dim_model: 256 - num_layers: 2 - attention: {type: mha, n_heads: 4} - feed_forward: {dim: 768, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: rope, theta: 10000} - dropout: 0.0 - shared_categorical_target_groups: [[byte_0, byte_1]] - tie_input_output_embeddings: true -``` - -Shared groups must contain categorical targets with identical decoder-ID -mappings. Weight tying aliases each categorical input table with its output -projection. A composite decoder may mix this branch with ordinary linear or -MLP branches. For variable-length byte patches, predict length first and encode -unused slots with an explicit padding category. - -For BERT objectives, autoregressive transformer targets must not include `mask` -in `categorical_decoder_special_tokens`. Inference excludes mask predictions, -which would invalidate the prefix used to generate later targets. The default -decoder vocabulary includes `other` and excludes `mask`. Configure an explicit -empty token list for a categorical target to exclude `other` as well. - -Initialization overrides inherit per semantic group and per weight/bias target. -A child overrides only the targets it specifies; `preserve` keeps the constructed -value. Parameters are initialized once per identity. The ingestion adapter -inherits the root ingestion policy. Branch overrides that older versions ignored -now take effect; exact resume of those older runs is rejected. - -`initialization_seed` accepts integers from zero through `2**63-1`, excluding -booleans. An explicit seed isolates constructor and custom-initializer randomness -from the run stream. Descendants derive seeds using SHA-256 over version 1, -namespace seed, relative branch path, and phase; explicit child seeds start a new -namespace. Omitted seeds preserve the existing RNG stream and initialization -traversal. Initialization seeds do not control runtime dropout. Dropout continues -using the run RNG and checkpointed per-rank state. - -Dataset freezing can follow the same tree: - -```yaml -dataset: - part: {metadata_config_path: configs/metadata_configs/raw-items.json} - criterion: {nextAction: CrossEntropyLoss} - freeze: - backbone: - freeze: [attention.qkv] - ingestion: - branches: - subitems: - freezing_except: [free_parameter, ingestion.output_projection] - item: - freeze: [] -``` - -A local child selector replaces the inherited decision, including selective -unfreezing. `freeze: []` unfreezes the scope; `freezing_except: []` freezes it. -A node containing only `branches` delegates without changing inherited decisions. -Merge parameters retain their parent's policy; the adapter has its own dataset -selector. Unknown branches, policies below leaves, and contradictory alias -selections fail. Freezing leaves dropout active and permits gradients through -frozen modules. At optimizer boundaries, frozen gradients are removed **before** -AMP unscaling/overflow detection, so a frozen-only overflow cannot suppress a -healthy active update. Momentum and weight decay cannot update parameters whose -gradients are absent. - - -## Policy and mixed plans - -All algorithms use `training_plan.phases`. Declare policy sequence/preference -parts in `dataset_training`, and attach actions with `policy_binding` (one -interface) or `policy_bindings` (named interfaces). Run-wide settings remain in -`global_training`; no alternate run discriminator is needed. - -See the complete [supervised template](../../src/sequifier/templates/train-supervised.yaml), -[policy template](../../src/sequifier/templates/train-policy.yaml), and -[supervised-to-policy template](../../src/sequifier/templates/train-supervised-to-policy.yaml). -The [policy guide](../policy-learning.md) describes algorithm budgets and -[the mixed-plan guide](../supervised-policy-learning.md) describes artifact handoffs. - - -# Infer Command Guide - -For CPU numeric inference with externally calculated next-step inputs, see -[local forecast rollouts](../rollout.md). An explicit `rollout` configuration -selects a single seed window and saves a versioned trajectory through the -existing inference command. - -Models with `static_conditioning` read the processed static feature table -named in the preprocessing metadata. Each inference window receives the -configured static values for its `sequenceId`. -Autoregressive generation retains these values while temporal inputs slide -forward. - -Models with image ingestion need `image_path` for the original image or video -folders selected by the model. Use a string for one temporal view or a map from -view name to folder for multiple views. When the model has a preconditioning -image encoder, also set `preconditioning_image_path` to a string for the -`default` view or a map for named views. Preprocessing metadata records view -contracts and signatures, but omits folder paths. The folders must remain -available and unchanged after preprocessing. The inference loader applies the -same frame-to-`itemPosition` or time-column alignment and image transform used -during training. Timed sources also require the item-time index written by preprocessing. -PT and ONNX model execution accept image tensors and frame-valid masks; media -decoding stays outside the exported model. - -```yaml -image_path: {front: data/front_images, side: data/side_videos} -preconditioning_image_path: - front: data/sequence_front_images - side: data/sequence_side_images -``` - -`sequifier infer` produces predictions, probabilities, or embeddings from a -PyTorch (`.pt`) or ONNX (`.onnx`) model. - -```console -sequifier infer --config-path configs/infer.yaml -``` - -## Start here: ONNX - -> **GPU ONNX inference requires `onnxruntime-gpu`.** If you set `device: cuda`, -> install a build compatible with your CUDA and cuDNN versions. The CPU-only -> `onnxruntime` package does not provide `CUDAExecutionProvider`; setting -> `device: cuda` alone does not enable GPU inference. See the -> [ONNX Runtime CUDA compatibility guide](https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html). - -ONNX is the default training export and the deployment-oriented inference path. -Exports embed their execution contract. Select a training route when providing -model configuration through a training config: - -```yaml -project_root: . -model_path: models/event-model-best-5.onnx -training_config_path: configs/train.yaml -dataset: events -data_path: data/events-test.parquet -model_type: generative -device: cuda -``` - -The route supplies columns, types, objective, window sizes, and preprocessing -metadata. Add `part` when the dataset has several parts. Current ONNX exports -also embed this contract and metadata, so the route is optional. Older exports -need a training route or the full contract and metadata supplied explicitly. - -## ONNX or PT? - -| | ONNX (default) | PT | -| --- | --- | --- | -| Best fit | Portable, deployment-oriented inference. | Python/PyTorch workflows and easier configuration. | -| Runtime | ONNX Runtime on CPU or CUDA (with a CUDA-enabled ONNX Runtime installation). | PyTorch on CPU, CUDA, or MPS. | -| Configuration | Current exports embed their contract and metadata; older exports need a training route or explicit values. | Embeds its contract and metadata. | -| Behavior | Runs the exported graph; dropout requires a dropout-preserving export. | Retains PyTorch behavior and supports self-describing, multi-interface bundles. | - -Benchmark the target workload rather than assuming either runtime is faster. -Choose PT when portability matters less than a compact, self-contained config: - -```yaml -project_root: . -model_path: models/event-model-best-5.pt -data_path: data/events-test.parquet -model_type: generative -device: cuda -``` - -For multi-interface PT, add `model_interface`. `model_type` stays explicit for -both formats because inference may generate outputs or extract embeddings. -`project_root` is required (normally `.`), and `inference_batch_size` defaults -to `1`. - -## Effective configuration and validation - -Sequifier resolves inference configuration in this order: - -1. Compose `additional_config_paths`, then apply CLI overrides. -2. Fill omitted model-contract fields from a selected training route and/or a - self-describing PT artifact. -3. Load metadata from an explicit metadata path, selected dataset part, or PT - artifact, in that order. -4. Apply defaults and validate the complete configuration. - -Explicit values are assertions, not silent overrides. If an authored column, -type, objective, context, prediction length, interface, or metadata value -disagrees with its training config or portable artifact, inference stops and names the -conflicting field and source. Multiple model paths must share one contract. - -Relative model, data, and metadata paths resolve under `project_root`. -`additional_config_paths` also resolve there; fragments cannot include further -fragments or define the same field twice. - -## Fields - -### Model, data, and routing - -| Field | Default | Purpose | -| --- | --- | --- | -| `model_path` | Required | Model path, or a list of compatible model paths. | -| `model_type` | Required | `generative` or `embedding`. | -| `device` | `cpu` | ONNX: `cpu` or `cuda`; PT also supports `mps`. ONNX CUDA requires an installed runtime exposing `CUDAExecutionProvider`. | -| `project_root` | Required | Base for project paths; normally `.`. | -| `data_path` | Metadata test/last split | Input file or folder. Usually required with artifact-only inference because artifacts do not store split paths. | -| `preprocessing_data_path` | `null` | Derives the generated metadata path. | -| `metadata_config_path` | `null` | Explicit preprocessing metadata. | -| `image_path` | `null` | Original folder for one temporal view, or a view-to-folder map for every selected temporal view. Required when the model consumes temporal images. | -| `preconditioning_image_path` | `null` | Original folder for the `default` preconditioning view, or a view-to-folder map for named views. Required when the model uses preconditioning images. | -| `training_config_path` | `null` | Training config used to resolve a route. | -| `dataset` / `part` | `null` | Select a dataset and optional part from the training config. | -| `model_interface` | Implicit when unique | Select a route from a training config or portable artifact. | -| `read_format` | `parquet` | `csv`, `parquet`, or folder-based `pt`. | -| `write_format` | `csv` | `csv` or `parquet`. | -| `inference_batch_size` | `1` | Sequences processed per batch. | - -### Model contract - -The following fields are optional when supplied by a PT artifact or training -route, and otherwise required as applicable: `input_columns`, `target_columns`, -`column_data_types`, `target_column_types`, `training_objective`, -`context_length`, `target_offset`, and `prediction_length`. - -Preprocessing metadata may contain a Date/Datetime split column retained for -traceability. Inference ignores temporal metadata columns when `input_columns` -is omitted and rejects them if they are explicitly selected as model inputs or -targets. Other numeric columns from the same dataset remain usable normally. - -`window_stride` optionally evaluates several model windows inside each -stored preprocessing row. `null` uses the legacy right-aligned view. -`prediction_alignment_steps` defaults to `1` and must match the value in -dataset metadata. For aligned splits, it sets the number of consecutive -prediction groups evaluated per stored window. Aligned inference requires -`window_stride: null` and a stored width of at least `context_length + -target_offset + (prediction_alignment_steps - 1) × prediction_length`. -With `model_type: embedding`, aligned splits label each embedding by its target -`itemPosition` and omit positions outside the split. For causal models, the -embedding represents the context used to predict that row. Other splits label -embeddings by the input activation position. - -Folder inputs must carry `metadata.json` with matching storage and semantic -metadata, including category maps and normalization statistics. When using -metadata embedded in a portable artifact alone, inference can accept a wider -stored folder window if the remaining layout and semantics match. - -### Output and runtime options - -| Field | Default | Purpose | -| --- | --- | --- | -| `output_probabilities` | `false` | Write full distributions for categorical targets. Invalid for embeddings. | -| `decode_categories` | `true` | Decode categorical predictions to their original values. | -| `sample_from_distribution_columns` | `null` | Sample these categorical targets instead of using argmax. | -| `infer_with_dropout` | `false` | Enable dropout at inference; ONNX also requires dropout-preserving export. | -| `enforce_determinism` | `false` | Request deterministic PyTorch algorithms. | -| `seed` | `1010` | Random seed. | -| `autoregressive` | `false` | Feed predictions back for multi-step generation. | -| `generation_steps` | `null` | Required positive step count when autoregressive is enabled. | - -For an `autoregressive_transformer` decoder branch, same-position targets are -generated greedily in configured `target_columns` order. Returned categorical -probabilities are conditional on the preceding greedy predictions; they are not -independent marginals that can be resampled into a coherent joint result. -Consequently, `sample_from_distribution_columns` must not include a target from -an autoregressive transformer branch. External sampling remains available for -ordinary decoder branches. - -Autoregressive inference requires a forward-looking generative model, prediction length -`1`, and identical input and target columns. For tabular CSV or Parquet input, -it begins at the first input window for each sequence and generates the same -number of steps for every sequence. Folder-based PT input is processed using -its stored windows rather than this first-window reduction. - -## CLI overrides - -`--data-path`, `--input-columns`, `--metadata-config-path`, `--model-path`, -`--seed`, `--dataset`, `--part`, `--model-interface`, `--model-type`, -`--enforce-determinism`, and `--action-selection` override YAML values. -`--randomize` takes precedence over `--seed`. `--skip-metadata` skips external -metadata loading; inline metadata, a selected training route, or a self-describing -PT artifact must then provide the required metadata. `--skip-metadata` is -forecast-only and is rejected on policy inference. - -## Outputs - -- Generative predictions: `outputs/predictions//part-NNN.`. -- Categorical probabilities, when enabled: - `outputs/probabilities///part-NNN.`. -- Embeddings: `outputs/embeddings//part-NNN.`. - -`` is the model artifact filename without its extension. Prediction and -embedding outputs contain sequence and window identifiers. Probability files -contain only the class-probability columns for their target. Categorical -predictions are decoded when `decode_categories` is enabled, and normalized -real predictions are restored to their original scale. Every input writes one -or more numbered parts. - -## Portable depth models and dropout modes - -New ONNX artifacts embed the selected interface, vocabulary/normalization -metadata, layouts, input name mapping, output descriptors, and fixed capacities. -They can resolve inference without the training YAML. Older flat ONNX files keep -the existing training-config/explicit-metadata fallback. - -The `depth_transformer` ingestion branch is part of the saved model contract; -inference does not configure it again. Inference supplies the matching named -layout tensors and masks through the preprocessed PT input. - -```yaml -project_root: . -model_path: models/item-model-best-5.onnx -model_type: generative -read_format: pt -data_path: data/raw-items-split2 -device: cpu -infer_with_dropout: false -seed: 1010 -``` - -Use the corresponding embedding artifact with `model_type: embedding`, or change -`model_path` to a portable PT model. `read_format: pt` describes preprocessed -input; it is independent of the model artifact extension. Deep inputs retain -`[B,T,D]` and explicit boolean depth masks through batching. New graphs accept -symbolic batch size with fixed temporal/depth capacities, including a partial -last batch. Legacy static graphs repeat features and masks together and trim -synthetic outputs. Empty input chunks do not invoke ONNX Runtime. - -New ONNX graphs have a fixed dropout mode: - -| `export_with_dropout` | `infer_with_dropout` | Behavior | -| --- | --- | --- | -| false | false | Evaluation graph; deterministic dropout behavior. | -| false | true | Error: export a stochastic graph. | -| true | true | Stochastic graph; runtime dropout remains enabled. | -| true | false | Error: export an evaluation graph. | - -PT artifacts retain executable modules and can switch either way at inference. -Stochastic ONNX CLI inference seeds ORT once before session creation, then lets -successive calls advance the session RNG. Separate processes with the same seed, -provider, toolchain, graph, and call/batch ordering define the intended -repeatability boundary. Cross-provider identity and PT/ORT matching random draws -are not promised. The direct `Inferer` API never reseeds the process-global ORT -RNG. Legacy ONNX graphs lacking mode metadata cannot make this guarantee and -emit a warning when stochastic inference is requested. - -Selected metadata must match layout names, selected deep membership, capacity, -position column/base, and gap policy. Input validation rejects missing masks, -incorrect ranks/capacities, forbidden gaps, illegal categories, and non-finite -values. Legal masked values are sanitized before narrowing and inside the graph. -The declared provider must be available; CUDA inference requires a CUDA-enabled -ORT installation. Outputs stay indexed by outer item coordinates. - - -## Policy model interfaces - -`model_type` selects the authored route before artifact resolution. `generative` -and `embedding` use forecast schemas; `policy` uses `PolicyInferenceConfig` -directly. Unknown and cross-route fields fail validation even when set to their -default values. CLI overrides and Python configuration loaders follow the same -selection rules. - -Policy inference requires one portable PT artifact and one unambiguous policy -interface. Supply raw CSV/Parquet observation rows; history alignment, vocabulary -and normalization come from that artifact. Policy supports the common model path, -interface, data path, formats, device, seed, `enforce_determinism`, batch size fields, plus: - -| Policy field | Default / meaning | -| --- | --- | -| `output_path` | null; write policy predictions here instead of the generated output path | -| `action_selection` | `deterministic`; choose the policy mode. `stochastic` samples. | -| `desired_return` | null; fixed return budget for a return-conditioned policy | -| `desired_return_column` | `desired_return`; per-row budget when no fixed budget is supplied | -| `previous_action_columns` | {}; input-column mapping for recorded previous actions | -| `sequence_id_column`, `item_position_column` | `sequenceId`, `itemPosition`; order raw history rows | -| `output_probabilities` | false; include policy distribution outputs | -| `rollout` | null; execute the policy in a version-3 online environment | - -Policy assertions may specify `metadata_config_path`, `input_columns`, -`static_columns`, `target_columns`, `column_data_types`, `target_column_types`, -`training_objective`, `context_length`, `target_offset`, `prediction_length`, and -`decoder_support`. Every supplied assertion must match the selected artifact. -Forecast fields `sample_from_distribution_columns`, `infer_with_dropout`, -`decode_categories`, dataset/part selection, image path overrides, -`autoregressive`, and `generation_steps` are rejected on policy requests. -Policy-only return/action/history options and the policy-only `output_path` -are rejected on forecast requests. Embedding requests also reject generative -sampling, category decoding, probability output, autoregression and rollout fields; -dropout remains supported for embeddings. - -```yaml -project_root: . -model_type: generative -model_path: models/policy/model.pt -model_interface: actor -data_path: observations.csv -read_format: csv -device: cpu -seed: 1010 -enforce_determinism: true -action_selection: stochastic -rollout: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - units: {} - max_steps: 8 - seed_row_index: 0 - trajectory_path: outputs/trajectory.json -``` - -Rollout schemas are selected by the inference route; do not declare `rollout.type`. -Forecast rollout requires `bindings` and a version-1 environment. Policy rollout -rejects bindings and requires the distinct version-3 environment protocol, -including implementation identity and matching units. Both use `max_steps` and -optional `max_episode_steps`. Forecast `seed_window_index` counts prepared -windows; policy `seed_row_index` counts raw input rows, each zero-based in input -order. The legacy `seed_index` spelling is accepted within each route, but -providing both spellings is an error. Trajectory output metadata retains its -public `seed_index` key and existing units. - -`enforce_determinism` controls deterministic execution, independently of action -selection and seed. A seeded stochastic policy can reproduce its sampled sequence -under the same supported runtime, device and input ordering. Online evaluation -uses `action_selection` with the same meaning; its execution determinism is the -training run's `global_training.enforce_determinism`. See the -[environment protocols](../rollout.md) and [policy guide](../policy-learning.md). - - -With `model_type: generative`, a policy-bound interface defaults to policy -inference; other interfaces retain forecast behavior and supported formats. -Select `model_interface` when ambiguous. Optional `task: policy` requires one -policy-bound PT artifact; `task: forecast` requests forecasts, including from a -policy-bound interface with a supported forecast contract. Legacy -`model_type: policy` forces policy inference and conflicts with `task: forecast`. -Embedding rejects `task`; each route retains its own option and rollout schema. -The CLI accepts `--task policy|forecast`. - - -# Visualize Training Command Guide - -The `sequifier visualize-training` command reads the structured metric files generated during training and hyperparameter search to create interactive Plotly HTML visualizations of the training and validation losses. It supports viewing a single model's progress or comparing multiple models side-by-side. - -## Usage - -```console -# Visualize a single model -sequifier visualize-training my-model-name - -# Visualize multiple models side-by-side -sequifier visualize-training model-A,model-B,model-C - -# Visualize every run from a hyperparameter search -sequifier visualize-training my-hyperparameter-search - -# Visualize models listed in a text file -sequifier visualize-training path/to/models.txt --log-scale - -``` - -## Arguments - -Unlike other commands that rely on a YAML config, `visualize-training` is configured directly via command-line arguments. - -| Argument | Type | Default | Description | -| --- | --- | --- | --- | -| `models` | `str` | **Required** | A model name, hyperparameter-search name, comma-separated list of model names, or path to a `.txt` file containing model names (one per line). A search name includes all models named `[SEARCH]-run-[NUMBER]`. | -| `--log-scale` | `flag` | `False` | Use a logarithmic scale on the y-axis for the loss curves. | -| `--bucket-training-batches` | `int` | `null` | Smooths the training loss curve by averaging the loss over a specified number of batches. **Must be a multiple of the logged batch interval** used during training. | -| `--project-root` | `str` | `.` | The root directory of your Sequifier project. | - -For a single-dataset model, the command reads `logs/[MODEL_NAME]/[MODEL_NAME]-training-full.csv` and `logs/[MODEL_NAME]/[MODEL_NAME]-validation-full.csv`. The corresponding files without `-full` contain only condensed global-loss records. - -## Outputs - -The interactive HTML reports are saved in the `outputs/visualization/` directory. - -* **Single Model:** `outputs/visualization/[MODEL_NAME]-training-visualization.html` (Includes global losses and normalized variable validation losses if applicable). -* **Multiple Models:** `outputs/visualization/multi-model-training-visualization.html` (Side-by-side comparison of validation and training losses across all specified models). -* **Hyperparameter Search:** `outputs/visualization/[SEARCH_NAME].html` (Includes all valid runs and lists skipped invalid runs and their reasons). - -If every run in a hyperparameter search is invalid, Sequifier still creates the report with an empty plot and the invalid-run list. - -When comparing multiple models, their initial baseline validation loss must match unless `SKIP_BASELINE_CHECK` or `SEQUIFIER_SKIP_BASELINE_CHECK` is set. - - -# Hyperparameter Search Command Guide - -`sequifier hyperparameter-search` searches over complete canonical training -configurations with Optuna. It supports Bayesian optimization, random sampling, -finite grid search, cooperative pruning, and custom single- or multi-objective -evaluation. - -```console -sequifier hyperparameter-search --config-path configs/hyperparameter-search.yaml -``` - -## Minimal search configuration - -Every search starts from a canonical training config named by -`base_config_path`. The `parameters` tree describes fixed replacements and -search spaces using the same paths as the training schema. - -```yaml -base_config_path: train.yaml -name: transformer-width-search -model_config_write_path: configs/hp-search -method: bayesian -trials: 40 - -parameters: - global_training: - context_length: [64, 128] - batch_size: [16, 32] - learning_rate: - low: 0.0001 - high: 0.001 - log: true - model: - backbone: - architecture: - num_layers: {low: 4, high: 8, step: 2} - training_plan: - epochs: [2, 4] -``` - -`base_config_path` resolves relative to the hyperparameter-search entry file. -The base may itself be a composed training config. If the search config supplies -`project_root`, that value is used in every generated trial; otherwise the base -training config's root is inherited. - -For a singleton base, use `model.interface`, `dataset.part`, and -`training_plan.epochs`. Named paths remain available for multi-value bases. - -The historical self-contained search schema and historical flat training base -configs are not accepted. Every generated trial is validated as an authored -canonical `SequifierConfig` before training begins, so unknown paths, invalid -references, incompatible component types, and cross-field violations fail with -their canonical validation paths. - -`model_name` and `project_root` cannot appear in `parameters`. Generated model -names use `[name]-run-[index]`, and `project_root` is controlled by the -top-level search field. - -## Parameter expressions - -Fields omitted from `parameters` retain their base values. Parameter expressions -have these forms: - -| Form | Meaning | -| --- | --- | -| Scalar | Fixed replacement. | -| List on a scalar field | Categorical choices. | -| `{low, high, step?, log?, type?}` | Integer or float distribution. `type` may be `int` or `float`; otherwise the base value and bounds determine it. | -| `{choices: [...]}` or `{$choices: [...]}` | Categorical choices for an entire value, including mappings and lists. | -| `{fixed: value}` or `{$fixed: value}` | Unambiguous fixed replacement for a mapping or list. | -| Numeric keys under a base list | Recursive replacement of zero-based list entries. | -| `{variants: [...]}` or `{$variants: [...]}` | Paired partial mapping variants, optionally followed by independently sampled sibling fields. | - -Integer ranges default to `step: 1`. Grid search requires a `step` for float -ranges because an unstepped float interval is infinite. Logarithmic integer -ranges require `step: 1`; logarithmic float ranges cannot use `step`. - -A direct list of lists samples a complete list-valued field. Prefer the explicit -`choices` wrapper for mapping or list candidates: - -```yaml -parameters: - model: - backbone: - architecture: - choices: - - dim_model: 128 - max_context_length: 512 - num_layers: 4 - attention: {type: mha, n_heads: 8} - feed_forward: {dim: 512, activation: swiglu} - - dim_model: 256 - max_context_length: 512 - num_layers: 6 - attention: {type: gqa, n_heads: 16, n_kv_heads: 4} - feed_forward: {dim: 1024, activation: swiglu} -``` - -Each choice replaces the complete overridden subtree, keeping coupled values -valid. The same mechanism can select complete ingestion or decoder components, -training phases, or other canonical subtrees. - -Use `variants` when candidates should patch the base mapping and other sibling -fields should remain independent. A variant that changes a component's `type` -starts from that new component shape rather than retaining fields belonging to -the old type. - -```yaml -parameters: - model: - interfaces: - event_prediction: - ingestion: - variants: - - {type: embedding, output_dim: 128} - - {type: passthrough, output_dim: 128} - dropout: [0.0, 0.1] -``` - -## Search controls - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `base_config_path` | `str` | Yes | - | Canonical training config used as the base. | -| `parameters` | `mapping` | Yes | - | Recursive fixed values and search spaces. May be empty. | -| `project_root` | `str` | No | Base value | Root written to every generated training config. | -| `additional_config_paths` | `str`, `list[str]`, or `null` | No | `null` | Complementary direct fragments of the search config. Relative paths resolve against the entry config's `project_root`; recursive composition and duplicate fields are rejected. | -| `name` | `str` | Yes | - | Study name and generated-run prefix. | -| `model_config_write_path` | `str` | Yes | - | Directory, under `project_root`, for generated trial configs. | -| `method` | `bayesian`, `sample`, or `grid` | No | `bayesian` | Optuna TPE, random, or exhaustive finite-grid sampling. | -| `trials` | `int` | Except grid | - | Target total number of completed or pruned runs in the persisted study. | -| `global_seed` | `int` or `null` | No | `null` | Sampler seed. Training seeds belong in canonical `parameters`. | -| `prune_trials` | `bool` | No | `true` | Enables cooperative pruning. Distributed pruning remains experimental. | -| `pruning_warmup_epochs` | `int` or `null` | No | `null` | Complete epochs before pruning may begin. Mutually exclusive with batch warmup. | -| `pruning_warmup_batches` | `int` or `null` | No | `null` | Training batches before pruning may begin. Mutually exclusive with epoch warmup. | -| `evaluation_metrics` | `list[str]` or `null` | No | `null` | Metric names expected from a custom evaluation script. | -| `evaluation_metric_directions` | `list[minimize\|maximize]` or `null` | With metrics | `null` | One optimization direction per metric. | -| `evaluation_script` | `str` or `null` | With metrics | `null` | Script invoked with the best exported model's evaluation ID. | -| `evaluation_inference_config` | `str` or `null` | No | `null` | Inference config run before the custom evaluation script. | - -For grid search, omitting `trials` runs the complete finite grid. If it is -provided, it must exactly equal the grid size. For Bayesian and random search, -`trials` is a target total across invocations of the persisted study. If an -identical completed or pruned parameter set is proposed again, it is recorded -as a failed duplicate and does not consume a generated run number or count -toward the target. - -## Custom evaluation - -Without custom metrics, Sequifier minimizes the best validation loss. With one -metric it performs single-objective optimization; with several it records the -Pareto front. - -```yaml -evaluation_metrics: [accuracy, latency_ms] -evaluation_metric_directions: [maximize, minimize] -evaluation_inference_config: configs/infer-validation.yaml -evaluation_script: scripts/evaluate.py -``` - -The evaluation script receives the exported model's evaluation ID, formatted -as `-best-`, as its only -argument. It must write -`outputs/evaluations/[evaluation-id].json` under `project_root`, containing all -configured metric names. Extra metrics are allowed but produce a warning and -are ignored by the optimization. - -## CLI and outputs - -The search definition comes from YAML. `--skip-metadata` is the only -configuration-related command flag: it validates the canonical base without -loading metadata-derived values, so all required authored fields must already -be present. - -Generated canonical training configs are written below -`model_config_write_path`. The Optuna SQLite study is persisted at -`state/optuna/[name].db` under `project_root`, allowing later -invocations to continue toward the configured total. - - -# Distributed and Multi-Node Training in Sequifier - -Sequifier natively supports multi-GPU and multi-node training using PyTorch's `DistributedDataParallel` (DDP) and `FullyShardedDataParallel` (FSDP). - -## 1. Prerequisites: Preprocessing for Distributed Training - -To use distributed training, your data must be sharded into multiple files so that different GPUs can read different chunks simultaneously without memory bottlenecks. - -In your `preprocess.yaml`, you **must** write sharded output: - -```yaml -merge_output: false -``` - -For production multi-GPU training, use PyTorch tensor shards: - -```yaml -write_format: pt -``` - -*Note: Distributed training is not supported if your data is kept as a single `csv` or `parquet` file. You must use `merge_output: false` to generate a folder of sharded files.* - -> **Beta Notice for Parquet in Distributed Training:** -> While `write_format: parquet` is supported for distributed training, it is currently considered **Beta**. Because Parquet chunk reading relies on Polars' multi-threading, using it alongside PyTorch's multiprocess `DataLoader` in heavy multi-GPU environments can lead to CPU thread contention, high RAM usage, or NCCL timeouts. -> **Recommendation:** For production multi-GPU runs, use `write_format: pt`. It relies on native PyTorch serialization and is significantly more stable under heavy hardware loads. - - -## 2. Configuration: `train.yaml` - -Once your data is preprocessed into `.pt` shards, or beta `.parquet` shards, you need to tell the Sequifier training engine to expect a distributed environment. - -In your `train.yaml`, configure the canonical `global_training` block: - -```yaml -global_training: - read_format: pt # or parquet for beta sharded Parquet loading - distributed: true - data_parallelism: fsdp # or ddp - fsdp_cpu_offload: false # omit if using ddp; set true to offload FSDP parameters to CPU RAM - layer_type_dtypes: null # required for FSDP; use layer_autocast for mixed precision - torch_compile: inner # use inner or none for FSDP; use outer or none for DDP - world_size: 32 # The TOTAL number of GPUs across all nodes (e.g., 8 nodes * 4 GPUs = 32) - backend: nccl # 'nccl' is the standard and most efficient backend for NVIDIA GPUs - -``` - -When shards do not divide evenly across ranks, Sequifier automatically pads shorter ranks with repeated samples for step alignment. Those repeats are masked out of loss calculation, so each real sample contributes once. - -## 3. Launching the Training Job - -How you launch the training depends on whether you are using a single machine with multiple GPUs, or multiple machines (nodes) connected over a network. - -### Scenario A: Single-Node, Multi-GPU - -If you are running on a single machine that has multiple GPUs (e.g., an AWS EC2 instance with 4x A100s), Sequifier can handle process generation internally using `torch.multiprocessing.spawn`. - -You simply run the standard command: - -```bash -sequifier train --config-path configs/train.yaml - -``` - -Sequifier will read the `world_size` config parameter and automatically spawn that exact number of worker processes. - -### Scenario B: Multi-Node, Multi-GPU (HPC / Slurm) - -Sequifier cannot automatically spawn Python processes across physical network boundaries. For multi-node training, you must use an external cluster manager (like Slurm) combined with PyTorch's `torchrun` utility. - -When `sequifier` detects `torchrun` environment variables (like `RANK` and `WORLD_SIZE`), it bypasses its internal spawner and attaches to the distributed network established by the cluster. In that mode, the environment `WORLD_SIZE` is used. - -Here is a standard `sbatch` script template for launching Sequifier across multiple nodes: - -```bash -#!/bin/bash -#[SBATCH COMMANDS] - -MASTER_NODE=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1) - -srun torchrun \ - [-- torchrun args]... - $(which sequifier) train --config-path=configs/train.yaml -``` - -### Important Considerations for Multi-Node - -* **Batch Size:** The `batch_size` in your `train.yaml` is the **per-process** batch size. If `batch_size` is 100 and `world_size` is 32, each synchronized backward pass covers 3,200 samples. With gradient accumulation, the samples per optimizer update are `batch_size * world_size * accumulation_steps` (apart from a final partial accumulation window). -* **Learning Rate:** You may need to scale your `learning_rate` up if you drastically increase your global batch size via distributed training. -* **Data Access:** All nodes must have access to the same shared filesystem (e.g., NFS, GPFS) where the `project_root` and the sharded preprocessing output are stored. - - -# Policy learning - -For supervised checkpoint attachment, mixed real/categorical heads, full model -input schemas, and mixed supervised/policy plans, see the -[unified learning guide](supervised-policy-learning.md). The configuration below -documents the compatible single-categorical authoring surface. - -`sequifier train` accepts one `SequifierConfig` with ordered -`training_plan.phases`. Supervised, imitation, IQL, PPO, DPO and GRPO use the same -model/interface and artifact contracts. A policy binding selects named real or -categorical action heads. See the [training guide](configs/train.md) for phase -ownership, scheduling budgets, optimizer groups and transition validation. -Policy execution currently uses a local CPU/CUDA/MPS process; standalone -supervised training also supports distributed execution. - -## Authoring a run - -Use the [policy template](../src/sequifier/templates/train-policy.yaml), -[PPO template](../src/sequifier/templates/train-ppo.yaml), or -[multiphase example](example_inputs/configs/train-multiphase-control.yaml). -The latter contains imitation → IQL → DPO → GRPO → PPO. Generate its data with: - -```shell -python documentation/example_inputs/policy_control_data.py data -sequifier train --config-path documentation/example_inputs/configs/train-multiphase-control.yaml -``` - -All sources live in `dataset_training..parts.`. Use `type: sequence` -for raw tables or exported sequence manifests, prepared metadata parts for -supervised data, and `type: preferences` with `manifest` for preference pairs. -Phase source references are objects such as `{source: decisions.default}`; -DPO `preferences` uses the same reference shape. Every data-bearing dataset -selects its model interface explicitly when more than one is available. - -Run-wide `context_length`, normalization, validation fraction (0.2), episode -cache size (8), determinism and resume settings live in `global_training`. -`resume` defaults to `{policy: never, mode: reset_environments}`; use `if_exists` -or `required` with a checkpoint path, and `exact` for supported online adapters. -`policy_binding` applies to a single model interface; use `policy_bindings` keyed -by interface name for multiple interfaces. Environment units must match the binding. -No optimizer defaults live in this run-wide block. - -Default outputs are `models//model.pt`, -`outputs/training//report.json`, and -`outputs/training//phases`. The resolved configuration records -role defaults and execution contracts. Fingerprints and checkpoints include -phase settings and validate source, reference and environment identities on resume. - -## Selection and optimization - -Every phase has a unique identifier `name` and `handoff: best | last` (default -`last`). Best selection retains the earliest finite best on ties. The final -update always evaluates. Missing finite evaluation for `best` fails explicitly. -`last` selects the most recently completed optimizer transaction. The final -export uses the last phase's selected actor, with metric, transaction and weight -hash recorded; metrics from different phases are never compared globally. - -Each phase configures trainable roles under `optimization`. Imitation, DPO and -GRPO have `actor`; PPO has `actor` and `value`; IQL has `actor`, `value` and -`critic` (the twin Q networks share that optimizer). Omitted roles use defaults. -For example: - -```yaml -optimization: - actor: {learning_rate: 0.005, optimizer: {name: AdamW}, gradient_clip: 1.0} -``` - -Every role defaults to `learning_rate: 0.0003`, `optimizer: {name: Adam}`, -`scheduler: {name: ConstantLR, factor: 1.0, total_iters: 1}`, and -`gradient_clip: 1.0`. Unknown roles fail validation. - -Imitation, DPO and GRPO share these batching and scheduling fields: - -| Field | Default | -| --- | --- | -| `batch_size`, `accumulation_steps` | 32, 1; DPO counts pairs, GRPO counts whole trajectories | -| `evaluation_interval`, `checkpoint_interval` | 1; optimizer updates for imitation/DPO, completed collection/optimization updates for GRPO | - -Optimizers support Adam, AdamW, SGD and RMSprop. Their constructor arguments -belong alongside `name` (or in `arguments`); `lr` and parameters belong to the -role configuration. Schedulers advance on completed optimizer steps; plateau/compound -schedulers are unsupported. `OneCycleLR.total_steps` must equal the authored -maximum optimizer-step budget. Accumulation weights by the actual number of -examples, including final partial batches. Network stochastic layers are disabled -while gradients remain enabled; likelihood ratios use reproducible scores. - -## Offline sequences and imitation - -Sources accept CSV/Parquet tables, JSON rollout logs, committed sequence -manifests, or scalar prepared histories. A row has current observations and its -executed action; optional rewards/successors describe the resulting transition. -`PolicyTableColumns` exposes explicit observation/static/action/reward/group/ID -mappings. Temporal positions must be contiguous, zero-based at episode starts. -Fragments require a known preceding action. -`columns.action` conflicts with nonempty `columns.actions`; likewise for -`previous_action` and `previous_actions`. Empty plural maps (`{}`) are unused -and may coexist with a singular mapping, including structured actions. Reward-free sequences are ordinary -`SequenceEpisode` objects with absent transition capabilities, rather than -fabricated reward-bearing trajectories. - -`policy_imitation` uses `sources`, optional disjoint `evaluation.sources`, and -`epochs`. It minimizes action negative log likelihood at each valid causal -decision. The fixed `evaluation.monitor` is `{metric: validation_action_nll, mode: min}`. BC requires no -rewards. Return conditioning requires recorded rewards and either true task -termination or an explicit finite return horizon, using the declared discount -and scale. Unknown return tails are excluded. The input return label at decision -`t` includes reward `r_t`; the current action is never an input at that decision. - -Episode/group/duplicate-content components remain wholly in training or -validation. Explicit validation sources undergo the same overlap checks. -Episodes load through bounded caches; windows are constructed on demand. -Episode fitting and decision indexing are shared independently of imitation -batches, IQL transitions, and preference pairs. - -Offline validation uses `evaluation.sources: [{source: demonstrations.holdout}]`; -omitting sources preserves automatic splitting. `evaluation.monitor` fixes -`mode: min` and `metric: validation_action_nll` for imitation/IQL or -`validation_dpo_loss` for DPO. These fields are assertions, not tuning choices. -Legacy `validation_sources` (DPO: `validation_preferences`), `monitor`, and -`direction` normalize to this shape; combining any with `evaluation` fails. -Online evaluation retains its environment/episode options and nested monitor. - -## Offline IQL - -`iql` requires complete rewards, termination/truncation and bootstrap flags, -with genuine successors for bootstrapped decisions. Nonterminal transitions -bootstrap; true terminal transitions do not. Never substitute a reset -observation for a successor. - -The phase has `updates` (required), `batch_size` (64), `gamma` (0.99), -`expectile` (0.7), `inverse_temperature` (3), `max_weight` (100), and -`target_update_rate` (0.005). Independent value and twin-Q networks use -expectile regression and one-step Bellman regression with Polyak target-Q -updates. Actor extraction minimizes action NLL weighted by -`min(exp(inverse_temperature * (min(target_Q1,target_Q2)-V)), max_weight)`. -Each update samples transitions uniformly with replacement. Held-out unweighted -`validation_action_nll` is the minimizing selection monitor; Bellman residual, -values, Q and weight diagnostics are reported separately. - -The `optimization.actor`, `optimization.value` and `optimization.critic` roles -have independent optimizers and schedulers. All gradients are validated and -clipped once before any optimizer steps. Evaluation and checkpoint intervals -default to 100 updates. Critics are fresh even after another IQL phase and are -absent from the portable actor. - -## DPO preferences - -The inspectable manifest is separate from its sequence sources: - -```json -{ - "format": "sequifier.preference-pairs", - "version": 1, - "pairs": [{ - "pair_id": "pair-1", - "context_id": "initial-condition-1", - "chosen": {"source": "decisions", "episode_id": "good", "start": 0, "end": 8}, - "rejected": {"source": "decisions", "episode_id": "bad", "start": 0, "end": 6} - }] -} -``` - -Span bounds index the loaded episode/fragment rows, zero-based, start-inclusive -and end-exclusive. Both sides need -at least one executable action. Unequal lengths are permitted. Canonical -observations, static fields, prior-action histories and valid masks must match -at the branch point. Each subsequent score uses that alternative's actual -observations and prior actions. Duplicate pair IDs, identical/overlapping -alternatives, unsupported labels, non-finite features and incompatible initial -contexts fail before updates. - -Splitting joins shared context/group IDs, episode IDs and duplicate learning -content transitively; explicit validation manifests obey the same constraints. -At least two independent components are required for automatic splitting. -Source identities, exact spans, fingerprints and membership persist in the run -checkpoint. The manifest must state the preferences; reward is not interpreted -as a preference. - -`dpo` requires `preferences` and `updates`; optional `evaluation.sources` -selects explicit validation preference manifests using the same `[{source: dataset.part}]` -shape as other offline phases. Its fixed monitor is -`{metric: validation_dpo_loss, mode: min}`. `beta` defaults to 0.1 and must be -finite positive. `scoring_chunk_size` defaults to 64 causal windows. For the -summed FP32 action log likelihood `s`, it minimizes equally weighted pair losses: - -```text -margin = (s_actor(chosen)-s_actor(rejected)) - (s_ref(chosen)-s_ref(rejected)) -loss = -logsigmoid(beta * margin) -``` - -Padding and unselected context positions contribute zero. Losses are averaged -over pairs, without length normalization or an added supervised term. Reports -include lengths, margin and preference accuracy. Default selection minimizes -`validation_dpo_loss`. Reference scores are recomputed, avoiding stale caches. -A two-pass backward calculation uses the exact scalar margin derivative and -recomputes one chunk's differentiable likelihood at a time, bounding activation -memory without changing the summed sequence objective. - -DPO and GRPO each declare `reference: {source: phase_start}` (default) or -`{source: artifact, path: models/reference/actor.pt}`. A phase-start reference is -an independent weight snapshot before any phase update. Artifact references are -validated during preflight and reuse cached CPU weights; their files are checked -again before phase startup. References stay in evaluation mode with -gradients disabled. Full representations and actual weight hashes must match. -Embedded checkpoint reference weights survive removal of the original file; -a still-present artifact with changed weights is rejected on resume. - -## Online adapters - -PPO, GRPO, evaluation and policy rollout share the version-3 `OnlineEnvironmentAdapter` contract. Configure -`environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. -Factories must create fresh independent adapters for evaluation. An adapter -explicitly declares: - -```python -from sequifier.rl_api import EnvironmentCapabilities, OnlineResetResult, OnlineStepResult - -class Adapter: - protocol_version = 3 - implementation_id = "my-simulator.v1" - capabilities = EnvironmentCapabilities( - exact_snapshot=True, - group_start="scenario", - reproducible_group_start=True, - group_start_randomness="independent", - ) - - def reset(self, seed: int) -> OnlineResetResult: ... - def step(self, action: dict) -> OnlineStepResult: ... - def close(self) -> None: ... - def snapshot_state(self) -> dict: ... - def restore_state(self, state: dict) -> None: ... - def create_group_start(self, seed: int) -> object: ... - def reset_group(self, token: object, rollout_seed: int) -> OnlineResetResult: ... -``` - -Reset returns the current observation, static inputs and units. Step returns -immediate scalar reward, actual successor observation, boundary flags, units -and optional boolean success. Only true termination may omit the successor. -`close()` runs in `finally`. `implementation_id` must match the authored config -and change whenever simulator/reset/RNG semantics change. - -GRPO requires `group_start: snapshot | scenario`, explicit methods, and an opaque -finite JSON-serializable token. Resetting a sample must reproduce the token's -initial condition while independently seeding future environment randomness. -`group_start_randomness` documents whether the token captures initial simulator -randomness or only the scenario. Group/sample indices and run/phase seeds derive -separate environment and action streams. Repeating a reset seed alone does not -establish this capability. Exact GRPO continuation additionally requires the -adapter's documented `reproducible_group_start` guarantee. The bundled -`CategoricalControlAdapter` demonstrates a deterministic scenario token. - -Policy inference from supplied observation rows additionally requires -`EnvironmentCapabilities(seeded_reset=True)` and -`reset_from_observation(initial: OnlineResetResult, seed: int) -> OnlineResetResult`. -It must initialize the simulator without executing an action and return exactly -the supplied observation, static inputs and units. This is distinct from the -ordinary `reset(seed)` used during training. The bundled control adapter -implements both. Forecast rollout bindings retain their separate version-1 -contract. - -## Online categorical PPO - -`ppo` requires `environment`, `evaluation`, `total_env_steps` and `rollout_length`. -Normalization is inherited from the run representation; a phase assertion is -optional. One persistent training environment collects compact sealed horizons. -True successors bootstrap fixed-length horizon/truncation boundaries; termination -disables bootstrap. FP32 GAE feeds the clipped policy objective, entropy bonus, -and separate value regression, with optional value clipping. Actor and value -optimizers share no parameters. Old collection scores are checked before update, -and collection generations are independent of phase/run indices. - -Defaults: `optimization_epochs: 4`, `minibatch_size: 64`, `accumulation_steps: 1`, -`gamma: 0.99`, `gae_lambda: 0.95`, `clip_epsilon: 0.2`, -`value_coefficient: 0.5`, `entropy_coefficient: 0.01`, -`normalize_advantages: true`, `reward_scale: 1`. Optional positive `target_kl` -stops optimization against the old collection policy; `value_clip` is optional. -The `optimization.actor` and `optimization.value` roles have independent -optimizers/schedulers and the shared learning-rate and clipping defaults. -`allow_partial_rollout: true` executes exactly the remaining action budget; -false requires divisibility. `max_buffer_bytes` defaults to 536870912. -`archive_directory` optionally writes separate versioned collection records; -`archive_scores` defaults to false. Archived records are not training input. - -`evaluation_interval` and `checkpoint_interval` default to one complete -collection/optimization iteration. Evaluation uses a separate adapter, -independent seeds and a whole-evaluation POSIX/main-thread deadline. It evaluates -the live actor synchronously and restores every module's training mode and the -training RNG state afterward. It requires -`environment`, `seeds`, `max_episode_steps`, and `timeout_seconds`; -`action_selection: deterministic | stochastic` defaults to deterministic. Default -`evaluation.monitor: {metric: mean_completed_return, mode: max}` selects true completed returns. -`mean_completed_length` and `success_rate` are also supported. Censored episodes -and unavailable metrics are explicit; best selection fails when no finite -monitored value is available. The final iteration always evaluates. - -## GRPO groups and objective - -`grpo` requires `environment`, `evaluation`, `total_groups` and `max_steps`. -Defaults: `group_size: 4` (at least 2), `groups_per_update: 1`, -`optimization_epochs: 4`, `advantage_epsilon: 1e-6`, `clip_epsilon: 0.2`, -`kl_coefficient: 0.04`, `reward_scale: 1`, `scoring_chunk_size: 64`. -`old_policy_target_kl` optionally limits old-policy drift independently of the -reference penalty. No value network, value optimizer, or GAE is allocated. - -Each group contains independent sequential trajectories from the same opaque -start token under one frozen old actor generation. All groups in an iteration -are sealed before any update. Scores sum observed rewards times the fixed -reward scale, through termination, truncation or the explicit `max_steps` -finite-horizon cutoff. Cutoffs do not invent termination or bootstrapped rewards. -Incomplete/failed/non-finite groups are discarded and never optimized. Errors -report issued environment actions, including actions discarded with the group; -there is no retry. Optional `max_env_steps` is checked before collection using -the conservative `group_size * max_steps` bound for every requested group. -Memory is bounded by `max_buffer_bytes` (default 536870912), checked before -collection and as frames are appended. Optional archives are separate from the -training buffer. - -The precise first variant is: - -```text -A_i = (R_i - mean(R)) / (std_population(R) + advantage_epsilon) -r_it = exp(log pi_actor(a_it|h_it) - log pi_old(a_it|h_it)) -d_it = log pi_ref(a_it|h_it) - log pi_actor(a_it|h_it) -K_it = exp(d_it) - d_it - 1 -loss = mean_trajectories(mean_valid_decisions( - -min(r_it*A_i, clamp(r_it,1-epsilon,1+epsilon)*A_i) + kl_coefficient*K_it)) -``` - -Each trajectory has equal weight regardless of length; group sizes are fixed. -Minibatches contain whole trajectories; accumulation uses actual trajectory -counts. FP32 ratios/objectives/gradients are checked before applying an update. -Equal group scores give zero advantages; the reference penalty can still update -the actor. Reference and old-policy scores stay detached across optimization -epochs. The reported `sampled_reference_penalty` is a sampled estimator under -old-policy samples, not exact current-policy KL. - -Evaluation defaults to `evaluation.monitor: {metric: score_mean, mode: max}` and scores every -independent evaluation episode through its real boundary or declared horizon. -The evaluation score uses the same fixed reward scale. Evaluation/checkpoint -intervals count completed updates, with an explicit final evaluation. Reports -include score spread, zero-variance groups, lengths, cutoff reasons, start -tokens, sampled penalty, clip fraction, generations, optimizer steps and issued -environment actions. Sealed buffers are consumed once; stale generations fail. - -## Checkpoints, resume, and artifacts - -Named sources share indexed episode storage across phases and partitions. Table -sources are parsed once into temporary sorted Parquet storage with direct episode -row-group access. The episode cache stays bounded; partition views retain metadata -and validated transition indices. Imitation shuffles episodes and then decisions -within each episode to preserve storage locality. - -Inputs are resolved once into a run-owned catalog. Source resolution returns -verified file checksums, including the checksums checked against sequence manifests; -preparation reuses those resolved sources without resolving or hashing them again. -Every phase boundary explicitly rehashes all input files, manifests, metadata, -static data and preference files to detect modifications. Output and archive path -conflicts are checked separately before output mutation and at phase boundaries. -Prepared data supplies its fitting view and frozen-representation validation; -algorithm-specific transition checks remain with the IQL dataset. - -The policy-training checkpoint is `sequifier_policy_run`, format version 6. -Version-5 checkpoints are read and upgraded on the next save. -Completed results determine the phase boundary; optional version-2 phase state -records the latest committed transaction. It contains the resolved plan, frozen -representation, source/partition and adapter identities, run RNG, completed -phase records pairing each result with its actor checksum and result fingerprint. -Actor/report paths come from the result provenance. Live phase state contains -role-keyed networks and optimizers/schedulers/step counts, sampler, selection, metrics and -algorithm-specific progress. DPO/GRPO references are embedded. Earlier selected -actors remain in their phase artifacts. Checkpoints older than version 5 are -not supported. - -Writes use temporary files, flush them, and atomically replace their destinations. -At a handoff, the selected `actor.pt` is published with its result metadata, then -the run checkpoint commits its checksum and result. The checkpoint is the -authoritative phase boundary. An interrupted, uncommitted export is replaced on -retry; a committed handoff advances exactly once. Completed phases never rerun -or reselect their actor. - -Retain phase actor artifacts alongside the checkpoint. Resume checks their -checksums and result identities before output mutation, without loading or -rewriting earlier models. Phase `report.json` files are derived from committed -results and regenerated on resume; they are not recovery dependencies. No -separate phase manifest is needed. Representation and data/adapter/reference -mismatches still fail before output mutation. - -`resume.mode: exact` requires validated PPO simulator/RNG snapshots or GRPO's -reproducible group-start capability and matching implementation identity. PPO -checks the restored snapshot exactly; GRPO derives subsequent group tokens and -independent streams from persisted counters. `reset_environments` explicitly -resets PPO environments or restarts GRPO with a new derived stream, preserving -completed updates and reporting a trajectory discontinuity. Neither mode reuses -partial groups or old on-policy samples. Offline continuation restores its -sampler/order and optimizer transaction exactly. - -Portable artifacts are ordinary actor-only `sequifier_model` bundles. They -carry normalization, codecs, architecture, policy and unit provenance, with no -critics, references or trainer dependency. The final artifact comes from the -final phase's selected actor. The run report records order/config fingerprints, -source/reference provenance, selections, clocks and continuation modes. - -## Python and inference - -The Python API starts from the same authored schema as YAML: - -```python -from sequifier.config.train_config import SequifierConfig -from sequifier.config.training_resolution import resolve_training_run -from sequifier.train import run_training -import yaml - -with open("configs/train-policy.yaml") as stream: - authored = SequifierConfig.model_validate(yaml.safe_load(stream)) -resolved = resolve_training_run(authored) -result = run_training(resolved) -``` - -`run_training` selects the supervised engine, native policy runner, or mixed-plan -coordinator from the resolved phase algorithms. `load_train_config(path, {}, False)` -combines YAML loading and resolution. `run_policy_training` takes an internal -resolved policy run, rather than a separate authored configuration. Direct Python -inference can validate `PolicyInferenceConfig`, resolve it with -`policy_inference_config(request.model_dump(exclude_unset=True))`, and call -`sequifier.rl.inference.infer_policy`; it uses the same schema as the YAML loader. -`rl_api` exports phase schemas, policy bindings, action declarations, environment -capabilities, sequence records and `load_policy`. - -Use `model_type: generative` with a selected portable PT artifact for raw CSV/Parquet -observation rows. `enforce_determinism` requests deterministic execution; -`action_selection: deterministic | stochastic` selects the policy mode or sampling. -The seed is independent of both. Seeded sampling can reproduce a sequence under -the same supported runtime, device and input order. Online evaluation uses -`action_selection`, defaulting to deterministic, with the run's execution settings. -Evaluation reports record that choice. - -Policy `rollout.environment` declares the version-3 environment, `max_steps` -sets the collection budget, and `seed_row_index` selects a zero-based raw input row. -`trajectory_path` is required; `max_episode_steps` is optional. Policy rollout -rejects forecast bindings and generation settings. It requires seeded observation -reset; successors come from actual environment transitions. True termination may -omit a successor; truncation and collection cutoffs retain it. Forecast rollout -instead selects a prepared model window and follows the separate version-1 -protocol. See [inference](configs/infer.md), [rollout](rollout.md), and the -[Python PPO workflow](example_inputs/ppo_toy_control.py). - -CPU tests cover the reference execution contract. CUDA and MPS reproducibility -also depends on backend/operators and matching device/RNG topology. - - -# Supervised models and policy learning - -Policy learning uses the same model interfaces, input tensors, decoder branches, -categorical codecs, and portable artifacts as supervised learning. A policy -binding identifies the output heads that form an environment action. The -binding does not automatically add inputs or replace a pretrained decoder. - -## Post-train a supervised artifact - -An ordinary supervised PT model, or a supported supervised run checkpoint, can -initialize a policy run. Its interface does not need existing policy metadata. - -```yaml -project_root: . -model_name: posttrained -initialize_from: models/supervised/model.pt -policy_binding: - actions: - mode: {type: categorical, values: ["off", "on"]} - price: {type: real, scale: 0.5} -dataset_training: - demonstrations: - parts: - recorded: {type: sequence, paths: [data/demonstrations.parquet], format: table} -training_plan: - phases: - - name: imitate - algorithm: policy_imitation - sources: [{source: demonstrations.recorded}] - epochs: 5 - batch_size: 64 - optimization: - actor: {learning_rate: 0.0003} -``` - -Use the actual interface and target names from the source artifact. Quote YAML -labels such as `"off"` and `"on"` if the YAML parser interprets them as booleans. -Omitting categorical `values` selects the artifact's entire decoder vocabulary, -including any executable special tokens. Explicit values select a subset and -are reported as an adaptation when that changes the original support. - -The artifact provides the architecture, context length, target positions, -normalization, and class maps. A separately supplied model must agree with it. -Original weights load strictly. Only explicitly introduced exploration parameters -are initialized. The run report records the transfer and its new/reused parameters. - -## Real and categorical heads - -Every real action head requires an explicit positive `scale`, measured in the -head's physical units. Its pretrained scalar output is the Gaussian location; -categorical heads retain their logits and persisted decoder-ID semantics. - -| Setting | Meaning | -| --- | --- | -| `scale` | Fixed Gaussian standard deviation by default. | -| `learn_scale: true` | Learn a global log scale, bounded by `min_scale` and `max_scale`. | -| `lower` / `upper` | Transform the Gaussian onto a positive, negative, or finite open interval using a softplus or logistic bijection. | -| `atoms: {0.0: 0.05}` | Assign fixed probability mass to an exact real value, alongside the continuous component. Probabilities must sum to less than one. | -| `active_when: {mode: [on]}` | Activate a field only for these values of an earlier action field. | -| `inactive_value: 0.0` | Deterministic value used when a field is inactive. | - -Bounds change the deterministic prediction function and are reported as an -adaptation. Samples are never clipped and then scored as unclipped Gaussians. -Collection retains transform latents for stable replay near numerical boundaries. -Externally supplied exact endpoints require a declared point mass. Inactive -fields are point masses with zero log-probability contribution, rather than -Gaussians with zero variance. - -The existing decoder determines factorization. Ordinary branches predict -independent fields; an autoregressive branch samples in its existing target -order, embedding each sampled prefix value. Recorded actions are scored with -their recorded prefix. Selecting a later autoregressive target requires selecting -all its predecessors. Composite branches and tied weights are retained. Hashed -categorical outputs are normalized over canonical executable candidates. - -Likelihoods sum over active action fields once per decision. Supervised loss -weights do not alter PPO ratios or DPO likelihoods. Independent categorical and -unbounded Gaussian entropy is exact; dependent/transformed entropy uses a -score-function estimator that includes the sampled prefix distribution. These -estimates can have higher variance. The GRPO sampled reference penalty retains -its existing interpretation. - -All five algorithms consume this joint action contract: imitation, IQL, PPO, -DPO, and GRPO. IQL projects real action fields and embeds categorical fields -before evaluating its scalar twin Q-functions. - -## Inputs, history, and output positions - -The full persisted execution schema describes scalar, static, depth, image, and -mask tensors. Online adapters can return nested JSON values, or array/tensor -payloads that can be converted to portable values. Supply images with the -model's existing crop, channel, frame-capacity, and byte conventions. Depth -arrays and their named masks must match the source execution schema. - -Prepared sources reuse depth masks and the existing image store. Table sources -can provide nested Parquet payloads. Use `columns.actions` to map logical action -names to table columns; `columns.previous_actions` maps preceding recorded actions. -Exported sequence shards preserve named action records and optional real latents. - -No previous-action input is inserted by default. To feed an existing model input -from a lagged action, declare, for example: - -```yaml -policy_binding: - previous_action_inputs: {last_quantity: quantity} - initial_inputs: {last_quantity: 0.0} - actions: - quantity: {type: real, scale: 0.5} -``` - -`initial_inputs` supplies the first decision's lagged value. The existing input's -codec and normalization are retained. Temporal padding, action activity, and -missing training labels are distinct. Complete relevant action tuples are -required for likelihood training; missing predecessors are not silently masked. - -`decision_position` defaults to `-1`. Other negative positions select existing -decoder positions while preserving support and prediction geometry. A model -with several prediction positions does not automatically execute an action block. -Only the selected current decision is executed by these runners. - -Forward attention is persisted separately from the learning objective. A -bidirectional encoder can consume an entirely observed history. Each historical -decision is reconstructed at its actual availability boundary, so later -observations cannot leak into its likelihood. An input unavailable at action time -must be supplied through an explicit task adaptation. - -Return conditioning is opt-in for attached models with a `__policy_return` -input. Offline conditioning needs a known reward tail. Online conditioning needs -`initial_desired_return`; subsequent budgets follow the configured return update. - -## Mixed training plans - -Supervised and policy phases use the same `training_plan.phases` structure. -See the full [supervised-to-policy template](../src/sequifier/templates/train-supervised-to-policy.yaml). -Supervised prepared parts and policy sequence parts both belong to -`dataset_training`; each phase references the appropriate parts. No additional -run discriminator is needed. `global_training` owns execution and model/data -contracts. Each phase owns batching and `optimization.`: - -```yaml -training_plan: - phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential - sources: [{source: observations.train}] - batch_size: 64 - accumulation_steps: 1 - optimization: - model: {optimizer: {name: Adam}, learning_rate: 0.001} - - name: imitation - algorithm: policy_imitation - epochs: 5 - sources: [{source: demonstrations.recorded}] - batch_size: 64 - accumulation_steps: 1 - optimization: - actor: {optimizer: {name: Adam}, learning_rate: 0.0003} -``` - -The coordinator preflights model, source and environment contracts and commits -incoming artifacts before native phase execution. Phase-local checkpoints recover -active work; atomic receipts recover completed handoffs. Policy `model_interface` -selects an interface when source references are insufficient to identify it. -All interfaces remain in the model artifact; a shared backbone update can change -other interfaces' predictions. - -`optimizer_state` defaults to `reset`. Mixed-plan continuation is rejected because -artifact handoffs do not preserve optimization state. Native supervised-only and -compatible policy-only plans support explicit continuation under the -[training transition rules](configs/train.md#optimization-across-phases). -`handoff: last | best` selects phase weights; continuation requires the predecessor's -`last` handoff. Mixed plans execute locally. Exact online recovery requires the -adapter's snapshot capabilities. - -The Python API exposes `SequifierConfig`, `ResolvedSequifierConfig`, and -`run_learning_plan` from `sequifier.training_api`. The common entry point -`sequifier.train.run_training` dispatches a resolved plan to its appropriate -runner. `load_policy(..., binding=...)` attaches a task directly to a supervised -artifact; policy bindings never author a second model architecture. - -## Distribution-aware supervised losses and inference - -Existing MSE, MAE, cross-entropy, and hashed-component losses retain their -semantics. An interface may declare `prediction_distributions` for real targets -and select `GaussianNLLLoss` in `dataset_training.criterion`. This uses the same -real distribution implementation as policy likelihoods and stores learned scales -in the model artifact. - -Artifacts retain ordinary point-prediction execution. For an artifact containing -policy metadata, set `execution_mode: prediction` to select ordinary forecast -inference, or `execution_mode: policy` for action sampling. Joint policy -inference writes every executed field; `output_probabilities: true` also writes -the joint action log probability. Deterministic generation uses sequential -locations/argmax choices; it does not claim a global mode of an autoregressive -joint distribution. - -New model artifacts use format version 3; versions 1 and 2 remain readable. -Joint trajectories and PPO collection records use version 2. Existing -single-categorical configuration and inference APIs remain available as -compatibility surfaces. ONNX remains a prediction export, not a general training -checkpoint or an automatically exported stochastic sampler. - -This implementation has not been exercised with tests or training experiments -yet. Probability, architecture-parity, learning, and interruption acceptance -checks remain to be performed before release. - - -# Local numeric forecast rollouts - -For trajectory-backed categorical policy training and return-conditioned -execution through the online environment protocol, see [policy learning](policy-learning.md). - -The first rollout slice collects one seed-initialized episode or episode fragment -on CPU. It supports real scalar features and real static conditioning, with -one-step `causal` or `final_value` forecast models (`target_offset: 1`). Model -predictions and a persistent local adapter jointly produce the next input row. -There is no learning algorithm in this path. - -## Inference configuration - -Add this to an otherwise valid inference configuration for a model with inputs -`x`, `d` and target `x`: - -```yaml -device: cpu -model_type: generative -rollout: - max_steps: 3 - seed_window_index: 0 - trajectory_path: outputs/trajectory.json - environment: - factory: your_application.adapters:make_environment - config: {} # keyword arguments accepted by your factory - bindings: - - {input: x, source: prediction, key: x} - - {input: d, source: environment, key: d} -``` - -Replace the adapter factory with your application's importable `module:factory`. -It must implement the protocol below and provide the bound environment fields. - -Run the existing `sequifier infer --config-path configs/infer.yaml` command. -PT and existing schema-bearing ONNX models use the same coordinator and their -existing inference backends. This feature adds no export format. - -`seed_window_index` is required and selects one prepared model window, using zero-based -dataset/chunk order. Unlike legacy autoregression, selection happens before the -first-subsequence-per-sequence filter. It uses the usual inference window -selection rules, including `window_stride`. An out-of-range index fails. The -seed must have at least one valid position, end in a valid position, and contain -only left padding before its valid observations. Every feature and the attention -mask must have the same context length. Static values come from the existing -processed static feature table and stay constant. - -Each changing input must have exactly one binding. `prediction` keys name model -targets; `environment` keys name adapter outputs. Static inputs cannot have -bindings. Missing producers, duplicate input bindings, unknown predictions, and -extra or missing adapter fields fail clearly. Outputs that are only consumed by -the adapter do not need a feedback binding. - -The seed dataset uses the existing **preprocessed training representation**. -The adapter and stored trajectory use **original units**. The inference wrapper -decodes the seed and forecasts and encodes every successor using the fixed -training `mean`, `std`, and `normalize_real_columns` metadata. It uses the same -`std + 1e-9` transform as preprocessing, without fitting statistics on rollout -values. Float overflow, non-finite values, arrays in place of scalars, and -incompatible model inputs are rejected. Model padding stays zero with a false -attention mask. Features and attention masks slide together, appending a valid -successor position even for the last executed prediction. - -With `rollout` present, inference writes the trajectory to `trajectory_path` -(relative to `project_root` unless absolute). It replaces that file atomically -after successful collection and validation. This path collects one selected -seed with one model; it does not produce legacy flat prediction/probability -files. Omitting `rollout` preserves existing inference output and configuration -rules. - -## Adapter protocol - -Adapters declare `protocol_version = 1` and implement: - -```python -from sequifier.rollout_api import ResetRequest, StepRequest, StepResult - -def reset(self, request: ResetRequest) -> None: ... -def step(self, request: StepRequest) -> StepResult: ... -def close(self) -> None: ... -``` - -A `module:factory` receives `adapter.config` as keyword arguments and is called -once per collection. The adapter stays alive for all steps and closes on success -or failure, including reset, prediction, and result-validation failures. A -factory that fails during construction owns cleanup of any partial resources. -Python callers can pass a supplied adapter instance to `collect_rollout`. - -`reset` receives the episode ID, complete seed history, validity mask, static -inputs, and random seed. It initializes the environment from this history and -must not execute a transition or replace the seed. `step` receives the current -observation, all named forecasts, static inputs, and zero-based step ID. It -returns external fields, one finite scalar reward, and separate Python boolean -`terminated` and `truncated` flags. Reset and step requests contain independent -copies of coordinator data. Adapters must return the final pre-reset successor -and must not autoreset. - -Every prediction executes exactly once before collection can end. A boundary -stops further prediction and stepping. `generation_steps` is a collection -budget: exhausting it leaves `collection_cutoff: true` when the episode is -unfinished, without fabricating termination or truncation. Optional -`rollout.max_episode_steps` is an episode time limit; reaching it records -`truncated: true`. An adapter can also report its own truncation. Both boundary -flags may be true together; termination takes precedence for bootstrap. - -## Trajectory format and Python API - -The version-1 JSON artifact contains the domain seed history and mask, static -values, input bindings, prediction names, episode ID, and ordered transitions. -Each transition records the observation, predictions (the executed forecast -actions), external fields, reward, successor, step ID, and separate boundary -flags. `bootstrap_allowed` is false on termination and true otherwise, including -time-limit truncation with its recorded final successor. The artifact also -contains `total_return`, `collection_cutoff`, and inference provenance, including -the exact preprocessing statistics and seed origin. - -Seed history plus successors reconstruct every later history and attention mask; -they are not duplicated in each record. Artifacts record one episode fragment -and do not promise simulator snapshots or exact resumability. - -```python -from sequifier.rollout_api import load_trajectory, validate_trajectory - -trajectory = load_trajectory("outputs/trajectory.json") -validated = validate_trajectory(trajectory) -``` - -Saving and loading validate format/version, finite scalar data, seed shape and -mask, producer coverage, successor bindings, observation continuity, contiguous -episode/step IDs, total return, bootstrap semantics, and absence of transitions -after an episode boundary. The same validation is available for in-memory data. - -For direct Python collection, a predictor is a callable receiving a copied -`SeedHistory` in original units and returning a dictionary of scalar forecasts. -The collector owns adapter closure. `history_validator`, if provided, validates -the initial history and every successor (including a terminal successor) for -the predictor's representation; the inference integration uses the fixed codec -and existing `ExecutionSchema` here. - -```python -from sequifier.rollout_api import collect_rollout, save_trajectory - -# predictor, adapter, seed and config are provided by your application. -trajectory = collect_rollout( - predictor, adapter, seed, config, prediction_columns=["x"], max_steps=1000, -) -save_trajectory(trajectory, config.trajectory_path) -``` - -Multimodal and categorical observations, reset-only initialization, vectorized -environments, remote transport, learning algorithms, and new export support are -outside this slice. - -## Policy environments and reproducibility - -Policy training, online evaluation and policy rollout use `environment` with -`factory`, `config`, `implementation_id`, and `units` (empty means unitless). -This is a version-3 online environment, distinct from the forecast version-1 -protocol above. Policy rollout requires seeded observation reset; training -requires ordinary reset. Exact resume requires snapshot/restore capabilities. -GRPO also requires reproducible group starts. See the -[policy protocol reference](policy-learning.md#online-adapters). - -The inference route selects the rollout schema. Policy rejects forecast -`bindings`; forecast rejects policy implementation/units declarations. Neither -schema accepts a rollout discriminator. Both use `max_steps`, `trajectory_path`, -and optional `max_episode_steps`. Forecast `seed_window_index` indexes -prepared windows; policy `seed_row_index` indexes raw input rows before sequence -history preparation. Both indices are zero-based and out-of-range values fail -before environment creation. - -Use `enforce_determinism` for execution and policy `action_selection` for mode -versus sampling. A fixed seed can make stochastic actions reproducible under -the same supported runtime and ordering. - - -Both routes accept legacy `rollout.seed_index` as an alias for their respective -index field and reject both names in one declaration. Serialized trajectories -preserve the public metadata key `seed_index` and its original route-specific -units; this configuration rename does not change the output format. diff --git a/src/sequifier/docs.py b/src/sequifier/docs.py index 562bc487..0c5f045f 100644 --- a/src/sequifier/docs.py +++ b/src/sequifier/docs.py @@ -1,19 +1,42 @@ """Read the documentation bundled with the installed sequifier package.""" from importlib.resources import files +from pathlib import Path + +DOCUMENTATION_SOURCES = ( + "README.md", + "documentation/configs/preprocess.md", + "documentation/configs/train.md", + "documentation/configs/infer.md", + "documentation/commands/visualize-training.md", + "documentation/configs/hyperparameter-search.md", + "documentation/training/multi-gpu-training.md", + "documentation/policy-learning.md", + "documentation/supervised-policy-learning.md", + "documentation/rollout.md", +) GUIDE_HEADINGS = { "preprocess": "# Preprocess Command Guide", "train": "# Train Command Guide", "infer": "# Infer Command Guide", - "policy": "# Categorical policy learning", + "policy": "# Policy learning", } +def source_documentation(root: Path) -> str: + return "\n\n".join( + (root / name).read_text(encoding="utf-8") for name in DOCUMENTATION_SOURCES + ) + + def docs(topic: str | None = None) -> None: """Print all documentation or the guide for a single command.""" + resource = files("sequifier").joinpath("consolidated-docs.md") content = ( - files("sequifier").joinpath("consolidated-docs.md").read_text(encoding="utf-8") + resource.read_text(encoding="utf-8") + if resource.is_file() + else source_documentation(Path(__file__).resolve().parents[2]) ) if topic is not None: diff --git a/src/sequifier/io/actions.py b/src/sequifier/io/actions.py index 2601001b..b07b8594 100644 --- a/src/sequifier/io/actions.py +++ b/src/sequifier/io/actions.py @@ -7,18 +7,34 @@ from sequifier.io.values import domain_scalar, numeric_scalar -def action_latent_key(name, config): - key = "__policy_latent:" + name +def _metadata_key(name, config, prefix): + key = prefix + name while key in config.actions: - key = "__policy_latent:" + key + key = prefix + key return key +def action_latent_key(name, config): + return _metadata_key(name, config, "__policy_latent:") + + def action_component_key(name, config): - key = "__policy_component:" + name - while key in config.actions: - key = "__policy_component:" + key - return key + return _metadata_key(name, config, "__policy_component:") + + +def action_record_fields(config): + return set(config.actions) | { + key(name, config) + for name, field in config.actions.items() + if field.type == "real" + for key in (action_latent_key, action_component_key) + } + + +def inactive_action(field): + if field.type == "categorical": + return field.values.index(field.inactive_value) if field.active_when else 0 + return field.inactive_value def encode_action(value, config): @@ -64,18 +80,25 @@ def action_batch(values, config, *, device=None, encoded=True): values = list(values) if not encoded: values = [encode_action(value, config) for value in values] - result = { - name: torch.tensor( - [value[name] for value in values], - dtype=torch.long - if field.type == "categorical" - else ( - torch.float32 - if device is not None and torch.device(device).type == "mps" - else torch.float64 - ), + real_dtype = ( + torch.float32 + if device is not None and torch.device(device).type == "mps" + else torch.float64 + ) + + def tensor(key, dtype, *, default=None, required=False): + return torch.tensor( + [value[key] if required else value.get(key, default) for value in values], + dtype=dtype, device=device, ) + + result = { + name: tensor( + name, + torch.long if field.type == "categorical" else real_dtype, + required=True, + ) for name, field in config.actions.items() } latent_keys = { @@ -84,21 +107,11 @@ def action_batch(values, config, *, device=None, encoded=True): if field.type == "real" } for key in latent_keys: - result[key] = torch.tensor( - [value.get(key, float("nan")) for value in values], - dtype=torch.float32 - if device is not None and torch.device(device).type == "mps" - else torch.float64, - device=device, - ) + result[key] = tensor(key, real_dtype, default=float("nan")) for name, field in config.actions.items(): if field.type == "real": key = action_component_key(name, config) - result[key] = torch.tensor( - [value.get(key, -2) for value in values], - dtype=torch.long, - device=device, - ) + result[key] = tensor(key, torch.long, default=-2) return result diff --git a/src/sequifier/model/execution.py b/src/sequifier/model/execution.py new file mode 100644 index 00000000..d91a2ff9 --- /dev/null +++ b/src/sequifier/model/execution.py @@ -0,0 +1,86 @@ +"""Model execution facts, independent of dataset ownership and learning phases.""" + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ModelExecution: + model: Any + interfaces: dict + global_training: Any + + @classmethod + def from_config(cls, config): + interfaces = {} + for dataset in config.dataset_training.values(): + interfaces.setdefault(dataset.model_interface, dataset.interface) + return cls(config.model, interfaces, config.global_training) + + def dataset(self, name): + """Adapt an interface for consumers of the historical training contract.""" + from sequifier.config.train_config import ( + DatasetFreezingSpecModel, + ResolvedDatasetTrainingSpec, + ) + + interface = self.interfaces[name] + return ResolvedDatasetTrainingSpec( + name=name, + model_interface=name, + interface=interface, + parts={}, + criterion={ + target: "CrossEntropyLoss" if kind == "categorical" else "MSELoss" + for target, kind in interface.target_column_types.items() + }, + class_share_log_columns=[], + freeze=DatasetFreezingSpecModel(), + ) + + def training_config(self, *, device="cpu", **options): + """Preserve the public resolved-config adapter and its serialized shape.""" + from sequifier.config.train_config import ResolvedSequifierConfig + + return ResolvedSequifierConfig( + global_training=self.global_training, + model=self.model, + dataset_training={name: self.dataset(name) for name in self.interfaces}, + **{ + "project_root": ".", + "model_name": "loaded-model", + "seed": 0, + "device": device, + "training_plan": [], + "evaluation_sources": [], + "evaluation_monitor": None, + "export_generative_model": True, + "export_embedding_model": False, + "export_onnx": False, + "export_pt": False, + "embedding_layer_names": ["backbone.final_norm"], + **options, + }, + ) + + def retain_interfaces(self, config): + """Keep inactive interfaces when selecting supervised datasets.""" + existing = { + dataset.model_interface for dataset in config.dataset_training.values() + } + for name in self.model.interfaces: + if name in existing: + continue + key = name + while key in config.dataset_training: + key += "_unused" + selected = type(self)( + self.model, + {name: self.interfaces[name]}, + config.global_training, + ) + config.dataset_training[key] = ( + selected.training_config() + .dataset_training[name] + .model_copy(update={"name": key}) + ) diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index 1d5938c4..e22c29ba 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -67,12 +67,10 @@ class BuiltModel: def validate_hash_contracts(network: ComposableTransformerNetwork, config: Any) -> None: """Check persisted codec buffers after weights are restored.""" + from sequifier.model.execution import ModelExecution from sequifier.model.ingestions import EmbeddingFeatureIngestion, MultiHashEmbedding - interfaces = { - dataset.model_interface: dataset.interface - for dataset in config.dataset_training.values() - } + interfaces = ModelExecution.from_config(config).interfaces for name, interface in interfaces.items(): route = network.resolve_interface(name) for column, expected in interface.categorical_hash_contracts.items(): @@ -262,10 +260,9 @@ def _build_composable_network( logger: Any, ) -> BuiltModel: from sequifier.config.train_config import interface_build_view + from sequifier.model.execution import ModelExecution - resolved_interfaces = {} - for dataset in config.dataset_training.values(): - resolved_interfaces.setdefault(dataset.model_interface, dataset.interface) + resolved_interfaces = ModelExecution.from_config(config).interfaces objectives = {} for name, interface in resolved_interfaces.items(): objective_view = interface_build_view(config, interface) diff --git a/src/sequifier/rl/batching.py b/src/sequifier/rl/batching.py new file mode 100644 index 00000000..432fff64 --- /dev/null +++ b/src/sequifier/rl/batching.py @@ -0,0 +1,24 @@ +"""Batch mechanics without algorithm-specific weighting or sampling policies.""" + +import torch + + +def batches(values, size): + for start in range(0, len(values), size): + yield values[start : start + size] + + +def shuffled_groups(size, group_size, generator): + return batches(torch.randperm(size, generator=generator).tolist(), group_size) + + +class MetricTotals(dict): + def __init__(self, *names): + super().__init__(dict.fromkeys(names, 0.0)) + + def add(self, values, divisor=1): + for name in self: + self[name] += float(values[name].detach().sum()) / divisor + + def means(self, count): + return {name: total / count for name, total in self.items()} diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 45a14e9c..02f68809 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -11,11 +11,12 @@ restore_model_artifact, save_model_artifact, ) +from sequifier.artifacts.phase import PhaseArtifacts from sequifier.rl.policy import Policy def phase_actor_path(directory, index, name): - return Path(directory) / f"{index:03d}-{name}" / "actor.pt" + return PhaseArtifacts.for_phase(directory, index, name).model def validate_policy_weights(policy: Policy) -> None: diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index 06d38e2e..998aa053 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -110,13 +110,10 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): final = update == phase.updates if session.should_evaluate(final): held_out = evaluate_dpo(actor, reference, data.validation, phase) - session.observe(held_out["dpo_loss"]) - session.metrics.append( - { - "transaction": update, - "training_dpo_loss": loss_total, - **{f"validation_{key}": value for key, value in held_out.items()}, - } + session.record_evaluation( + held_out["dpo_loss"], + training_dpo_loss=loss_total, + **{f"validation_{key}": value for key, value in held_out.items()}, ) session.checkpoint(final, batch_order=batches.state_dict()) return session.finish( diff --git a/src/sequifier/rl/grpo_update.py b/src/sequifier/rl/grpo_update.py index 90bbdedd..58484c2c 100644 --- a/src/sequifier/rl/grpo_update.py +++ b/src/sequifier/rl/grpo_update.py @@ -2,6 +2,8 @@ import torch +from sequifier.rl.batching import MetricTotals, shuffled_groups + def grpo_objective( log_probability, old_log_probability, reference_log_probability, advantage, phase @@ -47,22 +49,17 @@ def update_grpo(session, buffer): raise ValueError("GRPO collection generation differs from actor") validate_old_scores(actor, buffer, phase.scoring_chunk_size) trajectories = buffer.trajectories - totals = { - key: 0.0 - for key in ( - "loss", - "sampled_reference_penalty", - "old_policy_approximate_kl", - "clip_fraction", - ) - } + totals = MetricTotals( + "loss", + "sampled_reference_penalty", + "old_policy_approximate_kl", + "clip_fraction", + ) scored, stopped = 0, False size = phase.batch_size * phase.accumulation_steps epoch = -1 for epoch in range(phase.optimization_epochs): - order = torch.randperm(len(trajectories), generator=session.sampler).tolist() - for start in range(0, len(order), size): - indices = order[start : start + size] + for indices in shuffled_groups(len(trajectories), size, session.sampler): if phase.old_policy_target_kl is not None: old_kl = 0.0 with torch.no_grad(): @@ -100,10 +97,7 @@ def update_grpo(session, buffer): ( values[0].sum() / len(trajectory.actions) / len(indices) ).backward() - for key, value in zip(totals, values): - totals[key] += float(value.detach().sum()) / len( - trajectory.actions - ) + totals.add(dict(zip(totals, values)), len(trajectory.actions)) scored += 1 session.step() if stopped: @@ -113,7 +107,7 @@ def update_grpo(session, buffer): buffer.consume() actor.grpo_policy_version = generation + 1 return { - **{key: value / scored for key, value in totals.items()}, + **totals.means(scored), "optimizer_steps": session.steps["actor"], "optimization_epochs": epoch + 1, "old_policy_kl_stopped": stopped, diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 518d2b4d..245b06c2 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -142,21 +142,16 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): phase.gamma, actor.device, ) - session.observe(held_out["action_nll"]) - session.metrics.append( - { - "transaction": update, - **{ - f"{role}_loss": float(loss.detach()) - for role, loss in losses.items() - }, - **{f"validation_{key}": val for key, val in held_out.items()}, - "advantage_mean": float(advantage.mean()), - "weight_mean": float(weight.mean()), - "weight_cap_fraction": float( - (weight >= phase.max_weight).float().mean() - ), - } + session.record_evaluation( + held_out["action_nll"], + **{ + f"{role}_loss": float(loss.detach()) + for role, loss in losses.items() + }, + **{f"validation_{key}": val for key, val in held_out.items()}, + advantage_mean=float(advantage.mean()), + weight_mean=float(weight.mean()), + weight_cap_fraction=float((weight >= phase.max_weight).float().mean()), ) session.checkpoint(final) return session.finish( diff --git a/src/sequifier/rl/joint_distribution.py b/src/sequifier/rl/joint_distribution.py index 891685f8..7f20338a 100644 --- a/src/sequifier/rl/joint_distribution.py +++ b/src/sequifier/rl/joint_distribution.py @@ -6,7 +6,9 @@ action_component_key, action_latent_key, action_record, + action_record_fields, decode_action, + inactive_action, ) from sequifier.io.preprocessing import denormalize_values, normalize_values from sequifier.model.prediction_distribution import RealPredictionDistribution @@ -109,11 +111,7 @@ def select(self, deterministic, generator): distribution.mode() if deterministic else distribution.sample(generator) ) field = self.actor.config.actions[name] - inactive = ( - field.values.index(field.inactive_value) - if field.type == "categorical" and field.active_when - else (0 if field.type == "categorical" else field.inactive_value) - ) + inactive = inactive_action(field) active = self._active(name, actions) actions[name] = torch.where(active, value, value.new_tensor(inactive)) if isinstance(distribution, RealPredictionDistribution): @@ -140,15 +138,7 @@ def log_prob(self, actions): fields = {key for key in actions if key in self.actor.config.actions} if fields != set(self.actor.config.actions): raise ValueError("joint likelihood requires every action field") - allowed = set(self.actor.config.actions) - for name, field in self.actor.config.actions.items(): - if field.type == "real": - allowed.update( - ( - action_latent_key(name, self.actor.config), - action_component_key(name, self.actor.config), - ) - ) + allowed = action_record_fields(self.actor.config) if set(actions) - allowed: raise ValueError("joint action contains unknown fields") parameters = self._parameters(actions) @@ -165,11 +155,7 @@ def log_prob(self, actions): distribution = self._field(name, parameters) active = self._active(name, actions) field = self.actor.config.actions[name] - inactive = ( - field.values.index(field.inactive_value) - if field.type == "categorical" and field.active_when - else (0 if field.type == "categorical" else field.inactive_value) - ) + inactive = inactive_action(field) if (value[~active] != inactive).any(): raise ValueError(f"{name}: inactive action differs from its point mass") safe = torch.where(active, value, distribution.mode().detach()) @@ -201,7 +187,7 @@ def categorical_probabilities(self, actions): probabilities = distribution.distribution.probs if field.active_when: point = torch.zeros_like(probabilities) - point[:, field.values.index(field.inactive_value)] = 1 + point[:, inactive_action(field)] = 1 probabilities = torch.where( self._active(name, actions)[:, None], probabilities, point ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 8abe7995..48a9ab13 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -3,6 +3,7 @@ from dataclasses import replace from pathlib import Path +from sequifier.artifacts.phase import PhaseArtifacts from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism from sequifier.io.atomic import write_json_atomic @@ -205,8 +206,7 @@ def commit(state): "resolved_phase": phase.model_dump(mode="json"), "representation_fingerprint": representation.identity, "incoming_weight_hash": incoming_weight_hash, - "actor_path": str(actor_path), - "report_path": str(actor_path.with_name("report.json")), + **PhaseArtifacts(actor_path).describe(), }, ) recovery.continuation = getattr(actor, "_phase_continuation", None) diff --git a/src/sequifier/rl/ppo_update.py b/src/sequifier/rl/ppo_update.py index cebc73ff..51a99689 100644 --- a/src/sequifier/rl/ppo_update.py +++ b/src/sequifier/rl/ppo_update.py @@ -4,6 +4,8 @@ import torch +from sequifier.rl.batching import MetricTotals, batches, shuffled_groups + def ppo_objectives( log_probability, @@ -72,16 +74,9 @@ def update_ppo(session, value, buffer): actor.eval() # Keep dropout off while retaining autograd. value.eval() validate_collection_scores(actor, value, buffer, phase.minibatch_size) - totals = { - key: 0.0 - for key in ( - "policy_loss", - "value_loss", - "entropy", - "approximate_kl", - "clip_fraction", - ) - } + totals = MetricTotals( + "policy_loss", "value_loss", "entropy", "approximate_kl", "clip_fraction" + ) ratio_chunks = [] count = 0 norms = {"actor": [], "value": []} @@ -89,13 +84,8 @@ def update_ppo(session, value, buffer): group_size = phase.minibatch_size * phase.accumulation_steps epochs = 0 for epoch in range(phase.optimization_epochs): - order = torch.randperm(len(buffer), generator=session.sampler).tolist() - for start in range(0, len(order), group_size): - group = order[start : start + group_size] - minibatches = [ - group[offset : offset + phase.minibatch_size] - for offset in range(0, len(group), phase.minibatch_size) - ] + for group in shuffled_groups(len(buffer), group_size, session.sampler): + minibatches = list(batches(group, phase.minibatch_size)) if phase.target_kl is not None: kl_sum = 0.0 with torch.no_grad(): @@ -135,8 +125,7 @@ def update_ppo(session, value, buffer): raise ValueError("non-finite weighted PPO loss") actor_loss.backward() critic_loss.backward() - for key in totals: - totals[key] += float(metrics[key].detach().sum()) + totals.add(metrics) ratio_chunks.append(metrics["ratio"].detach().cpu()) count += len(indices) for role, norm in session.step().items(): @@ -160,7 +149,7 @@ def update_ppo(session, value, buffer): predictions.append(value(history).cpu()) residual_variance = (targets - torch.cat(predictions)).var(unbiased=False) target_variance = targets.var(unbiased=False) - result: dict[str, Any] = {key: total / count for key, total in totals.items()} + result: dict[str, Any] = totals.means(count) result.update( { "actor_optimizer_step": clocks["actor"], diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 7a6191bf..c503cad7 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -6,6 +6,7 @@ import torch +from sequifier.artifacts.phase import PhaseArtifacts from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save, write_json_atomic from sequifier.rl.checkpoint import phase_actor_path, save_policy @@ -252,7 +253,9 @@ def finish_phase(self, actor, result): save_policy(actor, path, metadata=result.as_dict()) weights = clone_weights(actor.network) self.completed.append( - CompletedPhase(result, file_checksum(path), fingerprint(result.as_dict())) + CompletedPhase( + result, **PhaseArtifacts(Path(path)).record(result.as_dict()) + ) ) self.incoming_actor = weights self.phase_state = None diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 9ca06c2d..9a46c494 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -343,6 +343,20 @@ def should_evaluate(self, final): def observe(self, metric): self.selection.observe(self.context.actor, metric, self.transaction) + def record_evaluation(self, metric, **record): + self.observe(metric) + self.metrics.append({"transaction": self.transaction, **record}) + + def optimization_state(self, *, include_steps=False): + return { + role: { + "optimizer": optimizer.state_dict(), + "scheduler": scheduler.state_dict(), + **({"steps": self.steps[role]} if include_steps else {}), + } + for role, (optimizer, scheduler) in self.optimization.items() + } + def should_checkpoint(self, final): return final or self.transaction % self.context.phase.checkpoint_interval == 0 @@ -357,14 +371,7 @@ def checkpoint(self, final, **extra): name: clone_weights(module) for name, module in self.networks.items() }, - "optimization": { - role: { - "optimizer": optimizer.state_dict(), - "scheduler": scheduler.state_dict(), - "steps": self.steps[role], - } - for role, (optimizer, scheduler) in self.optimization.items() - }, + "optimization": self.optimization_state(include_steps=True), "sampler": self.sampler.get_state(), "selection": self.selection.state_dict(), "transaction": self.transaction, @@ -390,13 +397,7 @@ def finish(self, clocks, provenance): name: clone_weights(module) for name, module in self.networks.items() }, - "optimization": { - role: { - "optimizer": optimizer.state_dict(), - "scheduler": scheduler.state_dict(), - } - for role, (optimizer, scheduler) in self.optimization.items() - }, + "optimization": self.optimization_state(), } elif hasattr(self.context.actor, "_phase_continuation"): del self.context.actor._phase_continuation diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index b8e54a59..f701082c 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -1,7 +1,5 @@ """Policy rollout through the same environment and causal history used in training.""" -import math -import warnings from copy import deepcopy from typing import Literal from uuid import uuid4 @@ -19,7 +17,8 @@ validate_reset, ) from sequifier.rl.policy import Policy -from sequifier.trajectory.schema import SeedHistory, Trajectory, Transition +from sequifier.trajectory.recording import TrajectoryRecorder, close_adapter +from sequifier.trajectory.schema import SeedHistory, Trajectory def collect_policy_rollout( @@ -90,15 +89,13 @@ def collect_policy_rollout( raise ValueError( "seeded reset must preserve the supplied observation and static inputs" ) - transitions, previous_actions, log_probabilities = [], [], [] + recorder = TrajectoryRecorder(episode_id, config.max_episode_steps) + previous_actions, log_probabilities = [], [] for step_id in range(max_steps): current = cursor.history.current decision = cursor.decide(deterministic=action_selection == "deterministic") result = cursor.step(decision) - truncated = result.truncated or ( - config.max_episode_steps is not None - and step_id + 1 >= config.max_episode_steps - ) + previous_actions.append(current.previous_action) log_probabilities.append(decision.log_probability) if spec.return_conditioned: @@ -110,46 +107,36 @@ def collect_policy_rollout( action_metadata = ( decision.action if isinstance(decision.action, dict) else {} ) - transitions.append( - Transition( - episode_id=episode_id, - step_id=step_id, - observation=dict(current.observation), - predictions=decision.domain_action, - action_latents={ - name: action_metadata[action_latent_key(name, spec)] - for name, field in spec.actions.items() - if field.type == "real" - }, - action_components={ - name: action_metadata[action_component_key(name, spec)] - for name, field in spec.actions.items() - if field.type == "real" - }, - external_inputs=successor or {}, - reward=result.reward, - successor_observation=successor, - terminated=result.terminated, - truncated=truncated, - bootstrap_allowed=not result.terminated, - ) + transition = recorder.append( + observation=dict(current.observation), + predictions=decision.domain_action, + action_latents={ + name: action_metadata[action_latent_key(name, spec)] + for name, field in spec.actions.items() + if field.type == "real" + }, + action_components={ + name: action_metadata[action_component_key(name, spec)] + for name, field in spec.actions.items() + if field.type == "real" + }, + external_inputs=successor or {}, + reward=result.reward, + successor_observation=successor, + terminated=result.terminated, + truncated=result.truncated, ) - if result.terminated or truncated: + if transition.terminated or transition.truncated: break - last = transitions[-1] - return Trajectory( + return recorder.finish( format="sequifier.policy-trajectory", version=2, - episode_id=episode_id, seed=seed, bindings=[ InputBinding(input=name, source="environment", key=name) for name in spec.observation_columns ], prediction_columns=spec.action_columns, - transitions=transitions, - collection_cutoff=not (last.terminated or last.truncated), - total_return=math.fsum(step.reward for step in transitions), metadata={ "policy_execution": { "config": spec.model_dump(mode="json"), @@ -166,9 +153,4 @@ def collect_policy_rollout( raise finally: policy.train(was_training) - try: - adapter.close() - except Exception as exc: - if not failed: - raise - warnings.warn(f"Adapter close also failed: {exc}", RuntimeWarning) + close_adapter(adapter, failed) diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index e4d847b6..0d831e5d 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -6,6 +6,7 @@ from sequifier.io.batch import collate_sequence_batches from sequifier.io.policy_dataset import imitation_example from sequifier.rl.batch_order import BatchOrder +from sequifier.rl.batching import batches as minibatches from sequifier.rl.offline_data import OfflineData from sequifier.rl.run_state import PhaseContext, PhaseSession @@ -55,8 +56,7 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin group = batches.next() session.zero_grad() total = 0.0 - for start in range(0, len(group), phase.batch_size): - indices = group[start : start + phase.batch_size] + for indices in minibatches(group, phase.batch_size): batch = collate_sequence_batches( [imitation_example(training, i) for i in indices] ) @@ -70,14 +70,11 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin final = batches.epoch == phase.epochs if session.should_evaluate(final): score = evaluate_imitation(actor, data.validation, phase.batch_size) - session.observe(score) - session.metrics.append( - { - "transaction": session.transaction, - "epochs_completed": batches.epoch, - "training_action_nll": total, - phase.evaluation.monitor.metric: score, - } + session.record_evaluation( + score, + epochs_completed=batches.epoch, + training_action_nll=total, + **{phase.evaluation.monitor.metric: score}, ) session.checkpoint(final, batch_order=batches.state_dict()) return session.finish( diff --git a/src/sequifier/rollout/coordinator.py b/src/sequifier/rollout/coordinator.py index 7a18632f..2c5ba3ad 100644 --- a/src/sequifier/rollout/coordinator.py +++ b/src/sequifier/rollout/coordinator.py @@ -1,7 +1,5 @@ """Shared prediction/execution ordering and single-environment collection.""" -import math -import warnings from collections.abc import Callable from copy import deepcopy from uuid import uuid4 @@ -21,7 +19,8 @@ numeric_scalar, validate_seed, ) -from sequifier.trajectory.schema import SeedHistory, Trajectory, Transition +from sequifier.trajectory.recording import TrajectoryRecorder, close_adapter +from sequifier.trajectory.schema import SeedHistory, Trajectory def collect_rollout( @@ -73,7 +72,7 @@ def collect_rollout( random_seed, ) adapter.reset(deepcopy(reset_request)) - transitions = [] + recorder = TrajectoryRecorder(episode_id, config.max_episode_steps) external_keys = { binding.key for binding in config.bindings @@ -119,49 +118,31 @@ def collect_rollout( )[binding.key] for binding in config.bindings } - truncated = result.truncated or ( - config.max_episode_steps is not None - and step_id + 1 >= config.max_episode_steps - ) + next_history = advance_history(history, successor) if history_validator is not None: history_validator(next_history.model_copy(deep=True)) - transitions.append( - Transition( - episode_id=episode_id, - step_id=step_id, - observation=observation, - predictions=predictions, - external_inputs=external, - reward=reward, - successor_observation=successor, - terminated=result.terminated, - truncated=truncated, - bootstrap_allowed=not result.terminated, - ) + transition = recorder.append( + observation=observation, + predictions=predictions, + external_inputs=external, + reward=reward, + successor_observation=successor, + terminated=result.terminated, + truncated=result.truncated, ) history = next_history - if result.terminated or truncated: + if transition.terminated or transition.truncated: break - last = transitions[-1] - return Trajectory( + return recorder.finish( format="sequifier.numeric-forecast-trajectory", - episode_id=episode_id, seed=initial_history, bindings=config.bindings, prediction_columns=prediction_columns, - transitions=transitions, - collection_cutoff=not (last.terminated or last.truncated), - total_return=math.fsum(transition.reward for transition in transitions), metadata=metadata or {}, ) except BaseException: failed = True raise finally: - try: - adapter.close() - except Exception as exc: - if not failed: - raise - warnings.warn(f"Adapter close also failed: {exc}", RuntimeWarning) + close_adapter(adapter, failed) diff --git a/src/sequifier/templates/train-dpo.yaml b/src/sequifier/templates/train-dpo.yaml index ccc03d1f..3ff24d48 100644 --- a/src/sequifier/templates/train-dpo.yaml +++ b/src/sequifier/templates/train-dpo.yaml @@ -8,74 +8,51 @@ model: dim_model: 32 num_layers: 1 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 64 + attention: {n_heads: 4} + feed_forward: {dim: 64} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: - - cue - - __policy_previous_action - target_columns: - - action + input_columns: [cue, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - cue: 2 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 4 - normalization: - source: identity + normalization: {source: identity} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} dataset_training: decisions: parts: default: type: sequence - paths: - - data/control-decisions.csv + paths: [data/control-decisions.csv] columns: episode_id: episodeId reward: reward terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: - cue: next_cue + successors: {cue: next_cue} group: contextId model_interface: actor rankings: parts: - default: - type: preferences - manifest: data/control-preferences.json + default: {type: preferences, manifest: data/control-preferences.json} training_plan: phases: - - name: preferences - algorithm: dpo - preferences: - source: rankings.default - updates: 100 - beta: 0.1 + - name: preferences + algorithm: dpo + preferences: {source: rankings.default} + updates: 100 + beta: 0.1 diff --git a/src/sequifier/templates/train-grpo.yaml b/src/sequifier/templates/train-grpo.yaml index 5b1a5017..7126cc5a 100644 --- a/src/sequifier/templates/train-grpo.yaml +++ b/src/sequifier/templates/train-grpo.yaml @@ -8,73 +8,47 @@ model: dim_model: 32 num_layers: 1 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 64 + attention: {n_heads: 4} + feed_forward: {dim: 64} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: - - cue - - __policy_previous_action - target_columns: - - action + input_columns: [cue, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - cue: 2 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 4 - normalization: - source: identity + normalization: {source: identity} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} training_plan: phases: - - name: group_improvement - algorithm: grpo - total_groups: 32 - groups_per_update: 2 - group_size: 4 - max_steps: 8 - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 - implementation_id: sequifier.categorical-control.v1 - evaluation: + - name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_update: 2 + group_size: 4 + max_steps: 8 environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 + config: {horizon: 8} implementation_id: sequifier.categorical-control.v1 - seeds: - - 17 - - 29 - - 43 - - 71 - max_episode_steps: 8 - timeout_seconds: 30 - monitor: - metric: score_mean - mode: max + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 + monitor: {metric: score_mean, mode: max} diff --git a/src/sequifier/templates/train-iql.yaml b/src/sequifier/templates/train-iql.yaml index 77a146f6..bace396a 100644 --- a/src/sequifier/templates/train-iql.yaml +++ b/src/sequifier/templates/train-iql.yaml @@ -8,67 +8,47 @@ model: dim_model: 32 num_layers: 2 max_context_length: 16 - attention: - n_heads: 4 - feed_forward: - dim: 128 + attention: {n_heads: 4} + feed_forward: {dim: 128} positional_encoding_scope: global interfaces: actor: - input_columns: - - OBSERVATION_COLUMN - - __policy_previous_action - target_columns: - - action + input_columns: [OBSERVATION_COLUMN, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - OBSERVATION_COLUMN: 2 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {OBSERVATION_COLUMN: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 16 validation_fraction: 0.2 - resume: - policy: if_exists + resume: {policy: if_exists} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - 0.0 - - 1.0 + action: {type: categorical, values: [0.0, 1.0]} dataset_training: demonstrations: parts: recorded: type: sequence - paths: - - PLEASE_FILL.csv + paths: [PLEASE_FILL.csv] columns: reward: reward terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: - OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN + successors: {OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN} model_interface: actor training_plan: phases: - - name: offline_improvement - algorithm: iql - sources: - - source: demonstrations.recorded - updates: 10000 - batch_size: 64 + - name: offline_improvement + algorithm: iql + sources: + - {source: demonstrations.recorded} + updates: 10000 + batch_size: 64 diff --git a/src/sequifier/templates/train-policy-from-supervised.yaml b/src/sequifier/templates/train-policy-from-supervised.yaml index 98412ff5..49d28caa 100644 --- a/src/sequifier/templates/train-policy-from-supervised.yaml +++ b/src/sequifier/templates/train-policy-from-supervised.yaml @@ -3,31 +3,19 @@ model_name: posttrained device: cpu initialize_from: models/supervised/model.pt global_training: - resume: - policy: if_exists - mode: reset_environments + resume: {policy: if_exists, mode: reset_environments} policy_binding: actions: - mode: - type: categorical - values: - - 'off' - - 'on' - quantity: - type: real - scale: 0.5 + mode: {type: categorical, values: ['off', 'on']} + quantity: {type: real, scale: 0.5} dataset_training: demonstrations: parts: - default: - type: sequence - paths: - - data/demonstrations.parquet - format: table + default: {type: sequence, paths: [data/demonstrations.parquet], format: table} training_plan: phases: - - name: imitation - algorithm: policy_imitation - sources: - - source: demonstrations.default - epochs: 5 + - name: imitation + algorithm: policy_imitation + sources: + - {source: demonstrations.default} + epochs: 5 diff --git a/src/sequifier/templates/train-policy.yaml b/src/sequifier/templates/train-policy.yaml index ac7fc514..a5445a54 100644 --- a/src/sequifier/templates/train-policy.yaml +++ b/src/sequifier/templates/train-policy.yaml @@ -8,68 +8,43 @@ model: dim_model: 32 num_layers: 2 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 128 - activation: swiglu + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: - - OBSERVATION_COLUMN - - __policy_previous_action - - __policy_return - target_columns: - - action + input_columns: [OBSERVATION_COLUMN, __policy_previous_action, __policy_return] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - OBSERVATION_COLUMN: 1 - __policy_previous_action: 6 - __policy_return: 1 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 4 validation_fraction: 0.2 episode_cache_size: 8 - resume: - policy: never + resume: {policy: never} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: true actions: - action: - type: categorical - values: - - 0.0 - - 1.0 + action: {type: categorical, values: [0.0, 1.0]} dataset_training: demonstrations: parts: - recorded: - type: sequence - paths: - - PLEASE_FILL.csv + recorded: {type: sequence, paths: [PLEASE_FILL.csv]} model_interface: actor training_plan: phases: - - algorithm: policy_imitation - epochs: 80 - sources: - - source: demonstrations.recorded - name: policy_imitation - batch_size: 64 - optimization: - actor: - learning_rate: 0.005 + - algorithm: policy_imitation + epochs: 80 + sources: + - {source: demonstrations.recorded} + name: policy_imitation + batch_size: 64 + optimization: + actor: {learning_rate: 0.005} diff --git a/src/sequifier/templates/train-ppo.yaml b/src/sequifier/templates/train-ppo.yaml index c4ae2a35..d5ab5760 100644 --- a/src/sequifier/templates/train-ppo.yaml +++ b/src/sequifier/templates/train-ppo.yaml @@ -8,79 +8,53 @@ model: dim_model: 32 num_layers: 1 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 64 + attention: {n_heads: 4} + feed_forward: {dim: 64} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: - - cue - - __policy_previous_action - target_columns: - - action + input_columns: [cue, __policy_previous_action] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - cue: 2 - __policy_previous_action: 6 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {cue: 2, __policy_previous_action: 6} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 4 - normalization: - source: identity - resume: - policy: if_exists - mode: exact + normalization: {source: identity} + resume: {policy: if_exists, mode: exact} policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: false actions: - action: - type: categorical - values: - - left - - right + action: {type: categorical, values: [left, right]} training_plan: phases: - - name: online_control - algorithm: ppo - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 - implementation_id: sequifier.categorical-control.v1 - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: + - name: online_control + algorithm: ppo environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: - horizon: 8 + config: {horizon: 8} implementation_id: sequifier.categorical-control.v1 - seeds: - - 17 - - 29 - - 43 - - 71 - max_episode_steps: 8 - timeout_seconds: 30 + total_env_steps: 8192 + rollout_length: 256 + optimization_epochs: 4 + minibatch_size: 64 + accumulation_steps: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + evaluation_interval: 2 + checkpoint_interval: 2 + evaluation: + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: {horizon: 8} + implementation_id: sequifier.categorical-control.v1 + seeds: [17, 29, 43, 71] + max_episode_steps: 8 + timeout_seconds: 30 diff --git a/src/sequifier/templates/train-supervised-to-policy.yaml b/src/sequifier/templates/train-supervised-to-policy.yaml index 49d55a05..ad4fea31 100644 --- a/src/sequifier/templates/train-supervised-to-policy.yaml +++ b/src/sequifier/templates/train-supervised-to-policy.yaml @@ -8,97 +8,67 @@ model: dim_model: 32 num_layers: 2 max_context_length: 4 - attention: - n_heads: 4 - feed_forward: - dim: 128 - activation: swiglu + attention: {n_heads: 4} + feed_forward: {dim: 128, activation: swiglu} positional_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: - - OBSERVATION_COLUMN - - __policy_previous_action - - __policy_return - target_columns: - - action + input_columns: [OBSERVATION_COLUMN, __policy_previous_action, __policy_return] + target_columns: [action] ingestion: type: embedding output_dim: 8 - feature_embedding_dims: - OBSERVATION_COLUMN: 1 - __policy_previous_action: 6 - __policy_return: 1 - decoder: - type: linear - prediction_length: 1 - support: 1 - categorical_decoder_special_tokens: - action: [] + feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} + decoder: {type: linear, prediction_length: 1, support: 1} + categorical_decoder_special_tokens: {action: []} global_training: context_length: 4 validation_fraction: 0.2 episode_cache_size: 8 - resume: - policy: never + resume: {policy: never} training_objective: causal torch_compile: none policy_binding: - previous_action_inputs: - __policy_previous_action: action - initial_inputs: - __policy_previous_action: start + previous_action_inputs: {__policy_previous_action: action} + initial_inputs: {__policy_previous_action: start} return_conditioned: true actions: - action: - type: categorical - values: - - 0.0 - - 1.0 + action: {type: categorical, values: [0.0, 1.0]} dataset_training: demonstrations: parts: - recorded: - type: sequence - paths: - - PLEASE_FILL.csv + recorded: {type: sequence, paths: [PLEASE_FILL.csv]} model_interface: actor observations: model_interface: actor parts: - train: - metadata_config_path: configs/metadata_configs/observations.json - criterion: - action: CrossEntropyLoss + train: {metadata_config_path: configs/metadata_configs/observations.json} + criterion: {action: CrossEntropyLoss} training_plan: phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential - sources: - - source: observations.train - batch_size: 64 - accumulation_steps: 1 - optimization: - model: - optimizer: - name: Adam - learning_rate: 0.001 - scheduler: - name: StepLR - arguments: - step_size: 1 - gamma: 0.99 - scheduler_step_on: epoch - gradient_clip: 1.0 - - algorithm: policy_imitation - epochs: 80 - sources: - - source: demonstrations.recorded - name: policy_imitation - batch_size: 64 - optimization: - actor: - learning_rate: 0.005 + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - {source: observations.train} + batch_size: 64 + accumulation_steps: 1 + optimization: + model: + optimizer: {name: Adam} + learning_rate: 0.001 + scheduler: + name: StepLR + arguments: {step_size: 1, gamma: 0.99} + scheduler_step_on: epoch + gradient_clip: 1.0 + - algorithm: policy_imitation + epochs: 80 + sources: + - {source: demonstrations.recorded} + name: policy_imitation + batch_size: 64 + optimization: + actor: {learning_rate: 0.005} diff --git a/src/sequifier/templates/train-supervised.yaml b/src/sequifier/templates/train-supervised.yaml index 578b2d80..5fdc61a0 100644 --- a/src/sequifier/templates/train-supervised.yaml +++ b/src/sequifier/templates/train-supervised.yaml @@ -1,57 +1,41 @@ project_root: . model_name: supervised device: cpu -global_training: - training_objective: causal - context_length: 48 - inference_batch_size: 10 +global_training: {training_objective: causal, context_length: 48, inference_batch_size: 10} model: backbone: architecture: dim_model: 128 num_layers: 3 - attention: - n_heads: 16 - feed_forward: - dim: 128 + attention: {n_heads: 16} + feed_forward: {dim: 128} dropout: 0.2 interface: - input_columns: - - EXAMPLE_INPUT_COLUMN_NAME - target_columns: - - EXAMPLE_TARGET_COLUMN_NAME - ingestion: - type: embedding - output_dim: 128 - decoder: - type: linear - prediction_length: 1 + input_columns: [EXAMPLE_INPUT_COLUMN_NAME] + target_columns: [EXAMPLE_TARGET_COLUMN_NAME] + ingestion: {type: embedding, output_dim: 128} + decoder: {type: linear, prediction_length: 1} dataset: - part: - metadata_config_path: PLEASE FILL - criterion: - EXAMPLE_TARGET_COLUMN_NAME: MSELoss + part: {metadata_config_path: PLEASE FILL} + criterion: {EXAMPLE_TARGET_COLUMN_NAME: MSELoss} training_plan: phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential - sources: - - source: default.default - batch_size: 10 - accumulation_steps: 1 - optimization: - model: - learning_rate: 0.0001 - optimizer: - name: AdamW - scheduler: - name: StepLR - arguments: - step_size: 1 - gamma: 0.99 - scheduler_step_on: epoch + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - {source: default.default} + batch_size: 10 + accumulation_steps: 1 + optimization: + model: + learning_rate: 0.0001 + optimizer: {name: AdamW} + scheduler: + name: StepLR + arguments: {step_size: 1, gamma: 0.99} + scheduler_step_on: epoch evaluation: true export_generative_model: true export_embedding_model: false diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 74232758..23c6095c 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -16,6 +16,7 @@ read_execution_artifact, save_model_artifact, ) +from sequifier.artifacts.phase import PhaseArtifacts, verify_artifact from sequifier.config.train_config import SequifierConfig, resolve_sequifier_config from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save, write_json_atomic @@ -98,24 +99,9 @@ def _supervised_config(config, phase, index, incoming, resume): for dataset in resolved.dataset_training.values(): previous = source.interface(dataset.model_interface) dataset.interface.policy = previous.policy - from sequifier.artifacts.model_config import execution_config + from sequifier.model.execution import ModelExecution - existing = { - dataset.model_interface for dataset in resolved.dataset_training.values() - } - for name in source.interface_names: - if name in existing: - continue - interface = source.interface(name) - placeholder = execution_config( - source.model, {name: interface}, resolved.global_training - ).dataset_training[name] - key = name - while key in resolved.dataset_training: - key += "_unused" - resolved.dataset_training[key] = placeholder.model_copy( - update={"name": key} - ) + ModelExecution.from_config(source).retain_interfaces(resolved) if incoming is not None: from sequifier.artifacts.model_config import validate_execution_attachment from sequifier.artifacts.model_export import model_execution_config @@ -164,9 +150,7 @@ def _run_supervised(config, phase, index, incoming, destination, resume): run.distributed.finalize() -def _policy_config( - config, phase, index, incoming, destination, resume, execution_model=None -): +def _policy_config(config, phase, index, incoming, destination, resume): from sequifier.config.training_resolution import policy_run_config return policy_run_config( @@ -221,35 +205,28 @@ def run_learning_plan(config) -> LearningRunResult: raise FileNotFoundError(checkpoint) if not 0 <= len(state["completed"]) <= len(phases): raise ValueError("learning run checkpoint exceeds the phase plan") - if ( - state["incoming"] - and file_checksum(state["incoming"]) != state["incoming_sha256"] - ): - raise ValueError("committed incoming model artifact changed") + verify_artifact( + state["incoming"], + state["incoming_sha256"], + "committed incoming model artifact changed", + ) # Validate model/source/environment contracts for the complete plan first. - from sequifier.artifacts.model_export import model_execution_config from sequifier.rl.preparation import RunInputs, preflight_environments, prepare_data - execution_model: dict[str, Any] | None = ( - read_execution_artifact(state["incoming"]).model_config.values - if state["incoming"] - else None - ) dependencies = {} for index, phase in enumerate(phases): if phase.algorithm == "supervised": resolved = _supervised_config( config, phase, index, state["incoming"], False ) - execution_model = model_execution_config(resolved) for dataset in resolved.dataset_training.values(): for part in dataset.parts.values(): path = str(Path(config.project_root) / part.metadata_config_path) dependencies[path] = file_checksum(path) else: - destination = ( - Path(config.phase_directory) / f"{index:03d}-{phase.name}" / "model.pt" - ) + destination = PhaseArtifacts.for_phase( + config.phase_directory, index, phase.name, "model.pt" + ).model run = _policy_config( config, phase, @@ -257,7 +234,6 @@ def run_learning_plan(config) -> LearningRunResult: state["incoming"], destination, False, - execution_model, ) inputs = RunInputs.resolve(run) dependencies.update(inputs.identity) @@ -265,11 +241,6 @@ def run_learning_plan(config) -> LearningRunResult: prepare_data(run, run.policy, inputs) if run.execution_model is None or run.policy_interface is None: raise ValueError("a policy phase has no resolved model interface") - execution_model = dict(run.execution_model) - execution_model["interfaces"] = dict(execution_model["interfaces"]) - selected = dict(execution_model["interfaces"][run.policy_interface]) - selected["policy"] = run.policy.model_dump(mode="json") - execution_model["interfaces"][run.policy_interface] = selected if "dependencies" in state and state["dependencies"] != dependencies: raise ValueError("learning plan source identities changed since its checkpoint") state["dependencies"] = dependencies @@ -294,48 +265,32 @@ def run_learning_plan(config) -> LearningRunResult: for index in range(len(state["completed"]), len(phases)): phase = phases[index] incoming = state["incoming"] - if incoming and file_checksum(incoming) != state["incoming_sha256"]: - raise ValueError( - "incoming model artifact changed since the committed boundary" - ) - destination = ( - Path(config.phase_directory) / f"{index:03d}-{phase.name}" / "model.pt" + verify_artifact( + incoming, + state["incoming_sha256"], + "incoming model artifact changed since the committed boundary", + ) + artifacts = PhaseArtifacts.for_phase( + config.phase_directory, index, phase.name, "model.pt" ) + destination = artifacts.model resuming = state["active"] and resume.policy != "never" if not state["active"]: state["active"] = True atomic_save(checkpoint, state) - receipt_path = destination.with_name("completed.json") - import json - - receipt = ( - json.loads(receipt_path.read_text()) - if resuming and receipt_path.is_file() - else None - ) phase_identity = fingerprint( { "phase": phase.model_dump(mode="json"), "incoming": state["incoming_sha256"], } ) + receipt = artifacts.read_receipt(phase_identity) if resuming else None if receipt is not None: - if receipt["identity"] != phase_identity or receipt[ - "sha256" - ] != file_checksum(destination): - raise ValueError("completed phase artifact or receipt changed") report = receipt["report"] else: runner = _run_supervised if phase.algorithm == "supervised" else _run_policy report = runner(config, phase, index, incoming, destination, resuming) - write_json_atomic( - receipt_path, - { - "identity": phase_identity, - "sha256": file_checksum(destination), - "report": report, - }, - ) + artifacts.write_receipt(phase_identity, report) state["completed"].append( { "name": phase.name, diff --git a/src/sequifier/trajectory/recording.py b/src/sequifier/trajectory/recording.py new file mode 100644 index 00000000..f5c7b9bf --- /dev/null +++ b/src/sequifier/trajectory/recording.py @@ -0,0 +1,48 @@ +"""Shared trajectory assembly and adapter cleanup for local collectors.""" + +import math +import warnings + +from sequifier.trajectory.schema import Trajectory, Transition + + +class TrajectoryRecorder: + def __init__(self, episode_id, max_episode_steps): + self.episode_id, self.max_episode_steps = episode_id, max_episode_steps + self.transitions = [] + + def append(self, *, terminated, truncated, **values): + step = len(self.transitions) + transition = Transition( + episode_id=self.episode_id, + step_id=step, + terminated=terminated, + truncated=truncated + or ( + self.max_episode_steps is not None + and step + 1 >= self.max_episode_steps + ), + bootstrap_allowed=not terminated, + **values, + ) + self.transitions.append(transition) + return transition + + def finish(self, **values): + last = self.transitions[-1] + return Trajectory( + episode_id=self.episode_id, + transitions=self.transitions, + collection_cutoff=not (last.terminated or last.truncated), + total_return=math.fsum(step.reward for step in self.transitions), + **values, + ) + + +def close_adapter(adapter, failed): + try: + adapter.close() + except Exception as exc: + if not failed: + raise + warnings.warn(f"Adapter close also failed: {exc}", RuntimeWarning) From 055c09a613a6a92271993476c20f29736e6896b8 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 11 Nov 2026 10:44:53 +0100 Subject: [PATCH 27/71] shorten --- documentation/learning-architecture.md | 7 + src/sequifier/config/policy_inputs.py | 112 ++++++++++ src/sequifier/config/train_config.py | 3 + src/sequifier/config/training_resolution.py | 70 +++---- src/sequifier/io/episode_features.py | 31 ++- src/sequifier/io/output_paths.py | 40 ++++ src/sequifier/io/policy_dataset.py | 26 ++- src/sequifier/io/policy_window.py | 5 +- src/sequifier/io/preference_dataset.py | 19 +- src/sequifier/io/prepared_policy_data.py | 2 + src/sequifier/rl/attachment.py | 27 +-- src/sequifier/rl/clipped_objective.py | 26 +++ src/sequifier/rl/grpo_update.py | 18 +- src/sequifier/rl/learner.py | 9 +- src/sequifier/rl/offline_data.py | 5 + src/sequifier/rl/ppo_update.py | 13 +- src/sequifier/rl/preparation.py | 106 ++-------- src/sequifier/rl/prepared_inputs.py | 213 ++++++++++++++++++++ src/sequifier/rl/run_config.py | 6 +- src/sequifier/training/coordinator.py | 39 ++-- src/sequifier/trajectory/importers.py | 8 +- src/sequifier/trajectory/tables.py | 4 +- 22 files changed, 582 insertions(+), 207 deletions(-) create mode 100644 src/sequifier/config/policy_inputs.py create mode 100644 src/sequifier/io/output_paths.py create mode 100644 src/sequifier/rl/clipped_objective.py create mode 100644 src/sequifier/rl/prepared_inputs.py diff --git a/documentation/learning-architecture.md b/documentation/learning-architecture.md index fff96bdf..b2a8664e 100644 --- a/documentation/learning-architecture.md +++ b/documentation/learning-architecture.md @@ -20,6 +20,13 @@ Policy bindings select existing targets; lagged actions and return conditioning explicit inputs. Categorical support, real scales/bounds/atoms and conditional activity have the same meaning during training, collection, replay and inference. +An internal preparation owner retains resolved source identities, episode catalogs +and partitions from metadata fitting through preflight and execution. Feature codecs +and execution schemas refresh dataset views without selecting the partitions again; +changes to episode interpretation or split settings create new views. Dependencies +are still revalidated before execution and at phase boundaries. This preparation +state is private and does not enter configuration serialization or checkpoints. + Historical decisions are reconstructed at their actual observation boundary. Termination disables bootstrap; truncation preserves the true final successor; a collection cutoff does not invent either boundary. Training/validation splits diff --git a/src/sequifier/config/policy_inputs.py b/src/sequifier/config/policy_inputs.py new file mode 100644 index 00000000..38b8b2bb --- /dev/null +++ b/src/sequifier/config/policy_inputs.py @@ -0,0 +1,112 @@ +"""Internal policy feature layout shared by fitting and model attachment.""" + +from dataclasses import dataclass, replace + +from sequifier.config.policy_config import PolicyBinding, ResolvedPolicyBinding + + +@dataclass(frozen=True, kw_only=True) +class PolicyInputLayout: + observation_columns: list[str] + static_columns: list[str] + categorical_columns: list[str] + observation_n_classes: dict[str, int] + static_n_classes: dict[str, int] + id_maps: dict[str, dict[str | int, int]] + + @classmethod + def select( + cls, + temporal, + static, + binding, + categorical, + n_classes, + id_maps, + *, + features_only=False, + ): + observations = [ + name + for name in temporal + if name not in binding.previous_action_inputs + and (name != "__policy_return" or not binding.return_conditioned) + ] + vocabulary_columns = observations if features_only else temporal + columns = set(vocabulary_columns) | set(static) + return cls( + observation_columns=observations, + static_columns=list(static), + categorical_columns=list(categorical), + observation_n_classes={ + k: n for k, n in n_classes.items() if k in vocabulary_columns + }, + static_n_classes={k: n for k, n in n_classes.items() if k in static}, + id_maps={k: dict(v) for k, v in id_maps.items() if k in columns}, + ) + + @classmethod + def for_fitting(cls, temporal, static, binding, metadata): + layout = cls.select( + temporal, + static, + binding, + [], + metadata.n_classes, + metadata.id_maps, + features_only=True, + ) + return replace( + layout, + categorical_columns=[ + name + for name in layout.observation_columns + layout.static_columns + if "int" in metadata.column_data_types[name].lower() + ], + ) + + @property + def categorical_n_classes(self): + return {**self.observation_n_classes, **self.static_n_classes} + + def binding_values(self): + return { + "observation_columns": self.observation_columns, + "static_columns": self.static_columns, + "categorical_columns": self.categorical_columns, + "observation_n_classes": self.observation_n_classes, + "static_n_classes": self.static_n_classes, + "id_maps": self.id_maps, + } + + +@dataclass(frozen=True, kw_only=True) +class PolicyFittingSpec(PolicyInputLayout): + """Raw episode features before an executable model schema exists.""" + + binding: PolicyBinding + context_length: int + interface_name: str + + @property + def actions(self): + return self.binding.actions + + @property + def action_columns(self): + return list(self.actions) + + @property + def return_conditioned(self): + return self.binding.return_conditioned + + @property + def returns(self): + return self.binding.returns + + @property + def input_encoding(self): + return "raw" + + +PolicyEpisodeSpec = ResolvedPolicyBinding | PolicyFittingSpec diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 16f588fa..53651665 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -23,6 +23,7 @@ BaseModel, ConfigDict, Field, + PrivateAttr, StrictStr, field_validator, model_validator, @@ -1403,6 +1404,8 @@ def resolve_supervised_phases(authored_phases): class ResolvedSequifierConfig(BaseModel): """Runtime configuration after all dataset parts and interfaces resolve.""" + _prepared_inputs: Any = PrivateAttr(default=None) + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") project_root: str diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index af067688..4a4bdc8c 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -175,14 +175,11 @@ def _inferred_metadata(config, name, binding): ) -def _fit_input_metadata(config, name, binding, metadata, root): +def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): """Fit missing raw codecs/moments on the same episode-safe training split.""" from types import SimpleNamespace - from sequifier.config.policy_config import ResolvedPolicyBinding - from sequifier.io.policy_dataset import PolicyDataCatalog - from sequifier.io.preference_dataset import prepare_preferences - from sequifier.rl.offline_data import prepare_offline + from sequifier.config.policy_inputs import PolicyFittingSpec, PolicyInputLayout phase = next( ( @@ -199,17 +196,10 @@ def _fit_input_metadata(config, name, binding, metadata, root): static = ( interface.static_conditioning.columns if interface.static_conditioning else [] ) - observations = [ - column - for column in interface.input_columns - if column not in binding.previous_action_inputs - and not (binding.return_conditioned and column == "__policy_return") - ] - categorical = [ - column - for column in observations + static - if "int" in metadata.column_data_types[column].lower() - ] + layout = PolicyInputLayout.for_fitting( + interface.input_columns, static, binding, metadata + ) + observations, categorical = layout.observation_columns, layout.categorical_columns missing_codecs = set(categorical) - metadata.n_classes.keys() missing_statistics = ( set(observations + static) @@ -220,29 +210,11 @@ def _fit_input_metadata(config, name, binding, metadata, root): not missing_statistics or config.global_training.normalization is not None ): return metadata - provisional = ResolvedPolicyBinding( - **binding.model_dump(), + provisional = PolicyFittingSpec( + **layout.binding_values(), + binding=binding, context_length=config.global_training.context_length, interface_name=name, - observation_columns=observations, - static_columns=static, - categorical_columns=categorical, - observation_n_classes={ - column: metadata.n_classes[column] - for column in observations - if column in metadata.n_classes - }, - static_n_classes={ - column: metadata.n_classes[column] - for column in static - if column in metadata.n_classes - }, - id_maps={ - column: mapping - for column, mapping in metadata.id_maps.items() - if column in observations + static - }, - input_schema={}, ) sources, preferences = TrainingSources(config, root).resolve(prepared=False) phase = phase.model_copy(deep=True) @@ -258,15 +230,11 @@ def _fit_input_metadata(config, name, binding, metadata, root): seed=config.seed, preferences=preferences, sources=sources, + episode_cache_size=config.global_training.episode_cache_size, ) - catalog = PolicyDataCatalog( - sources, provisional, config.global_training.episode_cache_size - ) - data = ( - prepare_preferences(run, phase, provisional, catalog, {}) - if phase.algorithm == "dpo" - else prepare_offline(run, phase, provisional, catalog, {}) - ) + inputs = prepared_inputs.resolve(run) + inputs.revalidate(inputs.identity, f"interfaces.{name}.fitting_sources") + data = prepared_inputs.prepare_phase(run, phase, provisional, inputs, fitting=True) fitting = data.fitting_dataset() fitted = fitting.fit_categorical_config() values = metadata.model_dump() @@ -317,7 +285,9 @@ def resolve_training_run(authored: SequifierConfig): resolved_config_from_model_config, ) from sequifier.rl.attachment import apply_binding, resolve_binding + from sequifier.rl.prepared_inputs import PreparedTrainingInputs + prepared_inputs = PreparedTrainingInputs() root = Path(authored.project_root).resolve() config = authored.model_copy(deep=True) config.project_root = str(root) @@ -443,7 +413,9 @@ def resolve_training_run(authored: SequifierConfig): value = _inferred_metadata(config, name, binding) binding = config.policy_bindings.get(name) if binding is not None: - value = _fit_input_metadata(config, name, binding, value, root) + value = _fit_input_metadata( + config, name, binding, value, root, prepared_inputs + ) interfaces[name] = _resolve_interface( name, interface, value, training, value.storage_layout ) @@ -528,6 +500,7 @@ def resolve_training_run(authored: SequifierConfig): for name, interface in interfaces.items() if interface.policy is not None } + resolved._prepared_inputs = prepared_inputs resolved.authored = config resolved.initialize_from = config.initialize_from resolved.model_path = str( @@ -585,6 +558,7 @@ def policy_run_config( from sequifier.artifacts.model_export import model_execution_config from sequifier.config.components import ResumeConfig from sequifier.rl.attachment import apply_binding, resolve_binding + from sequifier.rl.prepared_inputs import PreparedTrainingInputs from sequifier.rl.run_config import ResolvedPolicyRun names = {phase.model_interface for phase in phases} @@ -619,7 +593,7 @@ def policy_run_config( checkpoint = ( Path(config.phase_directory) / f"{index:03d}-{phases[0].name}" / "run.pt" ) - return ResolvedPolicyRun( + run = ResolvedPolicyRun( project_root=config.project_root, model_name=config.model_name, device=config.device, @@ -653,3 +627,5 @@ def policy_run_config( if destination is None else str(Path(destination).parent / "native"), ) + run._prepared_inputs = PreparedTrainingInputs.for_run(config) + return run diff --git a/src/sequifier/io/episode_features.py b/src/sequifier/io/episode_features.py index f3dae585..1b5d6727 100644 --- a/src/sequifier/io/episode_features.py +++ b/src/sequifier/io/episode_features.py @@ -1,8 +1,12 @@ """Fit and validate frozen policy features over an episode view.""" +from copy import deepcopy +from dataclasses import replace + import polars as pl from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.policy_inputs import PolicyFittingSpec from sequifier.io.policy_window import encode_feature from sequifier.io.preprocessing import ( _finite_mean_and_std, @@ -40,9 +44,12 @@ def values(self, episode, length, column): else [episode.static_inputs[column]] ) - def fit_categorical_config(self) -> ResolvedPolicyBinding: + def fit_categorical_config(self) -> ResolvedPolicyBinding | PolicyFittingSpec: """Fit only declared raw categorical features, using the training episodes.""" - values = self.config.model_dump() + values = { + key: deepcopy(getattr(self.config, key)) + for key in ("id_maps", "observation_n_classes", "static_n_classes") + } for column in self.config.categorical_columns: if ( column in self.config.id_maps @@ -73,13 +80,23 @@ def fit_categorical_config(self) -> ResolvedPolicyBinding: else "static_n_classes" ) values[key][column] = max(mapping.values()) + 1 - return ResolvedPolicyBinding.model_validate(values) + if isinstance(self.config, PolicyFittingSpec): + return replace(self.config, **values) + return ResolvedPolicyBinding.model_validate( + {**self.config.model_dump(), **values} + ) - def use_config(self, config: ResolvedPolicyBinding) -> None: - if self._validated_config == config: + def rebind(self, config): + """Refresh feature views without changing episode membership or validating windows.""" + if self.config == config: return self.config = config - self._validated_statistics = None + self._validated_config = self._validated_statistics = None + + def use_config(self, config: ResolvedPolicyBinding | PolicyFittingSpec) -> None: + if self._validated_config == config: + return + self.rebind(config) # Validate held-out values against the fixed training codec before updates. for episode, length in zip(self.episodes, self.lengths): for column, values in episode.observations.items(): @@ -88,7 +105,7 @@ def use_config(self, config: ResolvedPolicyBinding) -> None: for column, value in episode.static_inputs.items(): encode_feature(value, column, config) - self._validated_config = config.model_copy(deep=True) + self._validated_config = deepcopy(config) def validate_normalization(self, statistics): """Check FP32 transforms episode-by-episode without constructing windows.""" diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py new file mode 100644 index 00000000..a12b6c38 --- /dev/null +++ b/src/sequifier/io/output_paths.py @@ -0,0 +1,40 @@ +"""Shared path geometry and destination checks for training outputs.""" + +from pathlib import Path + + +def unique_paths(paths, message): + paths = [Path(path).resolve() for path in paths] + if len(set(paths)) != len(paths): + raise ValueError(message) + return paths + + +def contains_path(directory, path): + return directory == path or directory in path.parents + + +def paths_overlap(left, right): + return contains_path(left, right) or contains_path(right, left) + + +def validate_path_conflicts(paths, protected, *, label, check_each_other=True): + protected = {Path(path).resolve() for path in protected} + paths = [Path(path).resolve() for path in paths] + for path in paths: + others = protected | (set(paths) - {path} if check_each_other else set()) + for other in others: + if paths_overlap(path, other): + raise ValueError(f"{label} {path} conflicts with input/output {other}") + + +def require_file_destinations(paths): + for path in paths: + if path.is_dir() or any(p.exists() and not p.is_dir() for p in path.parents): + raise ValueError(f"output requires a file destination: {path}") + + +def require_directory_destinations(paths): + for path in paths: + if path.exists() and not path.is_dir(): + raise ValueError(f"archive requires a directory: {path}") diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index be2bcea0..0a9ddd9e 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -7,7 +7,7 @@ import torch from torch.utils.data import Dataset, Sampler -from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.policy_inputs import PolicyEpisodeSpec from sequifier.io.batch import SequifierBatch from sequifier.io.episode_corpus import EpisodeCorpus, EpisodeView, episode_content_key from sequifier.io.episode_features import EpisodeFeatures, return_labels @@ -21,7 +21,7 @@ class EpisodeDecisions(EpisodeFeatures): def __init__( self, sources: Sequence | EpisodeView, - config: ResolvedPolicyBinding, + config: PolicyEpisodeSpec, *, episode_cache_size: int = 8, ): @@ -85,6 +85,11 @@ def __len__(self): def sampler(self, generator): return EpisodeWindowSampler(self, generator) + def rebind(self, config): + if self.config != config: + self._labels.clear() + super().rebind(config) + def _episode_labels(self, index: int): if index not in self._labels: episode = self.episodes[index] @@ -164,10 +169,25 @@ def keys(index): class PolicyDataCatalog: """Run-owned corpora and reusable decision views for named input sources.""" - def __init__(self, sources, config, cache_size): + def __init__(self, sources, config, cache_size, *, preference_manifests=None): self.sources, self.config, self.cache_size = sources, config, cache_size self.corpora = {} self.datasets = {} + self.preference_manifests = ( + preference_manifests if preference_manifests is not None else {} + ) + + def preference_pairs(self, path): + if path not in self.preference_manifests: + from sequifier.trajectory.preferences import load_preferences + + self.preference_manifests[path] = load_preferences(path).pairs + return self.preference_manifests[path] + + def rebind(self, config): + self.config = config + for dataset in self.datasets.values(): + dataset.rebind(config) def dataset(self, names): key = tuple(names) diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py index d6e04728..d93fe066 100644 --- a/src/sequifier/io/policy_window.py +++ b/src/sequifier/io/policy_window.py @@ -3,6 +3,7 @@ from collections.abc import Mapping, Sequence from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.policy_inputs import PolicyEpisodeSpec from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view from sequifier.io.batch import SequifierInputs from sequifier.io.preprocessing import normalize_values @@ -32,13 +33,13 @@ def normalize_policy_features(features, metadata, interface): return features -def action_index(value, config: ResolvedPolicyBinding) -> int | dict[str, int | float]: +def action_index(value, config: PolicyEpisodeSpec) -> int | dict[str, int | float]: from sequifier.io.actions import encode_action return encode_action(value, config) -def encode_feature(value, column: str, config: ResolvedPolicyBinding): +def encode_feature(value, column: str, config: PolicyEpisodeSpec): from sequifier.io.policy_materialization import encode_value return encode_value(value, column, config) diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index cd5fc545..655f44e8 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -10,7 +10,6 @@ from sequifier.io.episode_features import EpisodeFeatures from sequifier.io.partitioning import split_components from sequifier.io.policy_window import PolicyWindowView, action_index -from sequifier.trajectory.preferences import load_preferences @dataclass(frozen=True) @@ -25,6 +24,10 @@ def contract(self): def fitting_dataset(self): return self.training.fitting_dataset() + def rebind(self, config): + for partition in (self.training, self.validation): + partition.rebind(config) + def validate(self, actor): for partition in (self.training, self.validation): partition.use_config(actor.config) @@ -125,10 +128,18 @@ def fitting_dataset(self): ) return self._fitting + def rebind(self, config): + if self.config == config: + return + self.config = config + self._validated_config = None + if self._fitting is not None: + self._fitting.rebind(config) + def use_config(self, config): if self._validated_config == config: return - self.config = config + self.rebind(config) view = PolicyWindowView(config) contexts = {} for pair in self.pairs: @@ -193,12 +204,12 @@ def prepare_preferences(run, phase, config, catalog, cache): ) if key in cache: return cache[key] - pairs = load_preferences(run.preferences[phase.preferences.source]).pairs + pairs = catalog.preference_pairs(run.preferences[phase.preferences.source]) validation = ( [ pair for ref in phase.evaluation.sources - for pair in load_preferences(run.preferences[ref.source]).pairs + for pair in catalog.preference_pairs(run.preferences[ref.source]) ] if phase.evaluation.sources else None diff --git a/src/sequifier/io/prepared_policy_data.py b/src/sequifier/io/prepared_policy_data.py index c7609bce..c0703eb4 100644 --- a/src/sequifier/io/prepared_policy_data.py +++ b/src/sequifier/io/prepared_policy_data.py @@ -18,4 +18,6 @@ def contract(self) -> list: ... def fitting_dataset(self) -> EpisodeFeatures: ... + def rebind(self, config) -> None: ... + def validate(self, actor: "Policy") -> None: ... diff --git a/src/sequifier/rl/attachment.py b/src/sequifier/rl/attachment.py index 7f7936dd..0239e55c 100644 --- a/src/sequifier/rl/attachment.py +++ b/src/sequifier/rl/attachment.py @@ -1,6 +1,7 @@ """Attach a policy task to a resolved model without rebuilding its interface.""" from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.policy_inputs import PolicyInputLayout from sequifier.model.execution_schema import ExecutionSchema @@ -49,27 +50,19 @@ def resolve_binding(execution, interface_name, binding): raise ValueError( "return conditioning requires an explicit __policy_return model input" ) + layout = PolicyInputLayout.select( + temporal, + static, + binding, + [k for k in interface.categorical_columns if k in temporal], + interface.n_classes, + interface.id_maps, + ) values.update( + **layout.binding_values(), actions=actions, action_order=[name for name in interface.target_columns if name in actions], return_conditioned=conditioned, - observation_columns=[ - name - for name in temporal - if name not in binding.previous_action_inputs - and (name != "__policy_return" or not conditioned) - ], - static_columns=list(static), - observation_n_classes={ - k: n for k, n in interface.n_classes.items() if k in temporal - }, - static_n_classes={k: n for k, n in interface.n_classes.items() if k in static}, - categorical_columns=[k for k in interface.categorical_columns if k in temporal], - id_maps={ - k: v - for k, v in interface.id_maps.items() - if k in set(temporal) | set(static) - }, input_schema=schema.to_dict(), input_statistics={ name: stats diff --git a/src/sequifier/rl/clipped_objective.py b/src/sequifier/rl/clipped_objective.py new file mode 100644 index 00000000..ffe6c9a1 --- /dev/null +++ b/src/sequifier/rl/clipped_objective.py @@ -0,0 +1,26 @@ +"""Elementwise clipped actor objective, independent of algorithm weighting.""" + +from dataclasses import dataclass + +import torch +from torch import Tensor + + +@dataclass(frozen=True) +class ClippedPolicyObjective: + loss: Tensor + ratio: Tensor + approximate_kl: Tensor + clip_fraction: Tensor + + +def clipped_policy_objective(log_probability, old_log_probability, advantages, epsilon): + log_ratio = log_probability.float() - old_log_probability.float() + ratio = log_ratio.exp() + clipped = ratio.clamp(1 - epsilon, 1 + epsilon) + return ClippedPolicyObjective( + loss=-torch.minimum(ratio * advantages, clipped * advantages), + ratio=ratio, + approximate_kl=ratio - 1 - log_ratio, + clip_fraction=((ratio - 1).abs() > epsilon).float(), + ) diff --git a/src/sequifier/rl/grpo_update.py b/src/sequifier/rl/grpo_update.py index 58484c2c..9a699fdb 100644 --- a/src/sequifier/rl/grpo_update.py +++ b/src/sequifier/rl/grpo_update.py @@ -3,26 +3,24 @@ import torch from sequifier.rl.batching import MetricTotals, shuffled_groups +from sequifier.rl.clipped_objective import clipped_policy_objective def grpo_objective( log_probability, old_log_probability, reference_log_probability, advantage, phase ): - log_ratio = log_probability.float() - old_log_probability.float() - ratio = log_ratio.exp() + policy = clipped_policy_objective( + log_probability, old_log_probability, advantage, phase.clip_epsilon + ) reference_delta = reference_log_probability.float() - log_probability.float() penalty = torch.expm1(reference_delta) - reference_delta - clipped = ratio.clamp(1 - phase.clip_epsilon, 1 + phase.clip_epsilon) - loss = ( - -torch.minimum(ratio * advantage, clipped * advantage) - + phase.kl_coefficient * penalty - ) - old_kl = ratio - 1 - log_ratio - if any(not torch.isfinite(t).all() for t in (ratio, penalty, loss, old_kl)): + loss = policy.loss + phase.kl_coefficient * penalty + old_kl = policy.approximate_kl + if any(not torch.isfinite(t).all() for t in (policy.ratio, penalty, loss, old_kl)): raise ValueError( "non-finite GRPO ratio, sampled reference penalty or objective" ) - return loss, penalty, old_kl, ((ratio - 1).abs() > phase.clip_epsilon).float() + return loss, penalty, old_kl, policy.clip_fraction @torch.no_grad() diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 48a9ab13..ffd9f4b0 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -9,7 +9,6 @@ from sequifier.io.atomic import write_json_atomic from sequifier.rl.checkpoint import load_policy, phase_actor_path, save_policy from sequifier.rl.preparation import ( - RunInputs, build_actor, preflight_environments, preflight_optimization, @@ -19,6 +18,7 @@ validate_declared_policy, validate_output_paths, ) +from sequifier.rl.prepared_inputs import PreparedTrainingInputs from sequifier.rl.references import PolicyReferences from sequifier.rl.registry import BUILTIN_ALGORITHMS from sequifier.rl.run_checkpoint import RunCheckpoint @@ -40,7 +40,7 @@ def _plan_contract(config): return values -def _initialize_actor(config, data): +def _initialize_actor(config): if config.initialize_from: source = load_policy( config.initialize_from, @@ -95,7 +95,8 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: config = policy_run_config(config, config.training_plan) configure_determinism(config.seed, config.enforce_determinism) plan = _plan_contract(config) - inputs = RunInputs.resolve(config) + inputs = PreparedTrainingInputs.for_run(config).resolve(config) + inputs.revalidate(inputs.identity, "run.source_identity") data_identity = inputs.identity validate_output_paths(config, inputs) checkpoint = Path(config.checkpoint_path) @@ -115,7 +116,7 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: recovery.validate_actor(actor, config.units) else: data = prepare_data(config, config.policy, inputs) - actor = _initialize_actor(config, data) + actor = _initialize_actor(config) representation = ActorRepresentation.from_actor(actor, config.units) if config.normalization is not None: require_equal( diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index e0bb88db..c751e7da 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -34,8 +34,13 @@ def contract(self): ] def use_config(self, config): + self.rebind(config) for part in (self.training, self.validation): part.use_config(config) + + def rebind(self, config): + for part in (self.training, self.validation): + part.rebind(config) if self.transitions is not None: for transitions in self.transitions: transitions.use_config(config) diff --git a/src/sequifier/rl/ppo_update.py b/src/sequifier/rl/ppo_update.py index 51a99689..cf1c0279 100644 --- a/src/sequifier/rl/ppo_update.py +++ b/src/sequifier/rl/ppo_update.py @@ -5,6 +5,7 @@ import torch from sequifier.rl.batching import MetricTotals, batches, shuffled_groups +from sequifier.rl.clipped_objective import clipped_policy_objective def ppo_objectives( @@ -17,10 +18,10 @@ def ppo_objectives( entropy, phase, ): - log_ratio = log_probability.float() - old_log_probability.float() - ratio = log_ratio.exp() - clipped = ratio.clamp(1 - phase.clip_epsilon, 1 + phase.clip_epsilon) - policy_loss = -torch.minimum(ratio * advantages, clipped * advantages) + policy = clipped_policy_objective( + log_probability, old_log_probability, advantages, phase.clip_epsilon + ) + policy_loss, ratio = policy.loss, policy.ratio value_error = (prediction.float() - target).square() if phase.value_clip is not None: clipped_value = old_value + (prediction.float() - old_value).clamp( @@ -28,7 +29,7 @@ def ppo_objectives( ) value_error = torch.maximum(value_error, (clipped_value - target).square()) value_loss = 0.5 * value_error - approximate_kl = ratio - 1 - log_ratio + approximate_kl = policy.approximate_kl tensors = (ratio, policy_loss, value_loss, entropy, approximate_kl) if any(not torch.isfinite(tensor).all() for tensor in tensors): raise ValueError("non-finite PPO probability ratio or objective") @@ -40,7 +41,7 @@ def ppo_objectives( "value_loss": value_loss, "entropy": entropy, "approximate_kl": approximate_kl, - "clip_fraction": ((ratio - 1).abs() > phase.clip_epsilon).float(), + "clip_fraction": policy.clip_fraction, "ratio": ratio, }, ) diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 8c0a8a16..a18280af 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -1,11 +1,15 @@ """Read-only policy run preparation and whole-plan preflight validation.""" -from dataclasses import dataclass from pathlib import Path import torch -from sequifier.io.policy_dataset import PolicyDataCatalog +from sequifier.io.output_paths import ( + require_directory_destinations, + require_file_destinations, + unique_paths, + validate_path_conflicts, +) from sequifier.io.prepared_policy_data import PreparedPolicyData from sequifier.model.factory import build_transformer_network from sequifier.rl.checkpoint import phase_actor_path, validate_policy_weights @@ -17,47 +21,9 @@ ) from sequifier.rl.policy import Policy, policy_execution_config from sequifier.rl.ppo_evaluation import require_evaluation_timer -from sequifier.rl.registry import BUILTIN_ALGORITHMS +from sequifier.rl.prepared_inputs import PreparedTrainingInputs, RunInputs # noqa: F401 from sequifier.rl.run_state import require_equal from sequifier.runtime.random_state import capture_rng_state, restore_rng_state -from sequifier.trajectory.sources import ( - ResolvedSequenceSource, - file_checksum, - resolve_source, -) - - -@dataclass(frozen=True) -class RunInputs: - """Resolved sources and verified file identities owned by one run.""" - - sources: dict[str, ResolvedSequenceSource] - checksums: dict[Path, str] - - @classmethod - def resolve(cls, config): - checksums = {} - sources = { - name: resolve_source(source, checksums=checksums) - for name, source in config.sources.items() - } - for path in config.preferences.values(): - path = Path(path) - if path not in checksums: - checksums[path] = file_checksum(path) - return cls(sources, checksums) - - @property - def identity(self): - return {str(path.resolve()): value for path, value in self.checksums.items()} - - def revalidate(self, expected, field): - """Rehash every dependency, including manifests, at each phase boundary.""" - require_equal( - expected, - {str(path.resolve()): file_checksum(path) for path in self.checksums}, - field, - ) def validate_output_paths(config, inputs): @@ -78,39 +44,20 @@ def validate_output_paths(config, inputs): for i, phase in enumerate(config.phases): actor = phase_actor_path(config.phase_directory, i, phase.name) outputs += [actor, actor.with_name("report.json")] - outputs = [path.resolve() for path in outputs] - archives = [ - Path(phase.archive_directory).resolve() - for phase in config.phases - if getattr(phase, "archive_directory", None) - ] - if len(set(outputs)) != len(outputs) or len(set(archives)) != len(archives): - raise ValueError("policy outputs and phase archives must have distinct paths") - for path in outputs: - if path.is_dir() or any(p.exists() and not p.is_dir() for p in path.parents): - raise ValueError(f"output requires a file destination: {path}") - for other in protected | set(outputs): - if ( - path == other - and other in protected - or path != other - and (path in other.parents or other in path.parents) - ): - raise ValueError( - f"policy output {path} conflicts with input/output {other}" - ) - for directory in archives: - if directory.exists() and not directory.is_dir(): - raise ValueError(f"archive requires a directory: {directory}") - for other in protected | set(outputs) | (set(archives) - {directory}): - if ( - directory == other - or directory in other.parents - or other in directory.parents - ): - raise ValueError( - f"archive {directory} conflicts with input/output {other}" - ) + message = "policy outputs and phase archives must have distinct paths" + outputs = unique_paths(outputs, message) + archives = unique_paths( + [ + Path(phase.archive_directory).resolve() + for phase in config.phases + if getattr(phase, "archive_directory", None) + ], + message, + ) + require_file_destinations(outputs) + validate_path_conflicts(outputs, protected, label="policy output") + require_directory_destinations(archives) + validate_path_conflicts(archives, protected | set(outputs), label="archive") def resolve_normalization(policy, declaration): @@ -167,16 +114,7 @@ def build_actor(config, fitted, statistics): def prepare_data(config, policy, inputs) -> list[PreparedPolicyData | None]: - catalog = PolicyDataCatalog(inputs.sources, policy, config.episode_cache_size) - prepared = {} - result = [] - for phase in config.phases: - entry = BUILTIN_ALGORITHMS[phase.algorithm] - if entry.prepare_data is None: - result.append(None) - continue - result.append(entry.prepare_data(config, phase, policy, catalog, prepared)) - return result + return PreparedTrainingInputs.for_run(config).prepare(config, policy, inputs) def validate_data(data, actor): diff --git a/src/sequifier/rl/prepared_inputs.py b/src/sequifier/rl/prepared_inputs.py new file mode 100644 index 00000000..a621e84e --- /dev/null +++ b/src/sequifier/rl/prepared_inputs.py @@ -0,0 +1,213 @@ +"""Run-owned source identities, episode catalogs and policy partitions.""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from sequifier.config.policy_config import PolicyBinding +from sequifier.config.policy_inputs import PolicyFittingSpec +from sequifier.fingerprints import fingerprint +from sequifier.trajectory.sources import ( + ResolvedSequenceSource, + file_checksum, + resolve_source, +) + +if TYPE_CHECKING: + from sequifier.io.policy_dataset import PolicyDataCatalog + from sequifier.io.prepared_policy_data import PreparedPolicyData + + +@dataclass(frozen=True) +class RunInputs: + sources: dict[str, ResolvedSequenceSource] + checksums: dict[Path, str] + _identity: dict[str, str] = field(init=False, repr=False) + + def __post_init__(self): + object.__setattr__( + self, + "_identity", + {str(path.resolve()): value for path, value in self.checksums.items()}, + ) + + @classmethod + def resolve(cls, config): + return PreparedTrainingInputs.for_run(config).resolve(config) + + @property + def identity(self): + return dict(self._identity) + + def revalidate(self, expected, field): + """Rehash every dependency, including manifests and retargeted links.""" + from sequifier.rl.run_state import require_equal + + require_equal( + expected, + {str(path.resolve()): file_checksum(path) for path in self.checksums}, + field, + ) + + +def _data_policy_contract(policy): + """Episode interpretation and eligibility, independent of fitted feature codecs.""" + binding = policy.binding if isinstance(policy, PolicyFittingSpec) else policy + values = binding.model_dump(mode="json") + return { + "binding": {key: values[key] for key in PolicyBinding.model_fields}, + "context_length": policy.context_length, + "interface_name": policy.interface_name, + "observation_columns": policy.observation_columns, + "static_columns": policy.static_columns, + "action_columns": policy.action_columns, + "input_encoding": policy.input_encoding, + } + + +@dataclass +class _PolicyPartitions: + catalog: "PolicyDataCatalog" + prepared: dict[tuple, "PreparedPolicyData"] = field(default_factory=dict) + + def rebind(self, policy): + self.catalog.rebind(policy) + for data in self.prepared.values(): + data.rebind(policy) + + +class PreparedTrainingInputs: + """Private, in-memory preparation shared from metadata fitting through execution. + + Source interpretation and split settings key reusable partitions. A changed + policy interpretation builds new views; changed codecs refresh existing views + without selecting the partitions again. Callers still revalidate dependencies + before executing a run and at every committed phase boundary. + """ + + def __init__(self): + self._sources = {} + self._checksums = {} + self._inputs = {} + self._partitions = {} + self._preference_manifests = {} + + @classmethod + def for_run(cls, config): + if getattr(config, "_prepared_inputs", None) is None: + config._prepared_inputs = cls() + return config._prepared_inputs + + def _phase_dependencies(self, config, phase, inputs): + if phase.algorithm == "dpo": + from sequifier.trajectory.preferences import load_preferences + + preferences = { + ref.source: config.preferences[ref.source] + for ref in [phase.preferences, *(phase.evaluation.sources or [])] + } + names = set() + for path in preferences.values(): + if path not in self._preference_manifests: + self._preference_manifests[path] = load_preferences(path).pairs + for pair in self._preference_manifests[path]: + names.update((pair.chosen.source, pair.rejected.source)) + if names - inputs.sources.keys(): + raise ValueError( + f"preferences reference unknown sources: {sorted(names - inputs.sources.keys())}" + ) + else: + preferences = {} + names = { + ref.source + for ref in [*phase.sources, *(phase.evaluation.sources or [])] + } + sources = {name: inputs.sources[name] for name in sorted(names)} + checksums = { + str(path): value + for source in sources.values() + for path, value in source.checksums.items() + } + checksums.update( + {path: inputs.checksums[Path(path)] for path in preferences.values()} + ) + return sources, preferences, checksums + + def resolve(self, config): + declarations = { + name: source.model_dump(mode="json") + for name, source in config.sources.items() + } + key = fingerprint({"sources": declarations, "preferences": config.preferences}) + if key not in self._inputs: + sources, checksums = {}, {} + for name, source in config.sources.items(): + source_key = fingerprint(declarations[name]) + if source_key not in self._sources: + self._sources[source_key] = resolve_source(source) + resolved = self._sources[source_key] + sources[name] = resolved + for path, checksum in resolved.checksums.items(): + if path in self._checksums and self._checksums[path] != checksum: + raise ValueError(f"source changed during preparation: {path}") + self._checksums[path] = checksum + checksums[path] = checksum + for path in config.preferences.values(): + path = Path(path) + if path not in self._checksums: + self._checksums[path] = file_checksum(path) + checksums[path] = self._checksums[path] + self._inputs[key] = RunInputs(sources, checksums) + return self._inputs[key] + + def prepare_phase(self, config, phase, policy, inputs, *, fitting=False): + from sequifier.io.policy_dataset import PolicyDataCatalog + from sequifier.rl.offline_data import prepare_offline + from sequifier.rl.registry import BUILTIN_ALGORITHMS + + entry = BUILTIN_ALGORITHMS[phase.algorithm] + prepare = ( + prepare_offline + if fitting and phase.algorithm == "iql" + else entry.prepare_data + ) + if prepare is None: + return None + sources, preferences, checksums = self._phase_dependencies( + config, phase, inputs + ) + key = fingerprint( + { + "sources": { + name: { + "source": source.source.model_dump(mode="json"), + "paths": [str(path) for path in source.paths], + "kind": source.kind, + } + for name, source in sources.items() + }, + "identity": checksums, + "preferences": preferences, + "policy": _data_policy_contract(policy), + "validation_fraction": config.validation_fraction, + "seed": config.seed, + "episode_cache_size": config.episode_cache_size, + } + ) + if key not in self._partitions: + self._partitions[key] = _PolicyPartitions( + PolicyDataCatalog( + sources, + policy, + config.episode_cache_size, + preference_manifests=self._preference_manifests, + ) + ) + partitions = self._partitions[key] + partitions.rebind(policy) + return prepare(config, phase, policy, partitions.catalog, partitions.prepared) + + def prepare(self, config, policy, inputs): + return [ + self.prepare_phase(config, phase, policy, inputs) for phase in config.phases + ] diff --git a/src/sequifier/rl/run_config.py b/src/sequifier/rl/run_config.py index 79fbf589..315aea5a 100644 --- a/src/sequifier/rl/run_config.py +++ b/src/sequifier/rl/run_config.py @@ -1,6 +1,8 @@ """Resolved inputs for the native policy runner; never an authored config.""" -from pydantic import BaseModel, ConfigDict +from typing import Any + +from pydantic import BaseModel, ConfigDict, PrivateAttr from sequifier.config.components import ResumeConfig from sequifier.config.policy_config import ResolvedPolicyBinding @@ -10,6 +12,8 @@ class ResolvedPolicyRun(BaseModel): + _prepared_inputs: Any = PrivateAttr(default=None) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) project_root: str model_name: str diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 23c6095c..cc1d00b0 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -20,6 +20,11 @@ from sequifier.config.train_config import SequifierConfig, resolve_sequifier_config from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save, write_json_atomic +from sequifier.io.output_paths import ( + contains_path, + unique_paths, + validate_path_conflicts, +) from sequifier.trajectory.sources import file_checksum @@ -211,8 +216,10 @@ def run_learning_plan(config) -> LearningRunResult: "committed incoming model artifact changed", ) # Validate model/source/environment contracts for the complete plan first. - from sequifier.rl.preparation import RunInputs, preflight_environments, prepare_data + from sequifier.rl.preparation import preflight_environments, prepare_data + from sequifier.rl.prepared_inputs import PreparedTrainingInputs + prepared_inputs = PreparedTrainingInputs.for_run(config) dependencies = {} for index, phase in enumerate(phases): if phase.algorithm == "supervised": @@ -235,7 +242,8 @@ def run_learning_plan(config) -> LearningRunResult: destination, False, ) - inputs = RunInputs.resolve(run) + inputs = prepared_inputs.resolve(run) + inputs.revalidate(inputs.identity, f"phases.{phase.name}.source_identity") dependencies.update(inputs.identity) preflight_environments(run) prepare_data(run, run.policy, inputs) @@ -247,20 +255,19 @@ def run_learning_plan(config) -> LearningRunResult: protected = {Path(path).resolve() for path in dependencies} if config.initialize_from: protected.add(Path(config.initialize_from).resolve()) - outputs = [ - checkpoint, - Path(config.model_path), - Path(config.report_path), - ] - if len(set(outputs)) != len(outputs): - raise ValueError("learning run output paths must be distinct") - for output in outputs: - if output in protected or any( - output in source.parents or source in output.parents for source in protected - ): - raise ValueError(f"learning output conflicts with an input: {output}") - directory = Path(config.phase_directory) - if any(directory == source or directory in source.parents for source in protected): + outputs = unique_paths( + [ + checkpoint, + Path(config.model_path), + Path(config.report_path), + ], + "learning run output paths must be distinct", + ) + validate_path_conflicts( + outputs, protected, label="learning output", check_each_other=False + ) + directory = Path(config.phase_directory).resolve() + if any(contains_path(directory, source) for source in protected): raise ValueError("phase_directory overlaps a learning input") for index in range(len(state["completed"]), len(phases)): phase = phases[index] diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 560be0bb..9ac97fce 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from pathlib import Path -from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.policy_inputs import PolicyEpisodeSpec from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.sequence import SequenceEpisode, TransitionColumns @@ -14,7 +14,7 @@ def sequence_from_trajectory( - trajectory: Trajectory, policy: ResolvedPolicyBinding, action_semantics="policy" + trajectory: Trajectory, policy: PolicyEpisodeSpec, action_semantics="policy" ) -> SequenceEpisode: episode = validate_trajectory(trajectory) if ( @@ -68,7 +68,7 @@ def sequence_from_trajectory( @dataclass(frozen=True) class TrajectoryReference: path: Path - policy: ResolvedPolicyBinding + policy: PolicyEpisodeSpec action_semantics: str = "policy" namespace: str = "" @@ -82,7 +82,7 @@ def load(self) -> SequenceEpisode: def source_episodes( - source: SequenceSourceConfig | ResolvedSequenceSource, policy: ResolvedPolicyBinding + source: SequenceSourceConfig | ResolvedSequenceSource, policy: PolicyEpisodeSpec ): resolved: ResolvedSequenceSource = ( resolve_source(source) if isinstance(source, SequenceSourceConfig) else source diff --git a/src/sequifier/trajectory/tables.py b/src/sequifier/trajectory/tables.py index 68a0406f..5c18aece 100644 --- a/src/sequifier/trajectory/tables.py +++ b/src/sequifier/trajectory/tables.py @@ -8,7 +8,7 @@ import polars as pl from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata -from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.policy_inputs import PolicyEpisodeSpec from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.io.images import ImageStore from sequifier.io.preprocessing import denormalize_values @@ -142,7 +142,7 @@ def decode(value, column): @dataclass(frozen=True) class TableEpisodeReference: source: SequenceSourceConfig - policy: ResolvedPolicyBinding + policy: PolicyEpisodeSpec sequence_id: str | int table: IndexedTable metadata: DatasetMetadata | None = None From d5d2d423584542208bd5f45821dcf10317bf8c1a Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 11 Nov 2026 11:06:51 +0100 Subject: [PATCH 28/71] config improvements --- documentation/configs/infer.md | 11 +- documentation/configs/train.md | 105 ++++-- documentation/policy-learning.md | 19 +- documentation/rollout.md | 4 +- .../unreleased-configuration-ownership.md | 20 + release-notes/unreleased-phase-ownership.md | 18 +- src/sequifier/artifacts/model_config.py | 70 ++++ src/sequifier/config/inference_common.py | 15 +- src/sequifier/config/policy_config.py | 45 +-- src/sequifier/config/policy_infer_config.py | 19 +- src/sequifier/config/policy_optimization.py | 66 ++-- src/sequifier/config/policy_plan.py | 44 --- src/sequifier/config/policy_train_config.py | 8 +- src/sequifier/config/prediction_config.py | 31 ++ src/sequifier/config/rollout_config.py | 17 - src/sequifier/config/train_config.py | 351 ++++++++---------- src/sequifier/config/training_resolution.py | 21 +- src/sequifier/infer.py | 2 +- src/sequifier/model/execution.py | 2 - src/sequifier/model/factory.py | 7 +- .../model/prediction_distribution.py | 53 ++- src/sequifier/rl/attachment.py | 24 +- src/sequifier/rl/checkpoint.py | 5 + src/sequifier/rl/inference.py | 2 +- src/sequifier/rl/joint_distribution.py | 4 +- src/sequifier/rl/learner.py | 14 +- src/sequifier/rl/policy.py | 17 +- src/sequifier/rl/preparation.py | 2 +- src/sequifier/rl/references.py | 1 + src/sequifier/rl/run_state.py | 7 +- src/sequifier/runtime/builder.py | 4 +- src/sequifier/sequifier.py | 9 +- src/sequifier/training/checkpoint_service.py | 10 +- src/sequifier/training/coordinator.py | 12 +- src/sequifier/training/engine.py | 33 +- src/sequifier/training/loss.py | 4 +- src/sequifier/training/scheduler_budgets.py | 10 +- src/sequifier/trajectory/export.py | 4 +- src/sequifier/trajectory/tables.py | 32 +- 39 files changed, 631 insertions(+), 491 deletions(-) create mode 100644 release-notes/unreleased-configuration-ownership.md create mode 100644 src/sequifier/config/prediction_config.py diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index 41eaf4fc..17544b00 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -273,8 +273,8 @@ With `model_type: generative`, the selected artifact interface determines the default task. `task: policy` requires one policy-bound portable PT artifact; `task: forecast` requests ordinary predictions, including from a policy-bound interface with a supported forecast contract. Select `model_interface` when -ambiguous. Legacy `model_type: policy` forces policy execution and conflicts with -`task: forecast`. Embedding rejects `task`. The CLI accepts `--task policy|forecast`. +ambiguous. `model_type` accepts only `generative` or `embedding`; use +`task: policy` for explicit policy execution. Embedding rejects `task`. The CLI accepts `--task policy|forecast`. Policy input is raw CSV/Parquet observation rows. History alignment, codecs and normalization come from the artifact. Common fields cover model/interface/data @@ -304,10 +304,9 @@ Policy rollout declares `environment: {factory: module:factory, config: {}, implementation_id: ..., units: {}}`, required `max_steps`, `seed_row_index` and `trajectory_path`, plus optional `max_episode_steps`. It rejects forecast `bindings`. Forecast rollout uses the distinct version-1 environment, requires bindings and -selects `seed_window_index`. Neither accepts `rollout.type`; legacy `seed_index` -aliases the route-specific index, but combining spellings fails. Both indices are -zero-based and validated before environment creation. Trajectory metadata retains -its existing `seed_index` key. See the [rollout guide](../rollout.md) and +selects `seed_window_index`. Neither accepts `rollout.type` or `seed_index`. +Both indices are zero-based and validated before environment creation. Trajectory +metadata uses the same route-specific index name. See the [rollout guide](../rollout.md) and [policy template](../../src/sequifier/templates/infer-policy.yaml). Execution determinism, action selection and seed are independent. A seeded diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 24123a84..b25e8598 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -76,7 +76,7 @@ training_plan: optimization: model: learning_rate: 0.0001 -evaluation: true + evaluation: true ``` The concise form expands before validation: @@ -86,7 +86,7 @@ The concise form expands before validation: | `model.interface` | `model.interfaces.default` | | `dataset.part` | `dataset_training.default.parts.default` | | `training_plan.epochs` | One sequential phase named `train` | -| `evaluation: true` | Evaluate the inferred single source | +| Phase `evaluation: true` | Evaluate the phase's inferred single source | These shortcuts can be used independently. A unique interface and training source are inferred; multiple interfaces or datasets require explicit names and @@ -136,7 +136,6 @@ global_training: target_offset: 1 window_stride: 1 inference_batch_size: 256 - save_interval_epochs: 1 model: backbone: architecture: @@ -213,6 +212,8 @@ training_plan: scheduler_step_on: epoch gradient_clip: 1.0 optimizer_state: reset + evaluation: + sources: [{source: events}] - name: complete_retraining epochs: 5 mode: interleaved @@ -235,9 +236,8 @@ training_plan: scheduler_step_on: epoch gradient_clip: 1.0 optimizer_state: reset -evaluation: - sources: - - source: events + evaluation: + sources: [{source: events}] export_generative_model: true export_embedding_model: false export_onnx: true @@ -259,9 +259,9 @@ computes loss over all valid decoded positions, regardless of ## Ownership and resolution -- `global_training` owns objective, window, optimizer, precision, - distribution, compilation, checkpoint, and data-loader behavior. Phase - entries own `epochs`. +- `global_training` owns objective, window, precision, distribution, + compilation, recovery checkpoint, and data-loader behavior. Phases own + budgets, batching, optimization, evaluation and epoch/transaction checkpoint cadence. - `model` contains exactly one backbone and one or more named interfaces. Different interface names create distinct ingestion and decoder weights; repeated references to one name share those weights. @@ -462,18 +462,71 @@ Momentum cycling defaults to off for both cycle schedulers in plan mode; explicit `cycle_momentum: true` is unsupported. `base_learning_rate` is only valid with `CyclicLR`. -## Single-phase compatibility +## Canonical training settings + +Batching and optimization are configured only on phases. Global aliases for +`batch_size`, `accumulation_steps`, `learning_rate`, `optimizer`, `scheduler`, +`scheduler_step_on`, and `gradient_clip` are rejected. CLI overrides use +`--global-training` and `--training-plan` YAML mappings with the same schema. + +Every phase owns `evaluation`, `evaluation_interval`, and `checkpoint_interval`. +For supervised training, intervals count completed phase epochs; for policy +training, they count algorithm transactions. The final epoch/update always +checks evaluation and saves a checkpoint. Supervised `evaluation` is optional; +`evaluation: true` selects the phase's unique training source. Multiple sources +require an explicit monitor for best handoff, validation saving or early stopping. +There is no top-level `evaluation` declaration. + +```yaml +training_plan: + phases: + - name: fit + algorithm: supervised + epochs: 10 + mode: sequential + sources: [{source: observations.train}] + evaluation: + sources: [{source: observations.validation}] + monitor: {source: observations.validation, metric: loss, mode: min} + evaluation_interval: 1 + checkpoint_interval: 2 +``` + +Epoch-based early stopping requires `evaluation_interval: 1`. +Time- and batch-based recovery checkpoints remain global execution settings. +Frozen normalization belongs to `global_training.normalization` or the incoming +artifact; phases cannot redeclare it. + +## Model prediction parameters and policy actions + +Model interfaces own prediction parameters. Policy bindings select targets as +actions and define their support, transforms, conditional activity and scale use. +They never create, overwrite or remove model parameter declarations. + +```yaml +model: + interfaces: + actor: + # Other interface fields omitted. + prediction_distributions: + throttle: + scale: {initial: 0.2, trainable: true, min: 0.0001, max: 100} +policy_bindings: + actor: + actions: + throttle: {type: real, scale: model, lower: 0, upper: 1} +``` + +A real action must explicitly choose `scale: model` or `scale: {fixed: 0.1}`. +The latter bypasses the model scale for that policy without deleting its learned +parameter. A fixed policy scale does not require a model distribution entry. +Categorical actions declare `values` and cannot configure real scales or bounds. +`GaussianNLLLoss` uses the model's Gaussian distribution and its scale; policy +bounds, point masses and activity conditions apply only to policy likelihoods. -For one phase, `global_training.batch_size` and `accumulation_steps` move onto -that phase; `learning_rate`, `optimizer`, `scheduler`, `scheduler_step_on`, and -`gradient_clip` move to `optimization.model` for supervised training only. -Multi-phase plans reject these fields, and duplicate global/local declarations -fail even when equal. Policy optimization must select its role explicitly. -Legacy supervised defaults retain no gradient clipping, require a single -optimizer's learning rate, and resolve null accumulation to 1. Explicit modern -roles use the defaults below. Composition, CLI and search normalize identically; -fingerprints use resolved phases. CLI flags use the hyphenated field names; -`--optimizer` and `--scheduler` accept YAML mappings. +When initializing from an artifact, a full `model` declaration may add prediction +distribution entries. The architecture and existing distribution declarations +must match; only newly declared scale parameters receive fresh initialization. ## Optimization across phases @@ -520,7 +573,7 @@ best weights before starting the next phase; its monitor state is phase-local. Supervised `scheduler_step_on: epoch` advances once after a completed phase epoch; ReduceLROnPlateau receives the configured evaluation monitor at that boundary -and requires `evaluation.monitor`. `batch` advances once per applied optimizer +and requires the phase's `evaluation.monitor` and `evaluation_interval: 1`. `batch` advances once per applied optimizer update, after accumulation, rather than once per microbatch. Policy roles only accept `scheduler_step_on: optimizer_step`, their default. Policy plateau and compound schedulers are unsupported. Constructor arguments are validated before @@ -528,13 +581,14 @@ training; the shared optimizer shape does not imply shared scheduler capabilitie For epoch OneCycleLR, the resolved `total_steps` is the phase's epoch count, or the sum of epochs in its contiguous continuation chain. An explicit value must -match. Batch OneCycleLR requires an explicit `total_steps`. Preflight counts loader +match. Batch OneCycleLR derives `total_steps`. An explicit value is an assertion. Preflight counts loader batches using the same sequential/interleaved source scheduler, phase seeds and accumulation rules as training, including partial updates flushed at dataset boundaries. The budget sums over a contiguous continuation chain. This calculation does not consume samples or advance loader RNG. Early stopping and explicitly skipped updates may consume less than the authored maximum. -Policy OneCycleLR requires the exact maximum authored optimizer-update budget: +Policy OneCycleLR derives the maximum optimizer-update budget below. An explicit +`total_steps` must match it: | Algorithm | Optimizer updates per role | | --- | --- | @@ -572,13 +626,14 @@ With several evaluation sources, configure an explicit monitor when validation-based saving or early stopping is enabled: ```yaml +# Inside a training_plan.phases entry: evaluation: sources: [{source: events}, {source: telemetry.main}] monitor: {source: events, metric: loss, mode: min} ``` -Checkpoint interval fields retain distinct behavior. `save_interval_epochs`, -`save_interval_batches`, and `save_interval_minutes` create persistent +Checkpoint interval fields retain distinct behavior. Phase `checkpoint_interval`, +global `save_interval_batches`, and global `save_interval_minutes` create persistent epoch/batch snapshots and refresh the rolling `latest` checkpoint; `save_latest_interval_minutes` refreshes only `latest`. `save_interval_val_loss` controls whether timed or batch snapshots also run diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 0db92ea7..5a4567aa 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -27,8 +27,8 @@ fraction, episode cache size, determinism and resume. Phases own batching, Each role defaults to Adam, learning rate `0.0003`, gradient clip `1.0`, and `ConstantLR` with factor `1.0`, total iterations `1`. Optimizers accept constructor arguments; grouped optimizer plans set their own rates. Policy schedulers advance -on optimizer steps; plateau/compound schedulers are unsupported. `OneCycleLR.total_steps` -must match the maximum phase budget. Unknown roles fail validation. +on optimizer steps; plateau/compound schedulers are unsupported. `OneCycleLR.total_steps` is derived from the phase budget; +an explicitly supplied value must match that budget. Unknown roles fail validation. Imitation, DPO and GRPO default to `batch_size: 32`, `accumulation_steps: 1`, `evaluation_interval: 1` and `checkpoint_interval: 1`. Accumulation uses actual @@ -45,7 +45,9 @@ manifests and prepared histories. At row `t`, observations and lagged actions fo inputs; the recorded current action is the label, and reward/successor describe its resulting transition. `columns` maps observations, static inputs, actions, previous actions, reward, successors, boundary flags, group and episode IDs. -Singular `action`/`previous_action` conflict with nonempty plural maps. +Use `actions`/`previous_actions` for named scalar column mappings. Structured +columns use `action_record`/`previous_action_record`; record and scalar forms +cannot be combined. Fragments require an explicit preceding action; file boundaries alone imply no termination. Forecast trajectories require `action_semantics: executed_predictions`. @@ -61,9 +63,9 @@ Imitation minimizes joint action NLL. Return conditioning requires rewards and a known tail: true termination or an explicit finite return horizon. The label at `t` includes `r_t`; unknown tails are excluded. Offline monitors are fixed to `{metric: validation_action_nll, mode: min}` for imitation/IQL and -`{metric: validation_dpo_loss, mode: min}` for DPO. Legacy `validation_sources` -(DPO: `validation_preferences`), `monitor` and `direction` normalize to `evaluation`; -combining old and new spellings fails. +`{metric: validation_dpo_loss, mode: min}` for DPO. Only +`evaluation.sources` and `evaluation.monitor` are accepted; alternate spellings +are rejected. IQL requires complete reward, terminated, truncated and bootstrap columns, plus true successors for bootstrapped transitions. It samples uniformly with replacement, @@ -176,3 +178,8 @@ For prediction, task routing, probability outputs and policy rollout, use the [inference reference](configs/infer.md#policy-model-interfaces) and [rollout guide](rollout.md). Deterministic execution, action selection and RNG seed are independent settings; reproducibility also depends on runtime/device/order. + +Model scale parameters and policy action behavior use separate schemas; see +[model prediction parameters and policy actions](configs/train.md#model-prediction-parameters-and-policy-actions). +Normalization is global or inherited from the artifact and cannot be configured +on an individual phase. diff --git a/documentation/rollout.md b/documentation/rollout.md index 836650d4..3cf2a1e8 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -49,8 +49,8 @@ codecs and normalization come from its artifact. See [Python example](example_inputs/ppo_toy_control.py). Both seed indices are required, zero-based, and checked before environment creation. -Legacy `seed_index` aliases the route-specific field; supplying both names fails. -Trajectory metadata retains its public `seed_index` key. No `rollout.type` is accepted. +Only the route-specific seed index name is accepted and recorded in trajectory +metadata. Neither `seed_index` nor `rollout.type` is accepted. ## Boundaries and trajectory API diff --git a/release-notes/unreleased-configuration-ownership.md b/release-notes/unreleased-configuration-ownership.md new file mode 100644 index 00000000..ac25aa8e --- /dev/null +++ b/release-notes/unreleased-configuration-ownership.md @@ -0,0 +1,20 @@ +# Configuration ownership + +- Give every algorithm phase its own evaluation declaration, monitor and cadence. + Supervised phases also own epoch checkpoint cadence through `checkpoint_interval`. +- Separate model-owned Gaussian scales (`initial`, `trainable`, `min`, `max`) from + policy action behavior. Real actions select `scale: model` or `scale: {fixed: ...}`; + action bounds, point masses and conditional activity remain task declarations. +- Preserve model scale parameters when a policy uses fixed exploration. Explicit + model declarations may add new scale parameters to an initialized artifact. +- Keep frozen normalization global or inherited from an artifact; remove phase copies. +- Derive OneCycleLR budgets for supervised and policy training. Explicit `total_steps` + values are assertions checked against the derived budget. +- Remove global optimization aliases, policy inference/evaluation aliases and rollout + seed aliases. CLI overrides use canonical `--global-training`/`--training-plan` + mappings. Scalar action mappings use `actions`/`previous_actions`; structured + records use `action_record`/`previous_action_record`. + +No configuration migration or compatibility layer is provided. Existing configuration +files and tests are intentionally left unchanged by this refactor. No tests were +written or run for this change. diff --git a/release-notes/unreleased-phase-ownership.md b/release-notes/unreleased-phase-ownership.md index aaf64f2e..3173df00 100644 --- a/release-notes/unreleased-phase-ownership.md +++ b/release-notes/unreleased-phase-ownership.md @@ -1,8 +1,8 @@ # Phase-owned training and explicit inference routes - Move batching and role-keyed optimization into each training phase. Supervised - uses `optimization.model`; policy roles remain algorithm-specific. Normalize - legacy single-phase global fields and remove the global reset switch. + uses `optimization.model`; policy roles remain algorithm-specific. Reject + legacy global optimization fields and remove the global reset switch. - Add phase `optimizer_state: reset | continue`, with reset as the default and validated compatible native continuation. Reject mixed-plan continuation and policy OneCycle continuation. Preserve supervised epoch, update and metric @@ -21,10 +21,10 @@ CPU smoke tests cover native supervised and all policy algorithms, mixed handoff/recovery, inference, rollout and configuration rejection. -Legacy single-phase global fields normalize to phase-local settings. Offline -policy evaluation uses `evaluation.sources` and `evaluation.monitor: {metric, -mode}`, with old fields accepted as aliases. Generative inference dispatches -from the selected interface, with optional `task` and legacy `model_type: policy`. -Rollout indices are `seed_window_index` (forecast) and `seed_row_index` (policy); -`seed_index` remains an input alias and the trajectory metadata key. Populated -singular/plural action mappings conflict; empty plural maps remain unused. +All algorithms own evaluation on their phase. Offline policy evaluation uses +`evaluation.sources` and `evaluation.monitor: {metric, mode}` without aliases. +Generative inference dispatches from the selected interface, with optional `task`; +`model_type` is `generative` or `embedding`. Rollout indices are `seed_window_index` +(forecast) and `seed_row_index` (policy), including trajectory metadata. +Scalar action mappings use `actions`/`previous_actions`; structured records use +`action_record`/`previous_action_record`. diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 7caa996c..223c1ee2 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -43,6 +43,17 @@ def attention(values): for name, interface in source["interfaces"].items(): if name not in current["interfaces"]: raise ValueError(f"initial model interface {name!r} was dropped") + for target, distribution in interface.get( + "prediction_distributions", {} + ).items(): + if to_jsonable_python(distribution) != to_jsonable_python( + current["interfaces"][name] + .get("prediction_distributions", {}) + .get(target) + ): + raise ValueError( + f"initial model prediction parameters differ at {name}.{target}" + ) for key in ( "input_columns", "target_columns", @@ -209,3 +220,62 @@ def execution_config(model, interfaces, global_training, *, device="cpu", **opti return ModelExecution(model, interfaces, global_training).training_config( device=device, **options ) + + +def extend_prediction_distributions(execution, declared_model): + """Allow explicit new scale parameters while preserving an initialized model.""" + exclude = {"interfaces": {"__all__": {"prediction_distributions"}}} + if declared_model.model_dump(exclude=exclude) != execution.model.model_dump( + exclude=exclude + ): + raise ValueError( + "declared model architecture differs from initialize_from model" + ) + for name, declared in declared_model.interfaces.items(): + interface = execution.interface(name) + distributions = dict(interface.prediction_distributions) + for target, field in declared.prediction_distributions.items(): + if target in distributions and distributions[target] != field: + raise ValueError( + f"{name}.{target}: declared prediction parameters differ from initialize_from" + ) + if ( + target not in interface.target_columns + or interface.target_column_types.get(target) != "real" + ): + raise ValueError( + f"{name}.{target}: prediction_distributions requires a real model target" + ) + distributions[target] = field + interface.prediction_distributions = distributions + execution.model.interfaces[name].prediction_distributions = dict(distributions) + return execution + + +def load_attached_model_weights(network, artifact): + """Restore source weights, initializing only newly declared model scale parameters.""" + from sequifier.model.prediction_distribution import PredictionScales + + expected = network.state_dict() + source = artifact.model_state_dict + missing = expected.keys() - source.keys() + allowed = set() + for name, route in network.interfaces.items(): + scales = getattr(route, "prediction_scales", None) + if not isinstance(scales, PredictionScales): + continue + original = artifact.model_config.values["interfaces"][name].get( + "prediction_distributions", {} + ) + allowed.update( + f"interfaces.{name}.prediction_scales.log_scales.{target}" + for target in scales.log_scales + if target not in original + ) + if missing - allowed: + raise ValueError( + f"initial model is missing weights: {sorted(missing - allowed)}" + ) + network.load_state_dict( + {**{key: expected[key] for key in missing}, **source}, strict=True + ) diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index 9ce3b4cd..01fb37f9 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -26,7 +26,7 @@ class InferenceRouteConfig(BaseModel): model_config = ConfigDict(extra="ignore") project_root: str = "." model_path: str | list[str] - model_type: Literal["generative", "embedding", "policy"] + model_type: Literal["generative", "embedding"] model_interface: str | None = None task: Literal["policy", "forecast"] | None = None @@ -47,11 +47,6 @@ def validate_interface(cls, value): @model_validator(mode="after") def validate_task(self): - if self.model_type == "policy" and self.task == "forecast": - raise ValueError( - "model_type: policy conflicts with task: forecast; " - "use model_type: generative" - ) if self.model_type == "embedding" and self.task is not None: raise ValueError("task is only supported for generative inference") return self @@ -64,9 +59,7 @@ class SelectedInferenceModel: interface_name: str @classmethod - def select( - cls, path, values, *, interface_name=None, model_type="generative", task=None - ): + def select(cls, path, values, *, interface_name=None, task=None): interfaces = values.get("interfaces") if not isinstance(interfaces, dict) or not interfaces: raise ValueError("model_config.interfaces must be a non-empty mapping") @@ -76,9 +69,7 @@ def select( for name, interface in interfaces.items() if interface.get("policy") is not None ] - if model_type == "policy" and len(policies) == 1: - interface_name = policies[0] - elif len(interfaces) > 1 and policies and task != "forecast": + if len(interfaces) > 1 and policies and task != "forecast": raise ValueError( "Artifact has multiple interfaces; select model_interface before choosing task" ) diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index be5554d8..1716a87f 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -20,28 +20,40 @@ class ReturnDefinition(PolicyConfigBase): finite_horizon: int | None = Field(default=None, gt=0) +class FixedActionScale(PolicyConfigBase): + fixed: float = Field(gt=0) + + class ActionFieldConfig(PolicyConfigBase): - """Distribution and domain of one named model target.""" + """Task-specific action support and selection of model or fixed scale.""" type: Literal["categorical", "real"] values: list[DomainValue] = Field(default_factory=list) - scale: float | None = Field(default=None, gt=0, allow_inf_nan=False) - learn_scale: bool = False - min_scale: float = Field(default=1e-4, gt=0) - max_scale: float = Field(default=100.0, gt=0) - lower: float | None = Field(default=None, allow_inf_nan=False) - upper: float | None = Field(default=None, allow_inf_nan=False) + scale: Literal["model"] | FixedActionScale | None = None + lower: float | None = None + upper: float | None = None active_when: dict[str, list[DomainValue]] = Field(default_factory=dict) inactive_value: DomainValue = 0.0 - # Fixed point masses allow exact endpoints alongside a continuous density. atoms: dict[float, float] = Field(default_factory=dict) @model_validator(mode="after") def validate_distribution(self): - if self.type == "real" and (self.scale is None or self.values): - raise ValueError("real actions require an explicit scale and no values") - if self.type == "categorical" and (self.scale is not None or self.learn_scale): - raise ValueError("categorical actions cannot configure a real scale") + if self.type == "real": + if self.values: + raise ValueError("real actions cannot declare categorical values") + if self.scale is None: + raise ValueError( + "real actions require scale: model or scale: {fixed: ...}" + ) + elif ( + self.scale is not None + or self.lower is not None + or self.upper is not None + or self.atoms + ): + raise ValueError( + "categorical actions cannot configure real scale, bounds, or atoms" + ) if len(set(self.values)) != len(self.values): raise ValueError("action labels must be unique") if ( @@ -50,15 +62,6 @@ def validate_distribution(self): and self.lower >= self.upper ): raise ValueError("action lower bound must be below upper bound") - if self.min_scale > self.max_scale: - raise ValueError("min_scale must not exceed max_scale") - if ( - self.scale is not None - and not self.min_scale <= self.scale <= self.max_scale - ): - raise ValueError("scale must lie between min_scale and max_scale") - if self.atoms and self.type != "real": - raise ValueError("point-mass mixtures are real action distributions") if ( any( not math.isfinite(value) diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 829ee736..7973c21c 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -13,7 +13,6 @@ ) from sequifier.config.policy_config import PolicyConfigBase from sequifier.config.policy_plan import OnlineEnvironmentConfig -from sequifier.config.rollout_config import normalize_seed_index from sequifier.helpers import normalize_path @@ -24,11 +23,6 @@ class PolicyRolloutConfig(PolicyConfigBase): seed_row_index: int = Field(ge=0) max_episode_steps: int | None = Field(default=None, gt=0) - @model_validator(mode="before") - @classmethod - def normalize_index(cls, values): - return normalize_seed_index(values, "seed_row_index") - class PolicyInferenceConfig(InferenceExecutionConfig): model_path: str @@ -58,16 +52,6 @@ class PolicyInferenceConfig(InferenceExecutionConfig): prediction_length: int | None = Field(default=None, gt=0) decoder_support: int | None = Field(default=None, gt=0) - @model_validator(mode="before") - @classmethod - def normalize_policy_alias(cls, values): - if isinstance(values, dict) and values.get("model_type") == "policy": - if values.get("task") == "forecast": - raise ValueError("model_type: policy conflicts with task: forecast") - values = dict(values) - values["model_type"] = "generative" - return values - @model_validator(mode="after") def validate_execution(self): if self.read_format not in {"csv", "parquet"} or self.write_format not in { @@ -108,11 +92,10 @@ def _policy_inference_route( path, model_config, interface_name=route.model_interface, - model_type=route.model_type, task=route.task, ) bindings.append(selection.policy_bound) - task = "policy" if route.model_type == "policy" else route.task + task = route.task if task is None: if any(bindings) and not all(bindings): raise ValueError( diff --git a/src/sequifier/config/policy_optimization.py b/src/sequifier/config/policy_optimization.py index 1532e925..c40726ea 100644 --- a/src/sequifier/config/policy_optimization.py +++ b/src/sequifier/config/policy_optimization.py @@ -74,7 +74,46 @@ def validate_components(self): raise ValueError(str(error)) from error return self - def build(self, module): + def continuation_settings(self): + """Compare optimizer settings independently of optional budget assertions.""" + values = self.model_dump(mode="python") + if self.scheduler.name == "OneCycleLR": + values["scheduler"].pop("total_steps", None) + return values + + def validate_budget(self, step_budget): + if ( + step_budget is not None + and self.scheduler.name == "OneCycleLR" + and self.scheduler.arguments.get("total_steps") is not None + and self.scheduler.arguments["total_steps"] != step_budget + ): + raise ValueError( + f"OneCycleLR.total_steps must equal the maximum authored optimizer-step budget ({step_budget})" + ) + + def resolve_budget(self, step_budget): + if self.scheduler.name != "OneCycleLR": + return self + if step_budget is None or step_budget <= 0: + raise ValueError( + "OneCycleLR requires a positive resolved optimizer-step budget" + ) + self.validate_budget(step_budget) + return self.model_copy( + update={ + "scheduler": self.scheduler.model_copy( + update={ + "arguments": { + **self.scheduler.arguments, + "total_steps": step_budget, + } + } + ) + } + ) + + def build(self, module, *, step_budget=None): from sequifier.model.parameter_catalog import ParameterCatalog from sequifier.training.optimization import create_optimizer @@ -86,10 +125,13 @@ def build(self, module): if isinstance(self.optimizer, OptimizerPlan) else None, ) + scheduler = self.scheduler + if scheduler.name == "OneCycleLR": + scheduler = self.resolve_budget(step_budget).scheduler arguments = ( - resolve_plan_scheduler_arguments(self.optimizer, self.scheduler) + resolve_plan_scheduler_arguments(self.optimizer, scheduler) if isinstance(self.optimizer, OptimizerPlan) - else self.scheduler.arguments + else scheduler.arguments ) return optimizer, get_scheduler_class(self.scheduler.name)( optimizer, **arguments @@ -101,16 +143,6 @@ class OptimizationConfig(RoleOptimizationConfig): scheduler_step_on: Literal["optimizer_step"] = "optimizer_step" - def validate_budget(self, step_budget): - if ( - step_budget is not None - and self.scheduler.name == "OneCycleLR" - and self.scheduler.arguments.get("total_steps") != step_budget - ): - raise ValueError( - f"OneCycleLR.total_steps must equal the maximum authored optimizer-step budget ({step_budget})" - ) - @model_validator(mode="after") def validate_policy_scheduler(self): if ( @@ -143,12 +175,4 @@ def validate_supervised_scheduler(self): raise ValueError("ReduceLROnPlateau requires scheduler_step_on: epoch") if self.scheduler.name in {"SequentialLR", "ChainedScheduler"}: raise ValueError("nested schedulers are not supported") - if ( - self.scheduler.name == "OneCycleLR" - and self.scheduler_step_on == "batch" - and "total_steps" not in self.scheduler.arguments - ): - raise ValueError( - "batch OneCycleLR requires an explicit total_steps optimizer-update budget" - ) return self diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 2cb06945..8475c2d9 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -46,48 +46,6 @@ class PhaseConfig(BaseModel): evaluation_interval: int = Field(default=1, gt=0) checkpoint_interval: int = Field(default=1, gt=0) - @model_validator(mode="before") - @classmethod - def normalize_evaluation(cls, values): - if not isinstance(values, dict): - return values - source_field = ( - "validation_preferences" - if "preferences" in cls.model_fields - else "validation_sources" - if "sources" in cls.model_fields - else None - ) - legacy = { - "validation_sources", - "validation_preferences", - "monitor", - "direction", - } & values.keys() - if "evaluation" in values and legacy: - raise ValueError( - f"Cannot combine {sorted(legacy)} with evaluation; " - "use evaluation.sources and evaluation.monitor.{metric,mode}" - ) - if not legacy: - return values - values = dict(values) - evaluation = {} - if source_field in values: - source = values.pop(source_field) - evaluation["sources"] = ( - [source] - if source_field == "validation_preferences" and source is not None - else source - ) - evaluation["monitor"] = { - new: values.pop(old) - for old, new in (("monitor", "metric"), ("direction", "mode")) - if old in values - } - values["evaluation"] = evaluation - return values - def optimizer_step_budget(self, training_size=None): raise NotImplementedError("built-in phase must declare its optimizer budget") @@ -229,7 +187,6 @@ class PPOPhaseConfig(PhaseConfig): name: str = Field(default="online_ppo", min_length=1) environment: OnlineEnvironmentConfig evaluation: OnlineEvaluationConfig - normalization: FrozenNormalizationConfig | None = None seed_policy: Literal["derived_per_reset"] = "derived_per_reset" total_env_steps: int = Field(gt=0) rollout_length: int = Field(gt=0) @@ -306,7 +263,6 @@ class GRPOPhaseConfig(BatchedPhaseConfig): algorithm: Literal["grpo"] environment: OnlineEnvironmentConfig evaluation: GRPOEvaluationConfig - normalization: FrozenNormalizationConfig | None = None reference: ReferenceConfig = Field(default_factory=ReferenceConfig) total_groups: int = Field(gt=0) groups_per_update: int = Field(default=1, gt=0) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index a477c1e6..9bea2e04 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -15,13 +15,13 @@ class PolicyTableColumns(PolicyConfigBase): item_position: NonemptyStr = "itemPosition" observations: dict[str, NonemptyStr] = Field(default_factory=dict) static_inputs: dict[str, NonemptyStr] = Field(default_factory=dict) - action: NonemptyStr | None = None + action_record: NonemptyStr | None = None actions: dict[str, NonemptyStr] = Field(default_factory=dict) previous_actions: dict[str, NonemptyStr] = Field(default_factory=dict) reward: NonemptyStr | None = "reward" terminated: NonemptyStr | None = "terminated" truncated: NonemptyStr | None = "truncated" - previous_action: NonemptyStr | None = None + previous_action_record: NonemptyStr | None = None successors: dict[str, NonemptyStr] = Field(default_factory=dict) bootstrap_allowed: NonemptyStr | None = None group: NonemptyStr | None = None @@ -30,8 +30,8 @@ class PolicyTableColumns(PolicyConfigBase): @model_validator(mode="after") def validate_action_mappings(self): for singular, plural in ( - ("action", "actions"), - ("previous_action", "previous_actions"), + ("action_record", "actions"), + ("previous_action_record", "previous_actions"), ): if getattr(self, singular) is not None and getattr(self, plural): raise ValueError( diff --git a/src/sequifier/config/prediction_config.py b/src/sequifier/config/prediction_config.py new file mode 100644 index 00000000..849aa75a --- /dev/null +++ b/src/sequifier/config/prediction_config.py @@ -0,0 +1,31 @@ +"""Model-owned prediction parameters, independent of task bindings.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class PredictionConfigBase(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + +class ModelScaleConfig(PredictionConfigBase): + """Model-owned Gaussian scale, including optional learned parameters.""" + + initial: float = Field(gt=0) + trainable: bool = False + min: float = Field(default=1e-4, gt=0) + max: float = Field(default=100.0, gt=0) + + @model_validator(mode="after") + def validate_range(self): + if not self.min <= self.initial <= self.max: + raise ValueError("scale.initial must lie between scale.min and scale.max") + return self + + +class PredictionDistributionConfig(PredictionConfigBase): + """A model output's Gaussian parameterization, independent of policy behavior.""" + + type: Literal["real"] = "real" + scale: ModelScaleConfig diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index 0e38a635..9d86e4d2 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -5,18 +5,6 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -def normalize_seed_index(values, canonical): - """Keep the legacy spelling route-local and out of serialized config.""" - if isinstance(values, dict) and "seed_index" in values: - if canonical in values: - raise ValueError( - f"rollout.seed_index and rollout.{canonical} cannot both be declared" - ) - values = dict(values) - values[canonical] = values.pop("seed_index") - return values - - class InputBinding(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) @@ -57,11 +45,6 @@ class RolloutConfig(BaseModel): seed_window_index: int = Field(ge=0) max_episode_steps: int | None = Field(default=None, gt=0) - @model_validator(mode="before") - @classmethod - def normalize_index(cls, values): - return normalize_seed_index(values, "seed_window_index") - @model_validator(mode="after") def reject_duplicate_producers(self): names = [binding.input for binding in self.bindings] diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 53651665..e2edca08 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -55,11 +55,7 @@ image_views, ) from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata -from sequifier.config.policy_config import ( - ActionFieldConfig, - PolicyBinding, - ResolvedPolicyBinding, -) +from sequifier.config.policy_config import PolicyBinding, ResolvedPolicyBinding from sequifier.config.policy_optimization import SupervisedOptimizationConfig from sequifier.config.policy_plan import ( DPOPhaseConfig, @@ -71,6 +67,7 @@ PPOPhaseConfig, ) from sequifier.config.policy_train_config import SequenceSourceConfig +from sequifier.config.prediction_config import PredictionDistributionConfig from sequifier.config.sources import TrainingSourceSpecModel from sequifier.helpers import ( ModelWindowView, @@ -137,70 +134,19 @@ def _unique_columns(value: list[str], usage: str) -> list[str]: "updates", "total_env_steps", "total_groups", + "evaluation", + "evaluation_interval", + "checkpoint_interval", + "batch_size", + "accumulation_steps", + "optimization", + "handoff", + "optimizer_state", } ) _TRAINING_PLAN_POLICY_FIELDS = frozenset({"curriculum_training", "curriculum_column"}) -_LEGACY_BATCH_FIELDS = frozenset({"batch_size", "accumulation_steps"}) -_LEGACY_OPTIMIZATION_FIELDS = frozenset( - {"learning_rate", "optimizer", "scheduler", "scheduler_step_on", "gradient_clip"} -) -_LEGACY_TRAINING_FIELDS = _LEGACY_BATCH_FIELDS | _LEGACY_OPTIMIZATION_FIELDS - - -def _normalize_legacy_training_fields(global_training, phases, *, search=False): - """Move authored legacy fields before validating phases or search values.""" - if not isinstance(global_training, dict): - return - fields = global_training.keys() & _LEGACY_TRAINING_FIELDS - if not fields: - return - replacements = ", ".join( - f"global_training.{key} -> training_plan.phases[0]." - + (key if key in _LEGACY_BATCH_FIELDS else f"optimization.model.{key}") - for key in sorted(fields) - ) - if not isinstance(phases, (list, dict)) or len(phases) != 1: - raise ValueError( - f"Legacy training fields require exactly one phase: {replacements}" - ) - phase = next(iter(phases.values())) if isinstance(phases, dict) else phases[0] - if not isinstance(phase, dict): - raise ValueError("Legacy training fields require a phase mapping") - supervised = phase.get("algorithm", "supervised") == "supervised" - optimization_fields = fields & _LEGACY_OPTIMIZATION_FIELDS - if optimization_fields and not supervised: - raise ValueError( - f"Legacy optimization fields require a supervised phase: {replacements}; " - "policy optimization must select optimization. explicitly" - ) - optimization = phase.get("optimization", {}) - if not isinstance(optimization, dict): - raise ValueError("Phase optimization must be a mapping") - role = optimization.get("model", {}) - if not isinstance(role, dict): - raise ValueError("optimization.model must be a mapping") - legacy_defaults = not search and supervised and "model" not in optimization - for key in fields: - target = phase if key in _LEGACY_BATCH_FIELDS else role - if key in target: - raise ValueError(f"Duplicate training declaration: {replacements}") - value = global_training.pop(key) - target[key] = 1 if key == "accumulation_steps" and value is None else value - if optimization_fields or legacy_defaults: - phase.setdefault("optimization", {})["model"] = role - if legacy_defaults: - role.setdefault("gradient_clip", None) - optimizer = role.get("optimizer", {}) - if role.get("learning_rate") is None and not ( - isinstance(optimizer, dict) and "groups" in optimizer - ): - raise ValueError( - "Legacy supervised training requires an explicit learning_rate" - ) - - @beartype def _normalize_dataset_part_surface(datasets: Any) -> None: if not isinstance(datasets, dict): @@ -331,21 +277,23 @@ def normalize_train_config_surface(values: Any) -> Any: }: phase.setdefault("sources", [{"source": implicit_source}]) - _normalize_legacy_training_fields( - normalized.get("global_training"), - training_plan.get("phases") if isinstance(training_plan, dict) else None, - ) - - evaluation = normalized.get("evaluation") - if isinstance(evaluation, bool): - if not evaluation: - normalized["evaluation"] = None - elif implicit_source is None: - raise ValueError( - "evaluation=true requires exactly one configured dataset source." - ) - else: - normalized["evaluation"] = {"sources": [{"source": implicit_source}]} + if isinstance(training_plan, dict): + for phase in training_plan.get("phases", []): + if not isinstance(phase, dict) or phase.get("algorithm") != "supervised": + continue + evaluation = phase.get("evaluation") + if isinstance(evaluation, bool): + if not evaluation: + phase["evaluation"] = None + else: + sources = phase.get("sources", []) + if len(sources) != 1: + raise ValueError( + "evaluation=true requires exactly one phase source" + ) + phase["evaluation"] = { + "sources": [{"source": sources[0]["source"]}] + } return normalized @@ -448,18 +396,6 @@ def normalize_train_config_parameter_surface( "phases": {0: phase_override}, } - global_override = normalized.get("global_training") - if ( - isinstance(global_override, dict) - and global_override.keys() & _LEGACY_TRAINING_FIELDS - ): - base_phases = base_values.get("training_plan", {}).get("phases") - if not isinstance(base_phases, list) or len(base_phases) != 1: - _normalize_legacy_training_fields(global_override, base_phases, search=True) - plan = normalized.setdefault("training_plan", {}) - phases = plan.setdefault("phases", {0: {}}) - _normalize_legacy_training_fields(global_override, phases, search=True) - return normalized @@ -485,7 +421,6 @@ class GlobalTrainingSpecModel(BaseModel): device_max_concat_length: int = Field(default=12, gt=0) log_interval: int = Field(default=10, gt=0) early_stopping_epochs: Optional[int] = Field(default=None, gt=0) - save_interval_epochs: int = Field(default=1, gt=0) save_latest_interval_minutes: Optional[float] = None save_interval_minutes: Optional[float] = None save_interval_batches: Optional[int] = None @@ -607,7 +542,9 @@ class ModelInterfaceSpecModel(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") input_columns: list[str] = Field(default_factory=list) - prediction_distributions: dict[str, ActionFieldConfig] = Field(default_factory=dict) + prediction_distributions: dict[str, PredictionDistributionConfig] = Field( + default_factory=dict + ) static_conditioning: StaticConditioningConfig | None = None preconditioning_image_encoder: ImageEncoderConfig | None = None target_columns: list[str] = Field(..., min_length=1) @@ -650,6 +587,8 @@ def validate_decoder_tokens(cls, value): @model_validator(mode="after") @beartype def validate_interface_contract(self): + if self.prediction_distributions.keys() - set(self.target_columns): + raise ValueError("prediction_distributions must name model targets") input_columns = set(self.input_columns) if image_targets := set(self.target_columns) & image_views(self.ingestion): raise ValueError( @@ -843,9 +782,47 @@ def validate_loss_weights(cls, value): return value +class EvaluationMonitorSpecModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + source: str + metric: str = "loss" + mode: Literal["min", "max"] = "min" + + +class EvaluationSpecModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) + monitor: Optional[EvaluationMonitorSpecModel] = None + + @model_validator(mode="after") + @beartype + def validate_monitor(self): + if any( + source.weight is not None or source.batches_per_selection is not None + for source in self.sources + ): + raise ValueError( + "Evaluation sources cannot configure weight or batches_per_selection" + ) + sources = [source.source for source in self.sources] + if len(sources) != len(set(sources)): + raise ValueError("Evaluation sources must be unique") + if self.monitor is not None and self.monitor.source not in sources: + raise ValueError("evaluation.monitor.source must be an evaluation source") + if len(sources) == 1 and self.monitor is None: + self.monitor = EvaluationMonitorSpecModel(source=sources[0]) + return self + + class TrainingPhaseSpecModel(BaseModel): model_config = ConfigDict(extra="forbid") + evaluation: EvaluationSpecModel | None = None + evaluation_interval: int = Field(default=1, gt=0) + checkpoint_interval: int = Field(default=1, gt=0) + algorithm: Literal["supervised"] = "supervised" handoff: Literal["last", "best"] = "last" @@ -951,7 +928,13 @@ def validate_unique_names(self): raise ValueError( "optimizer continuation requires identical parameter selection" ) - if previous.optimization_configs() != phase.optimization_configs(): + if { + role: value.continuation_settings() + for role, value in previous.optimization_configs().items() + } != { + role: value.continuation_settings() + for role, value in phase.optimization_configs().items() + }: raise ValueError( "optimizer continuation requires identical role optimization settings" ) @@ -965,40 +948,6 @@ def validate_unique_names(self): return self -class EvaluationMonitorSpecModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - source: str - metric: str = "loss" - mode: Literal["min", "max"] = "min" - - -class EvaluationSpecModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) - monitor: Optional[EvaluationMonitorSpecModel] = None - - @model_validator(mode="after") - @beartype - def validate_monitor(self): - if any( - source.weight is not None or source.batches_per_selection is not None - for source in self.sources - ): - raise ValueError( - "Evaluation sources cannot configure weight or batches_per_selection" - ) - sources = [source.source for source in self.sources] - if len(sources) != len(set(sources)): - raise ValueError("Evaluation sources must be unique") - if self.monitor is not None and self.monitor.source not in sources: - raise ValueError("evaluation.monitor.source must be an evaluation source") - if len(sources) == 1 and self.monitor is None: - self.monitor = EvaluationMonitorSpecModel(source=sources[0]) - return self - - class SequifierConfig(BaseModel): """Training configuration with singleton authoring normalization.""" @@ -1020,7 +969,6 @@ class SequifierConfig(BaseModel): phase_directory: str = "" dataset_training: dict[str, DatasetTrainingSpecModel] = Field(default_factory=dict) training_plan: TrainingPlanModel - evaluation: Optional[EvaluationSpecModel] = None export_generative_model: bool = True export_embedding_model: bool = False @@ -1059,15 +1007,9 @@ def validate_relationships(self): if self.model is None and self.initialize_from is None: raise ValueError("a run requires model or initialize_from") for phase in self.training_plan.phases: - if ( - phase.algorithm == "supervised" - and phase.handoff == "best" - and (self.evaluation is None or self.evaluation.monitor is None) - ): - raise ValueError("supervised best handoff requires evaluation.monitor") - for phase in self.training_plan.phases: + evaluation = phase.evaluation refs = list(getattr(phase, "sources", [])) + list( - getattr(getattr(phase, "evaluation", None), "sources", None) or [] + getattr(evaluation, "sources", None) or [] ) preference = getattr(phase, "preferences", None) if preference is not None: @@ -1080,14 +1022,34 @@ def validate_relationships(self): raise ValueError( "policy source references cannot configure supervised interleaving" ) - for phase in self.training_plan.phases: + if phase.algorithm != "supervised": + continue + role = phase.optimization_configs()["model"] + needs_monitor = ( + phase.handoff == "best" + or role.scheduler.name == "ReduceLROnPlateau" + or self.global_training.early_stopping_epochs is not None + or ( + evaluation is not None + and self.global_training.save_interval_val_loss + ) + ) + if needs_monitor and (evaluation is None or evaluation.monitor is None): + raise ValueError( + f"{phase.name}: best handoff, plateau scheduling, validation saving, and early stopping require evaluation.monitor" + ) + if ( + role.scheduler.name == "ReduceLROnPlateau" + and phase.evaluation_interval != 1 + ): + raise ValueError("ReduceLROnPlateau requires evaluation_interval: 1") if ( - phase.algorithm == "supervised" - and phase.optimization_configs()["model"].scheduler.name - == "ReduceLROnPlateau" - and (self.evaluation is None or self.evaluation.monitor is None) + self.global_training.early_stopping_epochs is not None + and phase.evaluation_interval != 1 ): - raise ValueError("ReduceLROnPlateau requires evaluation.monitor") + raise ValueError( + "early_stopping_epochs requires evaluation_interval: 1" + ) if any(phase.algorithm != "supervised" for phase in self.training_plan.phases): if self.global_training.distributed: raise ValueError("policy phases currently require local execution") @@ -1105,11 +1067,6 @@ def validate_relationships(self): raise ValueError( "supervised training requires training_objective and context_length" ) - if self.global_training.early_stopping_epochs is not None and ( - self.evaluation is None or self.evaluation.monitor is None - ): - raise ValueError("early stopping requires evaluation.monitor") - for interface in model.interfaces.values(): validate_embedding_layer_names( self.embedding_layer_names, @@ -1163,19 +1120,6 @@ def validate_relationships(self): for phase in self.training_plan.phases: for source in phase.sources: self._validate_source(source.source, f"training phase {phase.name!r}") - if self.evaluation is not None: - for source in self.evaluation.sources: - self._validate_source(source.source, "evaluation") - needs_monitor = len(self.evaluation.sources) > 1 and ( - self.global_training.save_interval_val_loss - or self.global_training.early_stopping_epochs is not None - ) - if needs_monitor and self.evaluation.monitor is None: - raise ValueError( - "Multiple evaluation sources require evaluation.monitor when " - "validation-based saving or early stopping is enabled." - ) - objective = get_objective_class(self.global_training.training_objective) is_bert = issubclass(objective, BERTObjective) is_next = issubclass(objective, NextOccurrenceObjective) @@ -1256,7 +1200,9 @@ class ResolvedDatasetPart(BaseModel): class ResolvedModelInterface(BaseModel): policy: ResolvedPolicyBinding | None = None - prediction_distributions: dict[str, ActionFieldConfig] = Field(default_factory=dict) + prediction_distributions: dict[str, PredictionDistributionConfig] = Field( + default_factory=dict + ) model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") depth_layouts: DepthLayoutRegistryModel = Field( @@ -1304,6 +1250,18 @@ class ResolvedModelInterface(BaseModel): storage_layout: StoredWindowLayout window_view: ModelWindowView + @model_validator(mode="after") + def validate_prediction_targets(self): + for name in self.prediction_distributions: + if ( + name not in self.target_columns + or self.target_column_types.get(name) != "real" + ): + raise ValueError( + f"{name}: prediction_distributions requires a real model target" + ) + return self + class ResolvedDatasetTrainingSpec(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") @@ -1332,6 +1290,11 @@ class ResolvedTrainingSource(BaseModel): class ResolvedTrainingPhase(BaseModel): model_config = ConfigDict(extra="forbid") + evaluation_sources: list[ResolvedTrainingSource] = Field(default_factory=list) + evaluation_monitor: EvaluationMonitorSpecModel | None = None + evaluation_interval: int = Field(default=1, gt=0) + checkpoint_interval: int = Field(default=1, gt=0) + algorithm: Literal["supervised"] = "supervised" handoff: Literal["last", "best"] = "last" @@ -1353,6 +1316,16 @@ def resolve_supervised_phases(authored_phases): ResolvedTrainingPhase( name=phase.name, epochs=phase.epochs, + evaluation_sources=[ + _source(ref.source, 1) for ref in phase.evaluation.sources + ] + if phase.evaluation is not None + else [], + evaluation_monitor=phase.evaluation.monitor + if phase.evaluation is not None + else None, + evaluation_interval=phase.evaluation_interval, + checkpoint_interval=phase.checkpoint_interval, handoff=phase.handoff, optimizer_state=phase.optimizer_state, batch_size=phase.batch_size, @@ -1426,8 +1399,19 @@ class ResolvedSequifierConfig(BaseModel): training_plan: list[ResolvedTrainingPhase | PolicyPhase] curriculum_training: bool = False curriculum_column: Optional[str] = None - evaluation_sources: list[ResolvedTrainingSource] - evaluation_monitor: Optional[EvaluationMonitorSpecModel] + + @property + def evaluation_sources(self) -> list[ResolvedTrainingSource]: + """All supervised evaluation dependencies, for loader and metric setup.""" + return list( + { + source.source: source + for phase in self.training_plan + if phase.algorithm == "supervised" + for source in phase.evaluation_sources + }.values() + ) + export_generative_model: bool export_embedding_model: bool embedding_layer_names: list[str] @@ -1584,9 +1568,13 @@ def _layout_for_part_path( @beartype def _evaluated_parts(config: SequifierConfig) -> set[str]: selected: set[str] = set() - if config.evaluation is None: - return selected - for source in config.evaluation.sources: + sources = [ + ref + for phase in config.training_plan.phases + if phase.algorithm == "supervised" and phase.evaluation is not None + for ref in phase.evaluation.sources + ] + for source in sources: dataset_name, _, part_name = source.source.partition(".") if part_name: selected.add(source.source) @@ -2235,11 +2223,6 @@ def resolve_sequifier_config( ) phases = resolve_supervised_phases(config.training_plan.phases) - evaluation_sources = ( - [_source(source.source, 1) for source in config.evaluation.sources] - if config.evaluation is not None - else [] - ) resolved = ResolvedSequifierConfig( project_root=config.project_root, model_name=config.model_name, @@ -2251,10 +2234,6 @@ def resolve_sequifier_config( training_plan=phases, curriculum_training=config.training_plan.curriculum_training, curriculum_column=config.training_plan.curriculum_column, - evaluation_sources=evaluation_sources, - evaluation_monitor=( - config.evaluation.monitor if config.evaluation is not None else None - ), export_generative_model=config.export_generative_model, export_embedding_model=config.export_embedding_model, embedding_layer_names=config.embedding_layer_names, @@ -2315,12 +2294,13 @@ def _override_part(config: SequifierConfig) -> str: for source in phase.sources if source.source.startswith(f"{dataset_name}.") } - if config.evaluation is not None: - refs.update( - source.source - for source in config.evaluation.sources - if source.source.startswith(f"{dataset_name}.") - ) + refs.update( + source.source + for phase in config.training_plan.phases + if phase.algorithm == "supervised" and phase.evaluation is not None + for source in phase.evaluation.sources + if source.source.startswith(f"{dataset_name}.") + ) if len(refs) == 1: return next(iter(refs)) raise ValueError( @@ -2407,18 +2387,11 @@ def load_train_config_with_source( raw[key] = args.pop(key) from sequifier.config.composition import merge_config_fragments - legacy_cli = { - key: args.pop(key) for key in list(args) if key in _LEGACY_TRAINING_FIELDS - } overrides = { key: args.pop(key) for key in ("global_training", "training_plan") if key in args } - if legacy_cli: - overrides["global_training"] = merge_config_fragments( - (overrides.get("global_training", {}), legacy_cli) - ) raw = merge_config_fragments((raw, overrides)) if args: raise ValueError(f"Unsupported training CLI overrides: {sorted(args)}") diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 4a4bdc8c..548182b4 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -258,12 +258,12 @@ def _supervised_subset(config): if phase.algorithm == "supervised" ] names = {ref.source.partition(".")[0] for phase in phases for ref in phase.sources} - if config.evaluation: - names.update( - ref.source.partition(".")[0] - for ref in config.evaluation.sources - if ref.source.partition(".")[0] in names - ) + names.update( + ref.source.partition(".")[0] + for phase in phases + if phase.evaluation is not None + for ref in phase.evaluation.sources + ) return config.model_copy( update={ "dataset_training": { @@ -298,8 +298,10 @@ def resolve_training_run(authored: SequifierConfig): read_execution_artifact(config.initialize_from).model_config.values, device=config.device, ) - if config.model is not None and config.model != source.model: - raise ValueError("declared model differs from initialize_from model") + if config.model is not None: + from sequifier.artifacts.model_config import extend_prediction_distributions + + extend_prediction_distributions(source, config.model) config.model = source.model.model_copy(deep=True) for field in ( "context_length", @@ -573,6 +575,9 @@ def policy_run_config( device=config.device, interface_name=name, ) + from sequifier.artifacts.model_config import extend_prediction_distributions + + extend_prediction_distributions(execution, config.model) apply_binding( execution, name, diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index 260f862c..c0664c9b 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -1296,7 +1296,7 @@ def predict(history): "enforce_determinism": config.enforce_determinism, "seed_origin": { "data_id": data_id, - "seed_index": rollout.seed_window_index, + "seed_window_index": rollout.seed_window_index, "sequence_id": int(windowed.sequence_ids[row]), "subsequence_id": int(windowed.subsequence_ids[row]), "model_start_position": int(windowed.model_start_positions[row]), diff --git a/src/sequifier/model/execution.py b/src/sequifier/model/execution.py index d91a2ff9..61b5cb5f 100644 --- a/src/sequifier/model/execution.py +++ b/src/sequifier/model/execution.py @@ -52,8 +52,6 @@ def training_config(self, *, device="cpu", **options): "seed": 0, "device": device, "training_plan": [], - "evaluation_sources": [], - "evaluation_monitor": None, "export_generative_model": True, "export_embedding_model": False, "export_onnx": False, diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index e22c29ba..8424eaf4 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -389,12 +389,9 @@ def _build_composable_network( preconditioning_image_encoder=image_prefix_encoder, prefix_projection=prefix_projection, ) - from sequifier.model.prediction_distribution import ( - PredictionScales, - prediction_scale_fields, - ) + from sequifier.model.prediction_distribution import PredictionScales - distributions = prediction_scale_fields(interface) + distributions = interface.prediction_distributions if distributions: routes[name].prediction_scales = PredictionScales(distributions).to(device) routes[name].depth_layouts = interface.depth_layouts diff --git a/src/sequifier/model/prediction_distribution.py b/src/sequifier/model/prediction_distribution.py index 394dd355..e3f09c3e 100644 --- a/src/sequifier/model/prediction_distribution.py +++ b/src/sequifier/model/prediction_distribution.py @@ -1,51 +1,44 @@ """Scalar prediction distributions, independent of the learning objective.""" import math +from types import SimpleNamespace import torch from torch import nn from torch.nn import functional as F -def prediction_scale_fields(interface): - """Retain existing scale modules when a task temporarily fixes exploration.""" - fields = dict(interface.prediction_distributions) - if interface.policy is not None: - for name, field in interface.policy.actions.items(): - if name not in fields or field.learn_scale or not fields[name].learn_scale: - fields[name] = field - return fields - - class PredictionScales(nn.Module): - """Optional global exploration parameters owned by the model artifact.""" + """Gaussian scales owned and persisted by the model interface.""" def __init__(self, fields): super().__init__() + self.fields = dict(fields) self.log_scales = nn.ParameterDict( { - name: nn.Parameter(torch.tensor(math.log(fields[name].scale))) - for name in sorted(fields) - if fields[name].learn_scale + name: nn.Parameter(torch.tensor(math.log(field.scale.initial))) + for name, field in sorted(fields.items()) + if field.scale.trainable } ) def ensure_fields(self, fields): - for name in sorted(fields): - field = fields[name] - if field.learn_scale and name not in self.log_scales: + for name, field in sorted(fields.items()): + if field.scale.trainable and name not in self.log_scales: self.log_scales[name] = nn.Parameter( - torch.tensor(math.log(field.scale)) + torch.tensor(math.log(field.scale.initial)) ) + self.fields.update(fields) - def scale(self, name, field): - if not field.learn_scale: - return field.scale - return ( - self.log_scales[name] - .clamp(math.log(field.min_scale), math.log(field.max_scale)) - .exp() - ) + def scale(self, name): + if name not in self.fields: + raise ValueError( + f"{name}: scale: model requires a prediction_distributions entry" + ) + spec = self.fields[name].scale + if not spec.trainable: + return spec.initial + return self.log_scales[name].clamp(math.log(spec.min), math.log(spec.max)).exp() class RealPredictionDistribution: @@ -55,11 +48,15 @@ class RealPredictionDistribution: represent endpoints; continuous samples retain transform latents for replay. """ - def __init__(self, location, scale, field): + def __init__(self, location, scale, field=None): self.dtype = torch.float32 if location.device.type == "mps" else torch.float64 self.location = location.to(self.dtype) self.scale = torch.as_tensor(scale, device=location.device).to(self.dtype) - self.field = field + self.field = ( + field + if field is not None + else SimpleNamespace(lower=None, upper=None, atoms={}) + ) if ( not torch.isfinite(self.location).all() or not torch.isfinite(self.scale).all() diff --git a/src/sequifier/rl/attachment.py b/src/sequifier/rl/attachment.py index 0239e55c..7cf5d527 100644 --- a/src/sequifier/rl/attachment.py +++ b/src/sequifier/rl/attachment.py @@ -24,6 +24,10 @@ def resolve_binding(execution, interface_name, binding): for name, field in actions.items(): if name not in interface.target_columns: raise ValueError(f"{name}: policy action is not a model target") + if field.scale == "model" and name not in interface.prediction_distributions: + raise ValueError( + f"{name}: scale: model requires a model prediction_distributions entry" + ) if field.type != interface.target_column_types[name]: raise ValueError(f"{name}: action type differs from the model") if field.type == "categorical" and not field.values: @@ -79,22 +83,14 @@ def resolve_binding(execution, interface_name, binding): def apply_binding(execution, interface_name, policy): - """Persist distribution modules even when a later task selects fewer heads.""" + """Attach task behavior without changing model-owned distribution parameters.""" interface = execution.interface(interface_name) - from sequifier.model.prediction_distribution import prediction_scale_fields - - distributions = prediction_scale_fields(interface) - for name, field in policy.actions.items(): - if ( - name not in distributions - or field.learn_scale - or not distributions[name].learn_scale - ): - distributions[name] = field - interface.prediction_distributions = distributions + for name, action in policy.actions.items(): + if action.scale == "model" and name not in interface.prediction_distributions: + raise ValueError( + f"{name}: scale: model requires a model prediction_distributions entry" + ) interface.policy = policy - authored = execution.model.interfaces[interface_name] - authored.prediction_distributions = distributions def attach_policy(network, execution, interface_name, binding): diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 02f68809..73ef7410 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -55,6 +55,7 @@ def load_policy( device: str = "cpu", interface_name: str | None = None, binding=None, + model=None, ) -> Policy: if isinstance(path, ModelArtifact): network, config, artifact = restore_model_artifact( @@ -78,6 +79,10 @@ def load_policy( raise ValueError("Select one policy model_interface") interface_name = candidates[0] assert interface_name is not None + if model is not None: + from sequifier.artifacts.model_config import extend_prediction_distributions + + extend_prediction_distributions(config, model) if binding is not None: from sequifier.rl.attachment import attach_policy diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index c723091e..c9035139 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -109,7 +109,7 @@ def infer_rollout(policy, data, config): { "model_path": config.model_path, "model_interface": config.model_interface, - "seed_index": rollout.seed_row_index, + "seed_row_index": rollout.seed_row_index, "enforce_determinism": config.enforce_determinism, "seed": config.seed, } diff --git a/src/sequifier/rl/joint_distribution.py b/src/sequifier/rl/joint_distribution.py index 7f20338a..2527a47a 100644 --- a/src/sequifier/rl/joint_distribution.py +++ b/src/sequifier/rl/joint_distribution.py @@ -79,7 +79,9 @@ def _field(self, name, parameters): location, actor.network.resolve_interface( actor.interface_name - ).prediction_scales.scale(name, field), + ).prediction_scales.scale(name) + if field.scale == "model" + else field.scale.fixed, field, ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index ffd9f4b0..39f2afcb 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -37,6 +37,11 @@ def _plan_contract(config): values = config.model_dump(mode="json") values.pop("resume") + for phase in values["phases"]: + for role in phase["optimization"].values(): + if role["scheduler"]["name"] == "OneCycleLR": + # total_steps is an assertion, not a separately configurable budget. + role["scheduler"].pop("total_steps", None) return values @@ -47,6 +52,7 @@ def _initialize_actor(config): device=config.device, interface_name=config.policy_interface, binding=config.policy, + model=config.model, ) validate_declared_policy(config.policy, source.config, "initialize_from.policy") from sequifier.artifacts.model_config import validate_execution_attachment @@ -128,14 +134,6 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: data_contract = [item.contract if item is not None else None for item in data] if recovery is not None: require_equal(data_contract, recovery.data_contract, "resume.partitions") - for phase in config.phases: - normalization = getattr(phase, "normalization", None) - if normalization is not None: - require_equal( - representation.values["statistics"], - resolve_normalization(actor.config, normalization), - f"phases.{phase.name}.normalization", - ) if recovery is None: recovery = RunCheckpoint( plan, diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 67744fd5..17d81a35 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -133,6 +133,13 @@ def _resolve_actions(self): ) for index, name in enumerate(order): field = self.config.actions[name] + if ( + field.scale == "model" + and name not in self.interface.prediction_distributions + ): + raise ValueError( + f"{name}: scale: model requires a model prediction_distributions entry" + ) if field.type != self.interface.target_column_types[name]: raise ValueError(f"{name}: action type differs from the model target") if set(field.active_when) - set(order[:index]): @@ -154,11 +161,13 @@ def _resolve_actions(self): f"{name}: activity condition uses an unknown category" ) if not hasattr(route, "prediction_scales"): - route.prediction_scales = PredictionScales(self.config.actions).to( - self.device - ) + route.prediction_scales = PredictionScales( + self.interface.prediction_distributions + ).to(self.device) else: - route.prediction_scales.ensure_fields(self.config.actions) + route.prediction_scales.ensure_fields( + self.interface.prediction_distributions + ) route.prediction_scales.to(self.device) def action_decoder_ids(self, name): diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index a18280af..8fc3c510 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -167,6 +167,6 @@ def preflight_optimization(config, actor, data): budget = phase.optimizer_step_budget(size) for optimization in phase.optimization_configs().values(): optimization.validate_budget(budget) - optimization.build(actor.network) + optimization.build(actor.network, step_budget=budget) budgets.append(budget) return budgets diff --git a/src/sequifier/rl/references.py b/src/sequifier/rl/references.py index 1d20a625..737c441c 100644 --- a/src/sequifier/rl/references.py +++ b/src/sequifier/rl/references.py @@ -58,6 +58,7 @@ def _artifact(self, path): device="cpu", interface_name=self.config.policy_interface, binding=self.config.policy, + model=self.config.model, ) self.recovery.representation.validate_actor( source, self.config.units, f"reference.{path}" diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 9a46c494..6b74e2b7 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -227,11 +227,14 @@ def __init__(self, context, *, modules=None, frozen=None): actor.network.get_submodule(path).requires_grad_(False) self.modules = {"actor": context.actor.network, **(modules or {})} self.networks = {**self.modules, **(frozen or {})} - self.configs = context.phase.optimization_configs() + self.configs = { + role: config.resolve_budget(context.step_budget) + for role, config in context.phase.optimization_configs().items() + } if self.modules.keys() != self.configs.keys(): raise ValueError("phase optimization roles disagree with trainable modules") self.optimization = { - role: self.configs[role].build(module) + role: self.configs[role].build(module, step_budget=context.step_budget) for role, module in self.modules.items() } continuation = getattr(actor, "_phase_continuation", None) diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index d5149323..f67da761 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -192,7 +192,9 @@ def build( validate_execution_attachment( artifact.model_config.values, model_execution_config(config) ) - network.load_state_dict(artifact.model_state_dict, strict=True) + from sequifier.artifacts.model_config import load_attached_model_weights + + load_attached_model_weights(network, artifact) if loaded is None and initial_artifact is None: revision = select_revision(config.model.backbone, config.project_root) if revision is not None: diff --git a/src/sequifier/sequifier.py b/src/sequifier/sequifier.py index 9eeda801..b5445c76 100644 --- a/src/sequifier/sequifier.py +++ b/src/sequifier/sequifier.py @@ -120,13 +120,8 @@ def setup_parser() -> ArgumentParser: ) parser_train.add_argument("-mn", "--model-name", type=str) parser_train.add_argument("-s", "--seed", type=int) - parser_train.add_argument("--batch-size", type=int) - parser_train.add_argument("--accumulation-steps", type=int) - parser_train.add_argument("--learning-rate", type=float) - parser_train.add_argument("--gradient-clip", type=float) - parser_train.add_argument("--scheduler-step-on", choices=("epoch", "batch")) - parser_train.add_argument("--optimizer", type=yaml.safe_load) - parser_train.add_argument("--scheduler", type=yaml.safe_load) + parser_train.add_argument("--global-training", type=yaml.safe_load) + parser_train.add_argument("--training-plan", type=yaml.safe_load) parser_infer.add_argument("-mp", "--model-path", type=str) parser_infer.add_argument("-s", "--seed", type=int) diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index 827fb4f1..515689e7 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -106,6 +106,12 @@ def strip_provenance(value: Any) -> Any: } for name, dataset in config.dataset_training.items() } + phases = [phase.model_dump(mode="python") for phase in config.training_plan] + for phase in phases: + for role in phase["optimization"].values(): + if role["scheduler"]["name"] == "OneCycleLR": + # Recomputed from the phase/data budget before optimizer construction. + role["scheduler"].pop("total_steps", None) return { "model": execution, "initialization_provenance": { @@ -127,9 +133,7 @@ def strip_provenance(value: Any) -> Any: "load_full_data_to_ram": training.load_full_data_to_ram, }, "datasets": datasets, - "training_plan": [ - phase.model_dump(mode="python") for phase in config.training_plan - ], + "training_plan": phases, } @classmethod diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index cc1d00b0..748f1a4d 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -51,6 +51,9 @@ def _supervised_config(config, phase, index, incoming, resume): source, _ = resolved_config_from_model_config( read_execution_artifact(incoming).model_config.values, device=config.device ) + from sequifier.artifacts.model_config import extend_prediction_distributions + + extend_prediction_distributions(source, config.model) model = source.model forward_attention = source.global_training.forward_attention or ( "bidirectional" @@ -76,12 +79,13 @@ def _supervised_config(config, phase, index, incoming, resume): "dataset_training": { name: value.model_dump() for name, value in authored.dataset_training.items() - if name in {ref.source.partition(".")[0] for ref in phase.sources} + if name + in { + ref.source.partition(".")[0] + for ref in [*phase.sources, *phase.evaluation_sources] + } }, "training_plan": {"phases": [phase_spec.model_dump()]}, - "evaluation": authored.evaluation.model_dump() - if authored.evaluation - else None, "export_pt": True, "export_onnx": config.export_onnx, "export_generative_model": config.export_generative_model, diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 006b3722..5a1b685f 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -122,7 +122,10 @@ def _evaluation_context( def _evaluate(self, run: Any, context: EvaluationContext) -> EvaluationResult: result = run.evaluation.evaluate( - run.network, run.config.evaluation_sources, run.datasets, context + run.network, + run.config.training_plan[context.phase_index].evaluation_sources, + run.datasets, + context, ) run.metrics.record_evaluation( result, @@ -160,8 +163,10 @@ def _check_pruning(self, run: Any) -> None: if should_prune: raise SystemExit(143) - def _update_monitor(self, run: Any, result: EvaluationResult) -> tuple[float, bool]: - monitor = run.config.evaluation_monitor + def _update_monitor( + self, run: Any, result: EvaluationResult, phase: Any + ) -> tuple[float, bool]: + monitor = phase.evaluation_monitor if monitor is None or monitor.source not in result.sources: return float("nan"), False value = result.sources[monitor.source].total_loss @@ -326,7 +331,7 @@ def flush_metrics( try: if ( state.global_batch_step == 0 - and config.evaluation_sources + and config.training_plan[0].evaluation_sources and config.global_training.calculate_validation_loss_on_initialization ): first_sources = [ @@ -598,7 +603,7 @@ def flush_metrics( if snapshot_due: if ( config.global_training.save_interval_val_loss - and config.evaluation_sources + and phase.evaluation_sources ): self._evaluate( run, @@ -647,23 +652,31 @@ def flush_metrics( batches_total=current_batches_total, ), ) - if config.evaluation_sources + if phase.evaluation_sources + and ( + phase_epoch % phase.evaluation_interval == 0 + or phase_epoch == phase.epochs + ) else EvaluationResult() ) if run.optimization.scheduler_policy.step_on == "epoch": - monitor = config.evaluation_monitor + monitor = phase.evaluation_monitor metric = ( evaluation.sources[monitor.source].total_loss if monitor is not None + and monitor.source in evaluation.sources else None ) run.optimization.step_scheduler(metric) state.iterator_positions = {} state.source_scheduler_state = scheduler.state_dict() - self._update_monitor(run, evaluation) + self._update_monitor(run, evaluation, phase) boundary = capture_runtime_boundary(run) boundary_identity = last_identity - if current_epoch % config.global_training.save_interval_epochs == 0: + if ( + phase_epoch % phase.checkpoint_interval == 0 + or phase_epoch == phase.epochs + ): run.checkpoints.save( CheckpointRequest(f"epoch-{current_epoch}", last_identity), run, @@ -671,7 +684,7 @@ def flush_metrics( patience = config.global_training.early_stopping_epochs if ( patience is not None - and config.evaluation_monitor is not None + and phase.evaluation_monitor is not None and state.epochs_without_improvement >= patience ): stop_requested = True diff --git a/src/sequifier/training/loss.py b/src/sequifier/training/loss.py index 5396a742..bf8d8171 100644 --- a/src/sequifier/training/loss.py +++ b/src/sequifier/training/loss.py @@ -261,8 +261,6 @@ def calculate( ) field = interface.prediction_distributions.get(target) - if field is None and interface.policy is not None: - field = interface.policy.actions.get(target) if field is None or field.type != "real": raise ValueError( f"{target}: GaussianNLLLoss requires a real prediction_distributions entry" @@ -277,7 +275,7 @@ def calculate( location = denormalize_values(location, statistics) observed = denormalize_values(observed, statistics) distribution = RealPredictionDistribution( - location, route.prediction_scales.scale(target, field), field + location, route.prediction_scales.scale(target) ) sums[target] = -distribution.log_prob(observed).sum() elif isinstance(criterion, AccurateCrossEntropyLoss): diff --git a/src/sequifier/training/scheduler_budgets.py b/src/sequifier/training/scheduler_budgets.py index d5f4c4bd..e7f01b4a 100644 --- a/src/sequifier/training/scheduler_budgets.py +++ b/src/sequifier/training/scheduler_budgets.py @@ -95,10 +95,18 @@ def validate_supervised_scheduler_budgets(config, datasets, active_phase_index): config, datasets, selected ) budget = sum(phase_budgets[selected] for selected in range(start, end)) - if role.scheduler.arguments["total_steps"] != budget: + explicit = role.scheduler.arguments.get("total_steps") + if explicit is not None and explicit != budget: raise ValueError( f"{phase.name}: OneCycleLR.total_steps must equal optimizer-update budget {budget}" ) + if budget <= 0: + raise ValueError( + f"{phase.name}: OneCycleLR requires a positive optimizer-update budget" + ) + role.scheduler = role.scheduler.model_copy( + update={"arguments": {**role.scheduler.arguments, "total_steps": budget}} + ) for dataset in datasets.values(): for part in dataset.parts.values(): if ( diff --git a/src/sequifier/trajectory/export.py b/src/sequifier/trajectory/export.py index 3479a776..77c81faf 100644 --- a/src/sequifier/trajectory/export.py +++ b/src/sequifier/trajectory/export.py @@ -27,11 +27,11 @@ def import_sequences(config) -> Path: ) destination.parent.mkdir(parents=True, exist_ok=True) columns = PolicyTableColumns( - action="__executed_action", + action_record="__executed_action", reward="__reward", terminated="__terminated", truncated="__truncated", - previous_action="__previous_action", + previous_action_record="__previous_action", bootstrap_allowed="__bootstrap", group="__group", episode_id="episodeId", diff --git a/src/sequifier/trajectory/tables.py b/src/sequifier/trajectory/tables.py index 5c18aece..baf65417 100644 --- a/src/sequifier/trajectory/tables.py +++ b/src/sequifier/trajectory/tables.py @@ -220,10 +220,10 @@ def table_features(data, policy, columns, static_data): (policy.static_columns, columns.static_inputs), ) ) - action_column = columns.action + action_column = columns.action_record action_columns = columns.actions or ( {name: name for name in policy.action_columns} - if columns.action is None + if columns.action_record is None else {name: action_column for name in policy.action_columns} ) inputs = set(observations.values()) @@ -231,7 +231,8 @@ def table_features(data, policy, columns, static_data): inputs.update(static_inputs.values()) if ( set(action_columns.values()) & inputs - or columns.previous_action in action_columns.values() + or columns.previous_action_record in action_columns.values() + or set(columns.previous_actions.values()) & set(action_columns.values()) ): raise ValueError( "current action labels require distinct input/previous-action columns" @@ -252,7 +253,7 @@ def table_features(data, policy, columns, static_data): filter( None, ( - columns.previous_action, + columns.previous_action_record, columns.bootstrap_allowed, columns.group, columns.episode_id, @@ -264,6 +265,15 @@ def table_features(data, policy, columns, static_data): f"policy table is missing columns: {sorted(required - set(data.columns))}" ) + for record in (columns.action_record, columns.previous_action_record): + if ( + record is not None + and data.schema[record] != pl.Null + and not isinstance(data.schema[record], pl.Struct) + ): + raise ValueError( + f"{record}: action records require a structured column; use actions/previous_actions for scalar columns" + ) return observations, static_inputs @@ -290,10 +300,10 @@ def identity_decode(value, column): if data.is_empty(): raise ValueError("sequence source changed while loading") - action_column = columns.action + action_column = columns.action_record action_columns = columns.actions or ( {name: name for name in policy.action_columns} - if columns.action is None + if columns.action_record is None else {name: action_column for name in policy.action_columns} ) observations, static_inputs = table_features(data, policy, columns, static_data) @@ -331,10 +341,10 @@ def constant(entries, label): if set(action_columns) != set(policy.actions): raise ValueError("table action mappings must cover every action field") actions = tuple( - canonical_action(dict(row[columns.action])) - if columns.action is not None + canonical_action(dict(row[columns.action_record])) + if columns.action_record is not None and not columns.actions - and isinstance(row[columns.action], dict) + and isinstance(row[columns.action_record], dict) else { name: decode(row[column], name) for name, column in action_columns.items() } @@ -356,9 +366,9 @@ def constant(entries, label): raise ValueError( "previous actions must lag recorded actions by one decision" ) - if columns.previous_action is not None: + if columns.previous_action_record is not None: preceding_actions = values( - columns.previous_action, + columns.previous_action_record, lambda value, _: None if value is None else (canonical_action(value)), ) if preceding_actions[1:] != actions[:-1]: From 83a46156c314279fe5961d41e2842aebcd3a0a99 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 11 Nov 2026 12:03:38 +0100 Subject: [PATCH 29/71] unify naming --- README.md | 25 +- docs/source/guides/naming.rst | 2 + docs/source/index.rst | 1 + .../configs/hyperparameter-search.md | 136 +++-- documentation/configs/infer.md | 76 +-- documentation/configs/naming.md | 202 +++++++ documentation/configs/preprocess.md | 86 ++- documentation/configs/train.md | 571 +++++++++++------- .../demos/self-contained-example.ipynb | 80 ++- .../configs/import-policy-sequences.yaml | 15 +- .../example_inputs/configs/infer-policy.yaml | 7 +- .../example_inputs/configs/infer-ppo-toy.yaml | 7 +- .../configs/rollout-ppo-toy.yaml | 10 +- .../example_inputs/configs/train-bc-toy.yaml | 71 ++- .../configs/train-iql-from-table.yaml | 88 ++- .../example_inputs/configs/train-iql-toy.yaml | 75 ++- .../configs/train-multiphase-control.yaml | 182 +++--- .../configs/train-policy-from-table.yaml | 78 ++- .../configs/train-return-conditioned.yaml | 84 ++- documentation/policy-learning.md | 32 +- documentation/rollout.md | 4 +- documentation/supervised-policy-learning.md | 21 +- documentation/training/multi-gpu-training.md | 17 +- .../unreleased-canonical-config-names.md | 23 + src/sequifier/artifacts/model_export.py | 2 +- .../canonical_hyperparameter_search_config.py | 134 ++-- src/sequifier/config/components.py | 143 ++--- src/sequifier/config/composition.py | 18 +- src/sequifier/config/depth_layout.py | 8 +- src/sequifier/config/freezing_config.py | 32 +- src/sequifier/config/image_config.py | 81 ++- src/sequifier/config/infer_config.py | 123 ++-- src/sequifier/config/inference_common.py | 27 +- src/sequifier/config/initialization_config.py | 34 +- src/sequifier/config/metadata.py | 13 +- src/sequifier/config/naming.py | 130 ++++ src/sequifier/config/optimizer_config.py | 9 +- src/sequifier/config/policy_config.py | 5 +- src/sequifier/config/policy_infer_config.py | 24 +- src/sequifier/config/policy_optimization.py | 11 +- src/sequifier/config/policy_plan.py | 83 +-- src/sequifier/config/policy_train_config.py | 20 +- src/sequifier/config/prediction_config.py | 6 +- src/sequifier/config/preprocess_config.py | 62 +- src/sequifier/config/probabilities.py | 23 +- src/sequifier/config/rollout_config.py | 10 +- src/sequifier/config/sources.py | 7 +- src/sequifier/config/train_config.py | 479 ++++----------- src/sequifier/config/training_resolution.py | 18 +- src/sequifier/docs.py | 1 + src/sequifier/helpers.py | 6 +- src/sequifier/hyperparameter_search.py | 42 +- src/sequifier/infer.py | 20 +- src/sequifier/integration/contexts.py | 2 +- src/sequifier/io/depth_preprocess.py | 46 +- src/sequifier/io/images.py | 12 +- src/sequifier/io/pt_payload.py | 6 +- src/sequifier/io/yaml.py | 4 +- src/sequifier/make.py | 123 ++-- src/sequifier/model/backbone.py | 26 +- src/sequifier/model/decoders.py | 4 +- src/sequifier/model/depth_ingestion.py | 8 +- src/sequifier/model/encoder_stack.py | 10 +- src/sequifier/model/execution_schema.py | 4 +- src/sequifier/model/factory.py | 14 +- src/sequifier/model/images.py | 6 +- src/sequifier/model/ingestion_compiler.py | 28 +- src/sequifier/model/ingestions.py | 92 ++- src/sequifier/model/initialization.py | 11 +- src/sequifier/model/layers.py | 80 +-- src/sequifier/model/network.py | 12 +- src/sequifier/model/static_conditioning.py | 10 +- src/sequifier/preprocess.py | 86 ++- src/sequifier/rl/attachment.py | 4 +- src/sequifier/rl/dpo.py | 8 +- src/sequifier/rl/grpo.py | 8 +- src/sequifier/rl/grpo_update.py | 4 +- src/sequifier/rl/inference.py | 8 +- src/sequifier/rl/iql.py | 12 +- src/sequifier/rl/learner.py | 2 +- src/sequifier/rl/models.py | 2 +- src/sequifier/rl/ppo.py | 4 +- src/sequifier/rl/ppo_update.py | 18 +- src/sequifier/rl/preparation.py | 2 +- src/sequifier/rl/run_state.py | 16 +- src/sequifier/rl/training.py | 2 +- src/sequifier/runtime/builder.py | 8 +- src/sequifier/sequifier.py | 94 ++- src/sequifier/templates/infer-policy.yaml | 7 +- src/sequifier/templates/train-dpo.yaml | 74 ++- src/sequifier/templates/train-grpo.yaml | 93 ++- src/sequifier/templates/train-iql.yaml | 71 ++- .../train-policy-from-supervised.yaml | 38 +- src/sequifier/templates/train-policy.yaml | 78 ++- src/sequifier/templates/train-ppo.yaml | 105 ++-- .../templates/train-supervised-to-policy.yaml | 122 ++-- src/sequifier/templates/train-supervised.yaml | 80 ++- src/sequifier/training/checkpoint_service.py | 2 +- src/sequifier/training/engine.py | 54 +- src/sequifier/training/optimization.py | 12 +- src/sequifier/training/runtime.py | 19 +- src/sequifier/training/scheduler_budgets.py | 7 +- src/sequifier/trajectory/sources.py | 6 +- 103 files changed, 2813 insertions(+), 2195 deletions(-) create mode 100644 docs/source/guides/naming.rst create mode 100644 documentation/configs/naming.md create mode 100644 release-notes/unreleased-canonical-config-names.md create mode 100644 src/sequifier/config/naming.py diff --git a/README.md b/README.md index 7e4f141d..c286b649 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ Run `sequifier docs` for all documentation, or `sequifier docs train`, `sequifie The processing and visualization commands have dedicated documentation pages: +- [canonical configuration names](./documentation/configs/naming.md) - [preprocess documentation](./documentation/configs/preprocess.md) - [train documentation](./documentation/configs/train.md) - [policy training](./documentation/policy-learning.md), including BC, IQL, DPO, GRPO and [online categorical PPO](./documentation/policy-learning.md#online-categorical-ppo) @@ -169,7 +170,7 @@ The basic input data format is this: The two columns "sequenceId" and "itemPosition" have to be present, and there must be one or more feature columns. -`sequifier preprocess` splits sequences into subsequences, normalises real variables and maps categorical variables to integers/tokens. Each stored subsequence contains `window_length` positions; `max_target_offset` reserves positions within that window for future targets. +`sequifier preprocess` splits sequences into subsequences, normalises real variables and maps categorical variables to integers/tokens. Each stored subsequence contains the number of positions selected by its split in `window_lengths`; `max_target_offset` reserves positions within that window for future targets. | sequenceId | subsequenceId | startItemPosition | leftPadLength | inputCol | [window_length - 1] | [window_length - 2] | ... | 0 | | --- | --- | --- | --- | --- | --- | --- | --- | --- | @@ -215,7 +216,7 @@ Once you have your data in the input format described above, you can train a tra sequifier preprocess ``` -5. **Reference the generated metadata.** Preprocessing outputs metadata at `configs/metadata_configs/[INPUT BASENAME].json`. For a single dataset and part, reference that file from `dataset.part.metadata_config_path` in `train.yaml`; named configurations use `dataset_training..parts..metadata_config_path`. Inference may still use `preprocessing_data_path` or `metadata_config_path`. +5. **Reference the generated metadata.** Preprocessing outputs metadata at `configs/metadata_configs/[INPUT BASENAME].json`. Reference that file from `dataset_training..parts..metadata_config_path` in `train.yaml`. Inference uses `metadata_config_path` or the selected artifact metadata. 6. **Configure and train the model.** Adapt `train.yaml` with your transformer hyperparameters, then run: @@ -223,7 +224,7 @@ Once you have your data in the input format described above, you can train a tra sequifier train ``` -7. **Configure inference.** Point `model_path` in `infer.yaml` at the default ONNX export. Keep the scaffold's explicit contract, or replace it with `training_config_path` and `dataset`; see the [ONNX/PT trade-offs](./documentation/configs/infer.md#onnx-or-pt). +7. **Configure inference.** Point `model_paths` in `infer.yaml` at the default ONNX export. Keep the scaffold's explicit contract, or replace it with `training_config_path` and `dataset`; see the [ONNX/PT trade-offs](./documentation/configs/infer.md#onnx-or-pt). **GPU ONNX inference:** `device: cuda` requires `onnxruntime-gpu` built for your CUDA and cuDNN versions. The CPU-only `onnxruntime` package does not @@ -255,7 +256,7 @@ current loss calculation uses all valid decoded positions, regardless of ### Autoregressive Inference -Autoregressive inference is allowed when the model is causal, all input variables are target variables, `target_offset` equals 1 and `prediction_length` equals 1. It is enabled by setting `autoregressive: true`, and `generation_steps` to the desired integer value. The number of generation_steps will be generated starting from the *first* complete subsequence in a sequence. +Autoregressive inference is allowed when the model is causal, all input variables are target variables, `target_offset` equals 1 and `prediction_length` equals 1. It is enabled by setting `autoregressive_generation: true`, and `generation_steps` to the desired integer value. The number of generation_steps will be generated starting from the *first* complete subsequence in a sequence. It iteratively predicts future values, by returning predictions at step t-1 as input for generating a prediction at t. Predictions for categorical target variables can be made using argmax or sampling. @@ -280,13 +281,13 @@ Next-occurrence causal modelling is a generalisation of final-value causal model ### Causal Embedding Model -sequifier also supports the export of causal embeddings, instead of predictions. It requires the following settings: `export_embedding_model: true` in the training config and `model_type: embedding` in the inference config. Selected activations are restricted to the configured final `prediction_length` positions and concatenated in configuration order along the feature dimension. +sequifier also supports the export of causal embeddings, instead of predictions. It requires the following settings: `export_embedding_model: true` in the training config and `task: embedding` in the inference config. Selected activations are restricted to the configured final `prediction_length` positions and concatenated in configuration order along the feature dimension. If you are interested in activations *other* than the last backbone layer, you can configure the exact layers you want to contribute to the export using `embedding_layer_names`. You can pass an ordered list, such as `[backbone.layers.1, decoder.branches.default.hidden_blocks.0]`, and the activations of these layers will be concatenated and output. Layer names follow the network hierarchy using zero-based indices: `backbone.layers.` selects a transformer block output, `backbone.final_norm` the normalized backbone output, and `decoder.branches..hidden_blocks.` an MLP decoder hidden-block output; the same scheme applies to BERT embedding models. -Backbone selectors contribute `dim_model` activations. Decoder MLP hidden-block selectors contribute their configured hidden width and receive the same flattened `decoding_support * dim_model` windows used during training. The default, `embedding_layer_names: [backbone.final_norm]`, preserves the final normalised backbone representation. +Backbone selectors contribute `model_dim` activations. Decoder MLP hidden-block selectors contribute their configured hidden width and receive the same flattened `support_length * model_dim` windows used during training. The default, `embedding_layer_names: [backbone.final_norm]`, preserves the final normalised backbone representation. ### BERT Model @@ -296,7 +297,7 @@ Configuration: - Preprocessing: Set `max_target_offset: 0` for equal-width input and target windows. - Training: Set `training_objective: bert`, configure `bert_spec`, and set decoder `prediction_length` equal to `context_length`. Enable generative and/or embedding export according to the desired inference. -- Inference: Set `model_type: generative` to reconstruct explicitly masked input, or `model_type: embedding` to output contextual representations. +- Inference: Set `task: forecast` to reconstruct explicitly masked input, or `task: embedding` to output contextual representations. Technical Details: BERT-style models use bidirectional attention and learn by reconstructing positions sampled according to `bert_spec`. Inference does not apply random masking; inputs that should be reconstructed must be masked explicitly, for example using `mask_column` during preprocessing. Embedding inference returns one contextual representation for every valid position in the input window. @@ -337,11 +338,11 @@ Separately, `temporal_conv` enables temporal convolutions on pass-through or emb `temporal_conv` uses causal, left-padded convolutions by default (`causal: true`). For causal models, keep the default `orientation: within_item_position` when -using `post_conv_norm: layer_norm` or `rmsnorm`. With +using `post_conv_normalization: layer_norm` or `rmsnorm`. With `orientation: within_column`, these norms operate across the full time axis, so earlier positions can depend on later ones despite the causal convolution and the transformer's attention mask. If `within_column` is needed in a causal -model, set `post_conv_norm: none`. +model, set `post_conv_normalization: none`. #### Categorical multi-hash embeddings and targets @@ -353,8 +354,10 @@ logical column name: model: interfaces: default: - input_columns: [product_id] - target_columns: [product_id] + input_columns: + - product_id + target_columns: + - product_id categorical_hashing: product_id: type: multi_hash diff --git a/docs/source/guides/naming.rst b/docs/source/guides/naming.rst new file mode 100644 index 00000000..c546aea7 --- /dev/null +++ b/docs/source/guides/naming.rst @@ -0,0 +1,2 @@ +.. include:: ../../../documentation/configs/naming.md + :parser: myst_parser.sphinx_ diff --git a/docs/source/index.rst b/docs/source/index.rst index 92ccac06..19611fc8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,6 +9,7 @@ Welcome to Sequifier's documentation! Training Inference Hyperparameter search + Configuration naming Visualizing training Distributed and multi-node training Training runtime architecture diff --git a/documentation/configs/hyperparameter-search.md b/documentation/configs/hyperparameter-search.md index 70e85d99..304b0e17 100644 --- a/documentation/configs/hyperparameter-search.md +++ b/documentation/configs/hyperparameter-search.md @@ -18,24 +18,41 @@ search spaces using the same paths as the training schema. ```yaml base_config_path: train.yaml name: transformer-width-search -model_config_write_path: configs/hp-search +trial_config_directory: configs/hp-search method: bayesian trials: 40 - parameters: global_training: - context_length: [64, 128] - batch_size: [16, 32] - learning_rate: - low: 0.0001 - high: 0.001 - log: true + context_length: + "$choices": + - 64 + - 128 model: backbone: architecture: - num_layers: {low: 4, high: 8, step: 2} + num_layers: + "$range": + low: 4 + high: 8 + step: 2 training_plan: - epochs: [2, 4] + phases: + '0': + epochs: + "$choices": + - 2 + - 4 + batch_size: + "$choices": + - 16 + - 32 + optimization: + model: + learning_rate: + "$range": + low: 0.0001 + high: 0.001 + log: true ``` `base_config_path` resolves relative to the hyperparameter-search entry file. @@ -43,8 +60,8 @@ The base may itself be a composed training config. If the search config supplies `project_root`, that value is used in every generated trial; otherwise the base training config's root is inherited. -For a singleton base, use `model.interface`, `dataset.part`, and -`training_plan.epochs`. Named paths remain available for multi-value bases. +Use the same named paths as the training base: `model.interfaces.`, +`dataset_training..parts.`, and indexed `training_plan.phases`. The historical self-contained search schema and historical flat training base configs are not accepted. Every generated trial is validated as an authored @@ -63,44 +80,56 @@ have these forms: | Form | Meaning | | --- | --- | -| Scalar | Fixed replacement. | -| List on a scalar field | Categorical choices. | -| `{low, high, step?, log?, type?}` | Integer or float distribution. `type` may be `int` or `float`; otherwise the base value and bounds determine it. | -| `{choices: [...]}` or `{$choices: [...]}` | Categorical choices for an entire value, including mappings and lists. | -| `{fixed: value}` or `{$fixed: value}` | Unambiguous fixed replacement for a mapping or list. | -| Numeric keys under a base list | Recursive replacement of zero-based list entries. | -| `{variants: [...]}` or `{$variants: [...]}` | Paired partial mapping variants, optionally followed by independently sampled sibling fields. | +| Primitive scalar | Fixed replacement. | +| Ordinary mapping | Recursive patch. | +| `{$range: {low, high, step?, log?, type?}}` | Numeric distribution; `type` is `int` or `float`. | +| `{$choices: [...]}` | Categorical selection of any scalar, list, or mapping value. | +| `{$fixed: value}` | Complete mapping or list replacement. | +| Integer keys under a base list | Recursive replacement of zero-based list entries. | +| `{$variants: [...]}` | Partial mapping variants, optionally with independently sampled sibling fields. | + +Bare lists, unprefixed operators, and bare numeric distribution mappings are +rejected. `$` is reserved for search operators; their meaning does not depend on +the base value's type. Integer ranges default to `step: 1`. Grid search requires a `step` for float ranges because an unstepped float interval is infinite. Logarithmic integer ranges require `step: 1`; logarithmic float ranges cannot use `step`. -A direct list of lists samples a complete list-valued field. Prefer the explicit -`choices` wrapper for mapping or list candidates: +Use `$choices` to sample complete scalar, mapping, or list values: ```yaml parameters: model: backbone: architecture: - choices: - - dim_model: 128 - max_context_length: 512 - num_layers: 4 - attention: {type: mha, n_heads: 8} - feed_forward: {dim: 512, activation: swiglu} - - dim_model: 256 - max_context_length: 512 - num_layers: 6 - attention: {type: gqa, n_heads: 16, n_kv_heads: 4} - feed_forward: {dim: 1024, activation: swiglu} + "$choices": + - model_dim: 128 + max_context_length: 512 + num_layers: 4 + attention: + type: mha + num_heads: 8 + feed_forward: + activation: swiglu + hidden_dim: 512 + - model_dim: 256 + max_context_length: 512 + num_layers: 6 + attention: + type: gqa + num_heads: 16 + num_kv_heads: 4 + feed_forward: + activation: swiglu + hidden_dim: 1024 ``` Each choice replaces the complete overridden subtree, keeping coupled values valid. The same mechanism can select complete ingestion or decoder components, training phases, or other canonical subtrees. -Use `variants` when candidates should patch the base mapping and other sibling +Use `$variants` when candidates should patch the base mapping and other sibling fields should remain independent. A variant that changes a component's `type` starts from that new component shape rather than retaining fields belonging to the old type. @@ -111,10 +140,15 @@ parameters: interfaces: event_prediction: ingestion: - variants: - - {type: embedding, output_dim: 128} - - {type: passthrough, output_dim: 128} - dropout: [0.0, 0.1] + "$variants": + - type: embedding + output_dim: 128 + - type: passthrough + output_dim: 128 + dropout: + "$choices": + - 0.0 + - 0.1 ``` ## Search controls @@ -124,19 +158,19 @@ parameters: | `base_config_path` | `str` | Yes | - | Canonical training config used as the base. | | `parameters` | `mapping` | Yes | - | Recursive fixed values and search spaces. May be empty. | | `project_root` | `str` | No | Base value | Root written to every generated training config. | -| `additional_config_paths` | `str`, `list[str]`, or `null` | No | `null` | Complementary direct fragments of the search config. Relative paths resolve against the entry config's `project_root`; recursive composition and duplicate fields are rejected. | +| `additional_config_paths` | `list[str]` or `null` | No | `null` | Complementary direct fragments of the search config. Relative paths resolve against the entry config's `project_root`; recursive composition and duplicate fields are rejected. | | `name` | `str` | Yes | - | Study name and generated-run prefix. | -| `model_config_write_path` | `str` | Yes | - | Directory, under `project_root`, for generated trial configs. | -| `method` | `bayesian`, `sample`, or `grid` | No | `bayesian` | Optuna TPE, random, or exhaustive finite-grid sampling. | +| `trial_config_directory` | `str` | Yes | - | Directory, under `project_root`, for generated trial configs. | +| `method` | `bayesian`, `random`, or `grid` | No | `bayesian` | Optuna TPE, random, or exhaustive finite-grid sampling. | | `trials` | `int` | Except grid | - | Target total number of completed or pruned runs in the persisted study. | -| `global_seed` | `int` or `null` | No | `null` | Sampler seed. Training seeds belong in canonical `parameters`. | +| `sampler_seed` | `int` or `null` | No | `null` | Sampler seed. Training seeds belong in canonical `parameters`. | | `prune_trials` | `bool` | No | `true` | Enables cooperative pruning. Distributed pruning remains experimental. | | `pruning_warmup_epochs` | `int` or `null` | No | `null` | Complete epochs before pruning may begin. Mutually exclusive with batch warmup. | | `pruning_warmup_batches` | `int` or `null` | No | `null` | Training batches before pruning may begin. Mutually exclusive with epoch warmup. | | `evaluation_metrics` | `list[str]` or `null` | No | `null` | Metric names expected from a custom evaluation script. | -| `evaluation_metric_directions` | `list[minimize\|maximize]` or `null` | With metrics | `null` | One optimization direction per metric. | -| `evaluation_script` | `str` or `null` | With metrics | `null` | Script invoked with the best exported model's evaluation ID. | -| `evaluation_inference_config` | `str` or `null` | No | `null` | Inference config run before the custom evaluation script. | +| `evaluation_metric_directions` | `list[min\|max]` or `null` | With metrics | `null` | One optimization direction per metric. | +| `evaluation_script_path` | `str` or `null` | With metrics | `null` | Script invoked with the best exported model's evaluation ID. | +| `evaluation_inference_config_path` | `str` or `null` | No | `null` | Inference config run before the custom evaluation script. | For grid search, omitting `trials` runs the complete finite grid. If it is provided, it must exactly equal the grid size. For Bayesian and random search, @@ -152,10 +186,14 @@ metric it performs single-objective optimization; with several it records the Pareto front. ```yaml -evaluation_metrics: [accuracy, latency_ms] -evaluation_metric_directions: [maximize, minimize] -evaluation_inference_config: configs/infer-validation.yaml -evaluation_script: scripts/evaluate.py +evaluation_metrics: +- accuracy +- latency_ms +evaluation_metric_directions: +- max +- min +evaluation_inference_config_path: configs/infer-validation.yaml +evaluation_script_path: scripts/evaluate.py ``` The evaluation script receives the exported model's evaluation ID, formatted @@ -173,6 +211,6 @@ loading metadata-derived values, so all required authored fields must already be present. Generated canonical training configs are written below -`model_config_write_path`. The Optuna SQLite study is persisted at +`trial_config_directory`. The Optuna SQLite study is persisted at `state/optuna/[name].db` under `project_root`, allowing later invocations to continue toward the configured total. diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index 17544b00..2562bfea 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -11,11 +11,9 @@ configured static values for its `sequenceId`. Autoregressive generation retains these values while temporal inputs slide forward. -Models with image ingestion need `image_path` for the original image or video -folders selected by the model. Use a string for one temporal view or a map from -view name to folder for multiple views. When the model has a preconditioning -image encoder, also set `preconditioning_image_path` to a string for the -`default` view or a map for named views. Preprocessing metadata records view +Models with image ingestion need `image_paths` for the original image or video +folders selected by the model. Use a mapping from each temporal view name to its folder, even for one view. When the model has a preconditioning +image encoder, also set `preconditioning_image_paths` to a view-to-folder mapping, including a `default` key for an unnamed view. Preprocessing metadata records view contracts and signatures, but omits folder paths. The folders must remain available and unchanged after preprocessing. The inference loader applies the same frame-to-`itemPosition` or time-column alignment and image transform used @@ -24,8 +22,10 @@ PT and ONNX model execution accept image tensors and frame-valid masks; media decoding stays outside the exported model. ```yaml -image_path: {front: data/front_images, side: data/side_videos} -preconditioning_image_path: +image_paths: + front: data/front_images + side: data/side_videos +preconditioning_image_paths: front: data/sequence_front_images side: data/sequence_side_images ``` @@ -50,13 +50,14 @@ Exports embed their execution contract. Select a training route when providing model configuration through a training config: ```yaml -project_root: . -model_path: models/event-model-best-5.onnx +project_root: "." training_config_path: configs/train.yaml dataset: events data_path: data/events-test.parquet -model_type: generative device: cuda +task: forecast +model_paths: +- models/event-model-best-5.onnx ``` The route supplies columns, types, objective, window sizes, and preprocessing @@ -77,15 +78,15 @@ Benchmark the target workload rather than assuming either runtime is faster. Choose PT when portability matters less than a compact, self-contained config: ```yaml -project_root: . -model_path: models/event-model-best-5.pt +project_root: "." data_path: data/events-test.parquet -model_type: generative device: cuda +task: forecast +model_paths: +- models/event-model-best-5.pt ``` -For multi-interface PT, add `model_interface`. `model_type` stays explicit for -both formats because inference may generate outputs or extract embeddings. +For multi-interface PT, add `model_interface`. `task` is required for both formats: `forecast`, `policy`, or `embedding`. `project_root` is required (normally `.`), and `inference_batch_size` defaults to `1`. @@ -115,15 +116,14 @@ fragments or define the same field twice. | Field | Default | Purpose | | --- | --- | --- | -| `model_path` | Required | Model path, or a list of compatible model paths. | -| `model_type` | Required | `generative` or `embedding`. | +| `model_paths` | Required | Non-empty list of compatible model paths; policy requires exactly one. | +| `task` | Required | `forecast`, `policy`, or `embedding`. | | `device` | `cpu` | ONNX: `cpu` or `cuda`; PT also supports `mps`. ONNX CUDA requires an installed runtime exposing `CUDAExecutionProvider`. | | `project_root` | Required | Base for project paths; normally `.`. | | `data_path` | Metadata test/last split | Input file or folder. Usually required with artifact-only inference because artifacts do not store split paths. | -| `preprocessing_data_path` | `null` | Derives the generated metadata path. | | `metadata_config_path` | `null` | Explicit preprocessing metadata. | -| `image_path` | `null` | Original folder for one temporal view, or a view-to-folder map for every selected temporal view. Required when the model consumes temporal images. | -| `preconditioning_image_path` | `null` | Original folder for the `default` preconditioning view, or a view-to-folder map for named views. Required when the model uses preconditioning images. | +| `image_paths` | `null` | View-to-folder mapping for every selected temporal view. Required when the model consumes temporal images. | +| `preconditioning_image_paths` | `null` | View-to-folder mapping for all preconditioning views. Required when the model uses preconditioning images. | | `training_config_path` | `null` | Training config used to resolve a route. | | `dataset` / `part` | `null` | Select a dataset and optional part from the training config. | | `model_interface` | Implicit when unique | Select a route from a training config or portable artifact. | @@ -150,7 +150,7 @@ dataset metadata. For aligned splits, it sets the number of consecutive prediction groups evaluated per stored window. Aligned inference requires `window_stride: null` and a stored width of at least `context_length + target_offset + (prediction_alignment_steps - 1) × prediction_length`. -With `model_type: embedding`, aligned splits label each embedding by its target +With `task: embedding`, aligned splits label each embedding by its target `itemPosition` and omit positions outside the split. For causal models, the embedding represents the context used to predict that row. Other splits label embeddings by the input activation position. @@ -170,7 +170,7 @@ stored folder window if the remaining layout and semantics match. | `infer_with_dropout` | `false` | Enable dropout at inference; ONNX also requires dropout-preserving export. | | `enforce_determinism` | `false` | Request deterministic PyTorch algorithms. | | `seed` | `1010` | Random seed. | -| `autoregressive` | `false` | Feed predictions back for multi-step generation. | +| `autoregressive_generation` | `false` | Feed predictions back for multi-step generation. | | `generation_steps` | `null` | Required positive step count when autoregressive is enabled. | For an `autoregressive_transformer` decoder branch, same-position targets are @@ -189,10 +189,11 @@ its stored windows rather than this first-window reduction. ## CLI overrides -`--data-path`, `--input-columns`, `--metadata-config-path`, `--model-path`, -`--seed`, `--dataset`, `--part`, `--model-interface`, `--model-type`, +`--data-path`, `--input-columns`, `--metadata-config-path`, `--model-paths`, +`--seed`, `--dataset`, `--part`, `--model-interface`, `--task`, `--enforce-determinism`, and `--action-selection` override YAML values. -`--randomize` takes precedence over `--seed`. `--skip-metadata` skips external +`--seed random` resolves a fresh integer seed once; the resolved configuration records it. +`--enforce-determinism true|false` explicitly overrides the YAML boolean. `--skip-metadata` skips external metadata loading; inline metadata, a selected training route, or a self-describing PT artifact must then provide the required metadata. `--skip-metadata` is forecast-only and is rejected on policy inference. @@ -223,18 +224,19 @@ inference does not configure it again. Inference supplies the matching named layout tensors and masks through the preprocessed PT input. ```yaml -project_root: . -model_path: models/item-model-best-5.onnx -model_type: generative +project_root: "." read_format: pt data_path: data/raw-items-split2 device: cpu infer_with_dropout: false seed: 1010 +task: forecast +model_paths: +- models/item-model-best-5.onnx ``` -Use the corresponding embedding artifact with `model_type: embedding`, or change -`model_path` to a portable PT model. `read_format: pt` describes preprocessed +Use the corresponding embedding artifact with `task: embedding`, or change +`model_paths` to a portable PT model. `read_format: pt` describes preprocessed input; it is independent of the model artifact extension. Deep inputs retain `[B,T,D]` and explicit boolean depth masks through batching. New graphs accept symbolic batch size with fixed temporal/depth capacities, including a partial @@ -269,12 +271,12 @@ ORT installation. Outputs stay indexed by outer item coordinates. ## Policy model interfaces -With `model_type: generative`, the selected artifact interface determines the -default task. `task: policy` requires one policy-bound portable PT artifact; -`task: forecast` requests ordinary predictions, including from a policy-bound -interface with a supported forecast contract. Select `model_interface` when -ambiguous. `model_type` accepts only `generative` or `embedding`; use -`task: policy` for explicit policy execution. Embedding rejects `task`. The CLI accepts `--task policy|forecast`. +`task` explicitly selects the operation. `task: policy` requires one +policy-bound portable PT artifact. `task: forecast` requests ordinary predictions, +including from a policy-bound interface with a supported forecast contract. +`task: embedding` extracts representations. Select `model_interface` when ambiguous. +The artifact validates task support; it never chooses the operation implicitly. +The CLI uses `--task forecast|policy|embedding`. Policy input is raw CSV/Parquet observation rows. History alignment, codecs and normalization come from the artifact. Common fields cover model/interface/data @@ -293,7 +295,7 @@ selection, read/write formats, device, seed, `enforce_determinism` and batch siz Optional assertions `metadata_config_path`, `input_columns`, `static_columns`, `target_columns`, `column_data_types`, `target_column_types`, `training_objective`, -`context_length`, `target_offset`, `prediction_length` and `decoder_support` must +`context_length`, `target_offset`, `prediction_length` and `decoder_support_length` must match the selected artifact. Unknown/cross-route options fail even at default values. Policy rejects external per-column sampling, dropout, category decoding, dataset/part selection, image overrides, autoregression and `generation_steps`. diff --git a/documentation/configs/naming.md b/documentation/configs/naming.md new file mode 100644 index 00000000..b04160e2 --- /dev/null +++ b/documentation/configs/naming.md @@ -0,0 +1,202 @@ +# Configuration naming reference + +Sequifier accepts one canonical spelling and location for each authored setting. +Removed names produce replacement guidance; they are not compatibility aliases. +These are breaking configuration changes. Update authored files and any embedded +configuration used to reconstruct a model or resume a run before reusing them. +Settings not listed here retain their names, scopes, and defaults. + +## Named structures and fixed shapes + +Use `model.interfaces.`, `dataset_training..parts.`, +`policy_bindings.`, and `training_plan.phases` for every run size. +`default` is an ordinary name, not an implicit authoring syntax. Omitted unique +references can still be inferred. Direct `training_plan.epochs`, singular +`model.interface`, `dataset`, dataset `part`, and training `policy_binding` are +rejected. Sequence import still has one singular `policy_binding`. + +Supervised evaluation is an explicit `{sources, monitor?}` declaration or `null`; +boolean shorthand is rejected. Optimizers and schedulers use `{name, arguments}`. +Initialization methods use `{method, ...}`, including parameterless methods. + +`model_paths` is always a non-empty list. `image_paths`, +`preconditioning_image_paths`, and preprocessing `preconditioning_image.paths` +are view-to-path mappings even for a single view. `additional_config_paths` is +a list or `null`. Preprocessing `window_lengths` supplies one width per split. +Generated metadata retains its selected-split `window_length`. + +## Dimensions, counts, and selection + +| Previous spelling | Canonical spelling | +| --- | --- | +| `dim_model` | `model_dim` | +| `feed_forward.dim` | `feed_forward.hidden_dim` | +| `n_heads` / `n_kv_heads` | `num_heads` / `num_kv_heads` | +| Static `embedding_size` | `total_embedding_dim` | +| `positional_encoding_scope` | `position_encoding_scope` | +| `post_conv_norm` | `post_conv_normalization` | +| `frame_pool` / `image_frame_pool` | `frame_pooling` / `image_frame_pooling` | +| Axis-pooling `mode` | `pooling` | +| Decoder `support` | `support_length` | +| Inference `decoder_support` | `decoder_support_length` | +| Depth-layout `context_length` | `max_depth_length` | +| `device_max_concat_length` | `max_concat_tensors` | +| Preprocessing `n_cores` | `num_workers` | +| Source `weight` | `selection_weight` | + +`model_dim`, `hidden_dim`, `output_dim`, `cell_dim`, `hidden_dims`, and +`feature_embedding_dims` describe different representation widths. +`total_embedding_dim` allocates aggregate static feature width. `context_length` +is temporal model input length; `max_context_length` is backbone capacity; +`max_depth_length` is child-collection capacity. `support_length` counts temporal +representations consumed by a decoder; `prediction_length` counts decoded positions. + +Source `selection_weight` changes ordering and burst frequency, not the total +amount consumed in an epoch. Class weights and loss weights retain their separate +loss-scaling meanings. + +## Training clocks and optimization + +A batch is one forward/backward microbatch. An optimizer step applies gradients +for a role. An iteration is a complete policy transaction: imitation, IQL, and +DPO perform one optimizer step per role; PPO and GRPO may perform several. +An epoch is one pass through the applicable data. Environment steps count +transitions; generation steps count temporal autoregressive output steps. + +| Previous spelling | Canonical spelling | +| --- | --- | +| `accumulation_steps` | `accumulation_batches` | +| PPO `minibatch_size` | `batch_size` | +| IQL/DPO `updates` | `iterations` | +| GRPO `groups_per_update` | `groups_per_iteration` | +| PPO `rollout_length` | `rollout_env_steps` | +| Supervised `evaluation_interval` | `evaluation_interval_epochs` | +| Policy `evaluation_interval` | `evaluation_interval_iterations` | +| Supervised `checkpoint_interval` | `checkpoint_interval_epochs` | +| Policy `checkpoint_interval` | `checkpoint_interval_iterations` | +| `log_interval` | `log_interval_batches` | +| `early_stopping_epochs` | `early_stopping_patience_epochs` | +| Supervised `scheduler_step_on: batch` | `scheduler_step_on: optimizer_step` | +| `gradient_clip` | `gradient_clip_norm` | +| PPO `value_clip` | `value_clip_delta` | +| PPO `target_kl` | `old_policy_target_kl` | +| GRPO `kl_coefficient` | `reference_kl_coefficient` | +| IQL `max_weight` | `max_advantage_weight` | + +Keep `epochs`, `optimization_epochs`, `total_env_steps`, `max_env_steps`, +`total_groups`, `group_size`, `max_steps`, `max_episode_steps`, and +`generation_steps`: their enclosing contracts identify different budgets. +`batch_size` always describes a microbatch; its elements are examples, +transitions, preference pairs, or trajectories depending on the algorithm. + +Keep optimizer roles `model`, `actor`, `value`, and `critic`. Third-party +constructor names and arguments remain unchanged under `arguments`. +`learning_rate` is Sequifier-owned; optimizer constructor `lr` is forbidden. +Scheduler `total_steps` remains an assertion against the resolved scheduler budget. + +## Freezing, checkpoints, and direction + +The dataset `freeze` container remains. Inside its component tree, `freeze` +becomes `freeze_groups` and `freezing_except` becomes `freeze_except_groups`. +Policy phase `freeze` becomes `freeze_modules`: its entries are actual module +paths, not semantic group selectors. Empty `freeze_groups` unfreezes a scope; +empty `freeze_except_groups` freezes it. + +| Previous spelling | Canonical spelling | +| --- | --- | +| `save_interval_batches` | `snapshot_interval_batches` | +| `save_interval_minutes` | `snapshot_interval_minutes` | +| `save_latest_interval_minutes` | `latest_checkpoint_interval_minutes` | +| `save_interval_val_loss` | `evaluate_on_snapshot` | +| `calculate_validation_loss_on_initialization` | `evaluate_before_training` | +| Monitor `mode` | `direction: min` or `direction: max` | + +Snapshots are retained checkpoints triggered by the global batch/time intervals. +Refreshing `latest` is distinct. `resume`, `initialize_from`, `optimizer_state`, +and `handoff` keep their separate lifecycle meanings. + +## Inference, sources, and seeds + +Inference requires `task: forecast`, `task: policy`, or `task: embedding`. +`model_type` is removed. The artifact validates the requested task; it never +chooses the task implicitly. Policy requires exactly one entry in `model_paths`. +`autoregressive` becomes `autoregressive_generation`. Inference's +`preprocessing_data_path` metadata shortcut is removed; use `metadata_config_path`. + +| Previous spelling | Canonical spelling | +| --- | --- | +| Inference `model_path` | `model_paths` | +| `image_path` | `image_paths` | +| `preconditioning_image_path` | `preconditioning_image_paths` | +| Preconditioning image `path` | `paths` | +| Global interface `metadata_config_path` | `interface_metadata_paths` | +| Sequence/preference `manifest` | `manifest_path` | + +`interface_metadata_paths` maps interface names to files. Per-part and inference +`metadata_config_path` still select one file. Source-local `project_root` is +removed; the run supplies it during resolution. + +Run `seed` accepts an integer or `random`. Random seeds resolve once to an integer +recorded in the effective configuration. `sampler_seed` controls an independent +search stream. Initialization and hash seeds stay scoped integer declarations. +`enforce_determinism`, `action_selection`, and `seed` remain independent controls. + +Keep `seed_window_index` and `seed_row_index`, model `previous_action_inputs` and +data `previous_action_columns`, scalar `actions` and structured `action_record`, +and model-owned versus fixed policy `scale`: these are meaningful distinctions. + +## Search expressions + +| Previous spelling | Canonical spelling | +| --- | --- | +| `global_seed` | `sampler_seed` | +| `method: sample` | `method: random` | +| `model_config_write_path` | `trial_config_directory` | +| `evaluation_script` | `evaluation_script_path` | +| `evaluation_inference_config` | `evaluation_inference_config_path` | +| `choices`, implicit categorical list | `$choices` | +| `fixed` | `$fixed` | +| `variants` | `$variants` | +| Bare numeric range mapping | `$range: {low, high, step?, log?, type?}` | + +Primitive scalars are fixed replacements; ordinary mappings recursively patch. +`$fixed` replaces a complete mapping or list. `$choices` samples any complete +value. `$variants` patches with selected partial mappings and may have sampled +siblings. Integer list keys patch indexed entries. Bare lists are rejected. +Search operators reserve `$` and never depend on the base field's type. + +`evaluation_metric_directions` uses `min` and `max`, matching monitor `direction`. +Training parameter paths are exactly the canonical training paths. + +## Cardinality, initialization, and dtype values + +| Previous spelling | Canonical spelling | +| --- | --- | +| `num_buckets`, `buckets` | `num_buckets` | +| `min_freq`, `min_count` | `min_count` | +| `top_k`, `k`, `max_categories` | `max_categories` | +| `column_types`, `column_data_types` | `column_data_types` | +| `xavier`, `glorot`, `glorot_uniform`, `xavier_uniform` | `xavier_uniform` | +| `glorot_normal`, `xavier_normal` | `xavier_normal` | +| Distribution `standard_deviation` | `std` | + +Numeric storage dtypes use canonical `Float32`, `Int64`, etc.; lowercase aliases +are rejected. Compute dtypes remain PyTorch-style `float32`, `bfloat16`, etc. +Standard methods such as `zeros`, `ones`, and `constant` remain separate methods, +even when some parameterizations are mathematically equivalent. + +Probability types describe their unchanged mathematics: + +| Previous type | Canonical type | Definition | +| --- | --- | --- | +| `normal_discretized_floor` | `clipped_rounded_normal_plus_one` | `max(round(Normal(mean, std)), 0) + 1` | +| `log_normal_discretized_floor` | `rounded_log_normal_plus_one` | `round(exp(Normal(mean, std))) + 1` | +| `poisson_floor` | `poisson_plus_one` | `Poisson(rate) + 1` | + +## CLI + +CLI flags use the YAML name with hyphens: preprocessing +`--preprocessing-data-path`, inference `--model-paths`, and +`--task forecast|policy|embedding`. All short aliases and abbreviated flags are +removed. `--randomize` becomes `--seed random`. Boolean configuration overrides +take explicit `true` or `false`, including `--enforce-determinism false`. diff --git a/documentation/configs/preprocess.md b/documentation/configs/preprocess.md index 56541404..66632d24 100644 --- a/documentation/configs/preprocess.md +++ b/documentation/configs/preprocess.md @@ -14,14 +14,13 @@ Values passed on the command line override the YAML before validation. | Flag | Overrides / Action | | :--- | :--- | -| `-r`, `--randomize` | Generates a random `seed`. The seed affects `between_sequence` split assignment. | -| `-dp`, `--data-path` | Overrides `preprocessing_data_path`. | -| `-sc`, `--selected-columns` | Overrides `selected_columns` with a space-separated list. Use `None` to process all columns. | +| `--seed` | Integer run seed, or `random` to resolve a fresh seed once. Affects `between_sequence` split assignment. | +| `--preprocessing-data-path` | Overrides `preprocessing_data_path`. | +| `--selected-columns` | Overrides `selected_columns` with a space-separated list. Use `None` to process all columns. | ## Composable Configuration Files -A preprocessing entry config may set `additional_config_paths` to one -non-empty string, a list of non-empty strings, or `null`. Relative paths +A preprocessing entry config may set `additional_config_paths` to a list of non-empty strings or `null`. Relative paths resolve against the entry config's `project_root`; absolute paths are used directly. Fragments are direct only and cannot include further fragments. They may share nested containers when their child fields are disjoint, but duplicate @@ -36,7 +35,7 @@ The configuration is defined in a YAML file (e.g., `preprocess.yaml`). Below are | Field | Type | Mandatory | Default | Description | | :--- | :--- | :--- | :--- | :--- | | `project_root` | `str` | **Yes** | - | The root directory of your Sequifier project. Usually `.` | -| `additional_config_paths` | `str`, `list[str]`, or `null` | No | `null` | Direct complementary YAML fragments. Relative paths resolve against `project_root`; recursive composition and duplicate fields are rejected. | +| `additional_config_paths` | `list[str]` or `null` | No | `null` | Direct complementary YAML fragments. Relative paths resolve against `project_root`; recursive composition and duplicate fields are rejected. | | `preprocessing_data_path` | `str` | **Yes** | - | Path to the raw input file or folder. | | `static_data_path` | `str` | No | `null` | Separate CSV or Parquet table with exactly one row per `sequenceId`. Preprocessing requires its ID set to match the sequence data in both directions and writes a processed static feature table. | | `static_categorical_columns` | `list[str]` | No | `[]` | Static columns to treat as categorical when their source dtype would not make that intent clear. | @@ -71,20 +70,24 @@ flat folder per named view. Untimed images are named one `{sequence_id}` video per sequence. Decoded video frames are numbered from zero in decode order. A separate `preconditioning_image` source contains one `{sequence_id}` image per sequence in each folder. Its -`path` can be one folder or a map of named view folders. +`paths` maps view names to folders, even for a single view. ```yaml image_sources: front: path: data/front_images type: images - alignment: {frames: 3, items: 2} + alignment: + frames: 3 + items: 2 side: path: data/side_videos type: videos - alignment: {frames: 1, items: 2} + alignment: + frames: 1 + items: 2 preconditioning_image: - path: + paths: front: data/sequence_front_images side: data/sequence_side_images ``` @@ -196,8 +199,7 @@ depth feature/position column. Names beginning with | Field | Type | Mandatory | Default | Description | | :--- | :--- | :--- | :--- | :--- | -| `window_length` | `int` | Conditional | - | One physical width for every split. Specify exactly one of `window_length` or `window_lengths`. | -| `window_lengths` | `list[int]` | Conditional | - | Physical widths in split order, with one entry per split. Required when `prediction_alignment_steps > 1`; widths from split 2 onward must equal the first width plus `prediction_alignment_steps - 1`. | +| `window_lengths` | `list[int]` | **Yes** | - | Physical widths in split order, with one entry per split. Required for every split; widths from split 2 onward must equal the first width plus `prediction_alignment_steps - 1`. | | `max_target_offset` | `int` | No | `1` | Number of future items retained after the model input window. Use `0` for BERT-style same-width inputs and targets; use `1` for causal next-item training. | | `split_ratios` | `list[float]`| Conditional | `null` | Ordered split proportions for `within_sequence` and `between_sequence`. Must sum to 1.0 and must be omitted for `value_cutoff`. | | `split_method` | `str` | No | `within_sequence` | How rows are assigned to splits: `within_sequence`, `between_sequence`, or `value_cutoff`. | @@ -230,10 +232,12 @@ To align validation or test predictions exactly to their split positions, config those split indices and the model's prediction view: ```yaml -window_length: 129 +window_lengths: [129, 129, 129] max_target_offset: 1 window_stride: 128 -prediction_aligned_splits: [1, 2] +prediction_aligned_splits: +- 1 +- 2 prediction_length: 2 target_offset: 1 ``` @@ -265,7 +269,7 @@ PT output. | Field | Type | Mandatory | Default | Description | | :--- | :--- | :--- | :--- | :--- | | `seed` | `int` | No | `1010` | Random seed for reproducibility. | -| `n_cores` | `int` | No | Max Cores | Maximum worker processes for eligible preprocessing stages. Depth preprocessing parallelizes a single Parquet file only when it has multiple row groups and `max_rows` is unset. | +| `num_workers` | `int` | No | Max Cores | Maximum worker processes for eligible preprocessing stages. Depth preprocessing parallelizes a single Parquet file only when it has multiple row groups and `max_rows` is unset. | | `batches_per_file` | `int` | No | `1024` | Buffer flush threshold per split. For flat input, counts sequence-window groups in PT and Parquet output (including merged output); a byte limit can flush earlier. For depth input, counts windows in PT output. | | `process_by_file` | `bool` | No | `true` | Memory optimization. If `true`, processes one input file at a time. | @@ -296,7 +300,7 @@ training and inference also check the width against the model's `context_length` To use the entire window, choose its width as `context_length + target_offset + (prediction_alignment_steps - 1) × prediction_length`. Earlier positions outside the split are masked from prediction output. -Set `window_lengths` when `prediction_alignment_steps > 1`. The first two +Set `window_lengths` explicitly for every split. The first two split widths can differ; every later split width must equal the first width plus `prediction_alignment_steps - 1`, even when that split uses distributed placement. @@ -306,9 +310,14 @@ the following gives aligned splits room for three two-position prediction groups: ```yaml -window_lengths: [131, 133, 133] +window_lengths: +- 131 +- 133 +- 133 max_target_offset: 1 -prediction_aligned_splits: [1, 2] +prediction_aligned_splits: +- 1 +- 2 prediction_length: 2 prediction_alignment_steps: 3 target_offset: 1 @@ -316,9 +325,9 @@ target_offset: 1 | Scenario | Suggested settings | Trade-off | | --- | --- | --- | -| Many short or varied-length sequences | Store the minimum width (for example, `window_length: 129` for `context_length: 128`, `max_target_offset: 1`), use a preprocessing `window_stride` near 128, and set training `window_stride: null`. | Limits padding and stores roughly one copy of long sequences, but targets near window boundaries have less preceding context. | +| Many short or varied-length sequences | Store the minimum width (for example, `window_lengths: [129, 129, 129]` for `context_length: 128`, `max_target_offset: 1`), use a preprocessing `window_stride` near 128, and set training `window_stride: null`. | Limits padding and stores roughly one copy of long sequences, but targets near window boundaries have less preceding context. | | More overlap during training | Keep that width; reduce the preprocessing `window_stride` to a fraction of the training `context_length`. | Roughly 2× or 4× as many stored events for long sequences. | -| Long sequences, several model views per stored window | Use a longer stored width (for example, `window_length: 513`, `window_stride: 384`, `context_length: 128`, training config `window_stride: 128`). | About 1.3× stored events on long sequences; short sequences pad to 513, and more model views cost more compute. | +| Long sequences, several model views per stored window | Use a longer stored width (for example, `window_lengths: [513, 513, 513]`, `window_stride: 384`, `context_length: 128`, training config `window_stride: 128`). | About 1.3× stored events on long sequences; short sequences pad to 513, and more model views cost more compute. | | Dense evaluation with nearly full preceding context at each window's right edge | Use `prediction_aligned_splits` for exact split coverage, or a small preprocessing `window_stride` for distributed evaluation. | More stored windows and more evaluation compute. | Self-attention lets the model use relationships between positions within a @@ -351,17 +360,18 @@ categorical column. Each key is a column name. For example, keep the 10,000 most frequent product IDs and hash every other product ID into 4,096 buckets: ```yaml -categorical_columns: [product_id] +categorical_columns: +- product_id cardinality_config: product_id: - top_k: 10000 + max_categories: 10000 hashing: num_buckets: 4096 seed: 42 ``` -`top_k` retains at most that many values, ranked by frequency. Alternatively, -`min_freq` retains every value occurring at least that many times. Both must be +`max_categories` retains at most that many values, ranked by frequency. Alternatively, +`min_count` retains every value occurring at least that many times. Both must be positive integers and cannot be used together. Retained values keep individual IDs; all remaining non-null values, including values first seen in later splits, map to a hash bucket when `hashing` is configured. Without `hashing`, @@ -422,7 +432,7 @@ After running `preprocess`, the following are generated: 1. **Data Files:** Located in `data/`. Depending on your configuration, these will be merged files such as `[NAME]-split0.parquet` (Training), `[NAME]-split1.parquet` (Validation), etc., or split folders such as `[NAME]-split0/` containing `.pt` or `.parquet` shards. 2. **Metadata Config:** Located in `configs/metadata_configs/[NAME].json`. * **Crucial:** This file contains the integer mappings for categorical variables (`id_maps`), statistics for real variables (`selected_columns_statistics`), and whether those variables were normalized (`normalize_real_columns`). - * **Next Step:** Reference this file from `dataset.part.metadata_config_path` in a singleton training config, or from `dataset_training..parts..metadata_config_path` in a named training config. In inference, either `preprocessing_data_path` or `metadata_config_path` can locate the metadata and its split paths. + * **Next Step:** Reference this file from `dataset_training..parts..metadata_config_path` in a training config. In inference, use `metadata_config_path` to locate external metadata and its split paths. ## Named depth layouts @@ -431,23 +441,35 @@ Depth behavior is explicit; a column named `subItemPosition` alone remains an ordinary flat feature. ```yaml -project_root: . +project_root: "." preprocessing_data_path: data/raw-items read_format: parquet write_format: pt merge_output: false -selected_columns: [accountType, subitemType, subitemAmount, nextAction] +selected_columns: +- accountType +- subitemType +- subitemAmount +- nextAction depth_layouts: subitems: position_column: subItemPosition - columns: [subitemType, subitemAmount] - context_length: 16 + columns: + - subitemType + - subitemAmount position_base: 0 allow_gaps: false -window_length: 129 + max_depth_length: 16 max_target_offset: 1 -split_ratios: [0.8, 0.1, 0.1] +split_ratios: +- 0.8 +- 0.1 +- 0.1 window_stride: 128 +window_lengths: +- 129 +- 129 +- 129 ``` Every file must contain the depth position column, which is read automatically @@ -472,7 +494,7 @@ statistics count occupied child slots. Both populations are selected before split extraction. Materialization encodes each item once and uses bounded windows and output batches; `batches_per_file` bounds the number of windows accumulated per split. For a single Parquet file with multiple row groups and -no `max_rows`, this adapter uses `n_cores` to parallelize auditing, metadata +no `max_rows`, this adapter uses `num_workers` to parallelize auditing, metadata fitting, and window materialization. Other depth inputs use the sequential merge path. diff --git a/documentation/configs/train.md b/documentation/configs/train.md index b25e8598..9a2b87ee 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -12,13 +12,12 @@ owns architecture. sequifier train --config-path configs/train.yaml ``` -## Singleton configuration +## Minimal named configuration -When a run has one model interface, one dataset, one part, and one phase, those -values can be authored directly without routing names or references: +Use the same named structure for one or many interfaces, datasets, parts, and phases: ```yaml -project_root: . +project_root: "." model_name: event-model device: cuda seed: 1010 @@ -30,17 +29,17 @@ global_training: model: backbone: architecture: - dim_model: 128 + model_dim: 128 max_context_length: 512 num_layers: 6 attention: type: mha - n_heads: 8 - n_kv_heads: 8 + num_heads: 8 + num_kv_heads: 8 output_projection: true feed_forward: - dim: 512 activation: swiglu + hidden_dim: 512 normalization: type: rmsnorm norm_first: true @@ -49,23 +48,19 @@ model: theta: 10000 dropout: 0.1 shared_layer_groups: [] - interface: - input_columns: - - event - target_columns: - - event - ingestion: - type: embedding - output_dim: 128 - decoder: - type: linear - prediction_length: 1 - support: 1 -dataset: - part: - metadata_config_path: configs/metadata_configs/events.json - criterion: - event: CrossEntropyLoss + interfaces: + default: + input_columns: + - event + target_columns: + - event + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 + support_length: 1 training_plan: phases: - epochs: 5 @@ -76,36 +71,48 @@ training_plan: optimization: model: learning_rate: 0.0001 - evaluation: true + evaluation: + sources: + - source: default.default +dataset_training: + default: + criterion: + event: CrossEntropyLoss + parts: + default: + metadata_config_path: configs/metadata_configs/events.json ``` -The concise form expands before validation: - -| Authored field | Canonical form | -| --- | --- | -| `model.interface` | `model.interfaces.default` | -| `dataset.part` | `dataset_training.default.parts.default` | -| `training_plan.epochs` | One sequential phase named `train` | -| Phase `evaluation: true` | Evaluate the phase's inferred single source | - -These shortcuts can be used independently. A unique interface and training -source are inferred; multiple interfaces or datasets require explicit names and -references. Singular and plural spellings cannot be combined. Because errors -are reported after expansion, they may use the canonical paths above. +All configurations use `model.interfaces`, `dataset_training`, dataset `parts`, +`policy_bindings` where applicable, and `training_plan.phases`. A unique interface +and source may be inferred when omitted. Singular aliases and direct phase fields +are rejected with canonical replacement paths. See the [naming reference](naming.md). To condition every model window on features from a separate per-sequence table, preprocess with `static_data_path` and configure the interface: ```yaml model: - interface: - input_columns: [event] - static_conditioning: - columns: [account_type, account_age] - feature_embedding_dims: {account_type: 16, account_age: 8} - target_columns: [event] - ingestion: {type: embedding, output_dim: 128} - decoder: {type: linear, prediction_length: 1, support: 1} + interfaces: + default: + input_columns: + - event + static_conditioning: + columns: + - account_type + - account_age + feature_embedding_dims: + account_type: 16 + account_age: 8 + target_columns: + - event + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 + support_length: 1 ``` The static columns are separate from temporal `input_columns` and targets. @@ -113,11 +120,11 @@ They are looked up by `sequenceId` for every sampled window, including windows from the middle of a sequence. `feature_embedding_dims` assigns a width to each static feature; real features with width greater than one use a learned projection. Alternatively, -`embedding_size` distributes a total width automatically when all static +`total_embedding_dim` distributes a total width automatically when all static features are categorical or all are real. For mixed types, specify `feature_embedding_dims`. If neither option is set, categorical widths use the existing cardinality-based default and real features use one channel. -The concatenated features are projected to the backbone `dim_model`. +The concatenated features are projected to the backbone `model_dim`. The model creates one prefix token at runtime and removes its output before decoding. Set backbone `max_context_length` to at least `global_training.context_length + 1`. @@ -125,7 +132,7 @@ at least `global_training.context_length + 1`. ## Canonical configuration ```yaml -project_root: . +project_root: "." model_name: event-model device: cuda seed: 1010 @@ -139,17 +146,17 @@ global_training: model: backbone: architecture: - dim_model: 128 + model_dim: 128 max_context_length: 512 num_layers: 6 attention: type: mha - n_heads: 8 - n_kv_heads: 8 + num_heads: 8 + num_kv_heads: 8 output_projection: true feed_forward: - dim: 512 activation: swiglu + hidden_dim: 512 normalization: type: rmsnorm norm_first: true @@ -173,7 +180,7 @@ model: decoder: type: linear prediction_length: 1 - support: 1 + support_length: 1 dataset_training: events: model_interface: event_prediction @@ -188,7 +195,7 @@ dataset_training: event: 1.0 freeze: backbone: - freeze: + freeze_groups: - attention.qkv training_plan: phases: @@ -198,22 +205,25 @@ training_plan: sources: - source: events.increment batch_size: 64 - accumulation_steps: 4 + accumulation_batches: 4 optimization: model: learning_rate: 0.0001 optimizer: name: AdamW - weight_decay: 0.01 + arguments: + weight_decay: 0.01 scheduler: name: StepLR - step_size: 1 - gamma: 0.99 + arguments: + step_size: 1 + gamma: 0.99 scheduler_step_on: epoch - gradient_clip: 1.0 + gradient_clip_norm: 1.0 optimizer_state: reset evaluation: - sources: [{source: events}] + sources: + - source: events - name: complete_retraining epochs: 5 mode: interleaved @@ -222,22 +232,25 @@ training_plan: - source: events batches_per_selection: 4 batch_size: 64 - accumulation_steps: 4 + accumulation_batches: 4 optimization: model: learning_rate: 0.0001 optimizer: name: AdamW - weight_decay: 0.01 + arguments: + weight_decay: 0.01 scheduler: name: StepLR - step_size: 1 - gamma: 0.99 + arguments: + step_size: 1 + gamma: 0.99 scheduler_step_on: epoch - gradient_clip: 1.0 + gradient_clip_norm: 1.0 optimizer_state: reset evaluation: - sources: [{source: events}] + sources: + - source: events export_generative_model: true export_embedding_model: false export_onnx: true @@ -278,7 +291,8 @@ behavior. For example: ```yaml dataset_training: events: - criterion: {event: AccurateCrossEntropyLoss} + criterion: + event: AccurateCrossEntropyLoss ``` The choice is per target and is checked when resuming a run. Class @@ -307,8 +321,11 @@ training config with a backbone and dataset part: model: interfaces: default: - input_columns: [accountId, merchantId] - target_columns: [accountId] + input_columns: + - accountId + - merchantId + target_columns: + - accountId categorical_hashing: accountId: type: multi_hash @@ -324,7 +341,7 @@ model: decoder: type: linear prediction_length: 1 - support: 1 + support_length: 1 dataset_training: default: model_interface: default @@ -384,14 +401,19 @@ training is disabled. When disabled, curriculum values are ignored and the ordinary sample shuffle is preserved. ```yaml -dataset: - part: - metadata_config_path: configs/metadata_configs/events.json - file_order: name training_plan: curriculum_training: true curriculum_column: difficulty - epochs: 5 + phases: + - epochs: 5 + name: train + mode: sequential +dataset_training: + default: + parts: + default: + metadata_config_path: configs/metadata_configs/events.json + file_order: name ``` `loss_weights` scales each target's contribution to the training and reported @@ -425,13 +447,15 @@ training_plan: ndim: 2 optimizer: name: Muon - momentum: 0.95 + arguments: + momentum: 0.95 learning_rate: 0.02 - id: remainder select: otherwise optimizer: name: AdamW - weight_decay: 0.01 + arguments: + weight_decay: 0.01 learning_rate: 0.0003 ``` @@ -457,7 +481,7 @@ With `CyclicLR`, each group's `learning_rate` is also its peak rate. Add a `base_learning_rate` below that peak to every optimizer group, and omit `scheduler.base_lr` and `scheduler.max_lr`. For the plan above, the two base rates could be `0.002` and `0.00003`, respectively. Configure the scheduler -with `name: CyclicLR`, `step_size_up: 2000`, and `scheduler_step_on: batch`. +with `name: CyclicLR`, `step_size_up: 2000`, and `scheduler_step_on: optimizer_step`. Momentum cycling defaults to off for both cycle schedulers in plan mode; explicit `cycle_momentum: true` is unsupported. `base_learning_rate` is only valid with `CyclicLR`. @@ -465,34 +489,40 @@ valid with `CyclicLR`. ## Canonical training settings Batching and optimization are configured only on phases. Global aliases for -`batch_size`, `accumulation_steps`, `learning_rate`, `optimizer`, `scheduler`, -`scheduler_step_on`, and `gradient_clip` are rejected. CLI overrides use +`batch_size`, `accumulation_batches`, `learning_rate`, `optimizer`, `scheduler`, +`scheduler_step_on`, and `gradient_clip_norm` are rejected. CLI overrides use `--global-training` and `--training-plan` YAML mappings with the same schema. -Every phase owns `evaluation`, `evaluation_interval`, and `checkpoint_interval`. -For supervised training, intervals count completed phase epochs; for policy -training, they count algorithm transactions. The final epoch/update always -checks evaluation and saves a checkpoint. Supervised `evaluation` is optional; -`evaluation: true` selects the phase's unique training source. Multiple sources -require an explicit monitor for best handoff, validation saving or early stopping. -There is no top-level `evaluation` declaration. +Every phase owns evaluation and checkpoint cadence. Supervised phases use +`evaluation_interval_epochs` and `checkpoint_interval_epochs`; policy phases use +`evaluation_interval_iterations` and `checkpoint_interval_iterations`. A policy +iteration is one complete algorithm transaction and can contain several optimizer +steps. Final epochs/iterations always evaluate when enabled and save a checkpoint. +Supervised `evaluation` is an explicit declaration or `null`; boolean shorthand +and top-level evaluation are rejected. Multiple sources require an explicit monitor +for best handoff, validation saving, or early stopping. ```yaml training_plan: phases: - - name: fit - algorithm: supervised - epochs: 10 - mode: sequential - sources: [{source: observations.train}] - evaluation: - sources: [{source: observations.validation}] - monitor: {source: observations.validation, metric: loss, mode: min} - evaluation_interval: 1 - checkpoint_interval: 2 + - name: fit + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - source: observations.train + evaluation: + sources: + - source: observations.validation + monitor: + source: observations.validation + metric: loss + direction: min + evaluation_interval_epochs: 1 + checkpoint_interval_epochs: 2 ``` -Epoch-based early stopping requires `evaluation_interval: 1`. +Epoch-based early stopping requires `evaluation_interval_epochs: 1`. Time- and batch-based recovery checkpoints remain global execution settings. Frozen normalization belongs to `global_training.normalization` or the incoming artifact; phases cannot redeclare it. @@ -507,14 +537,21 @@ They never create, overwrite or remove model parameter declarations. model: interfaces: actor: - # Other interface fields omitted. prediction_distributions: throttle: - scale: {initial: 0.2, trainable: true, min: 0.0001, max: 100} + scale: + initial: 0.2 + trainable: true + min: 0.0001 + max: 100 policy_bindings: actor: actions: - throttle: {type: real, scale: model, lower: 0, upper: 1} + throttle: + type: real + scale: model + lower: 0 + upper: 1 ``` A real action must explicitly choose `scale: model` or `scale: {fixed: 0.1}`. @@ -530,33 +567,40 @@ must match; only newly declared scale parameters receive fresh initialization. ## Optimization across phases -Each phase owns `batch_size` (32 by default), `accumulation_steps` (1), and +Each phase owns `batch_size` (32 by default), `accumulation_batches` (1), and `optimization.`. Supervised phases have the single role `model`, meaning the trainable model parameters. Imitation, DPO and GRPO have `actor`; PPO also has `value`; IQL has `actor`, `value`, and `critic`. Unknown roles fail validation. Omitted roles use Adam, learning rate 0.0003 and gradient clip 1.0. Set -`gradient_clip: null` to disable clipping. Optimizer plans take their rates from +`gradient_clip_norm: null` to disable clipping. Optimizer plans take their rates from their groups. Supervised roles default to StepLR (`step_size: 1`, `gamma: 0.99`); policy roles default to ConstantLR (`factor: 1`, `total_iters: 1`). ```yaml training_plan: phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential - sources: [{source: observations.train}] - batch_size: 64 - accumulation_steps: 1 - optimizer_state: reset - optimization: - model: - optimizer: {name: Adam} - learning_rate: 0.001 - scheduler: {name: StepLR, arguments: {step_size: 1, gamma: 0.99}} - scheduler_step_on: epoch - gradient_clip: 1.0 + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - source: observations.train + batch_size: 64 + accumulation_batches: 1 + optimizer_state: reset + optimization: + model: + optimizer: + name: Adam + arguments: {} + learning_rate: 0.001 + scheduler: + name: StepLR + arguments: + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip_norm: 1.0 ``` `optimizer_state: reset | continue` describes the transition **into** this phase; @@ -573,7 +617,7 @@ best weights before starting the next phase; its monitor state is phase-local. Supervised `scheduler_step_on: epoch` advances once after a completed phase epoch; ReduceLROnPlateau receives the configured evaluation monitor at that boundary -and requires the phase's `evaluation.monitor` and `evaluation_interval: 1`. `batch` advances once per applied optimizer +and requires the phase's `evaluation.monitor` and `evaluation_interval_epochs: 1`. `optimizer_step` advances once per applied optimizer update, after accumulation, rather than once per microbatch. Policy roles only accept `scheduler_step_on: optimizer_step`, their default. Policy plateau and compound schedulers are unsupported. Constructor arguments are validated before @@ -592,16 +636,16 @@ Policy OneCycleLR derives the maximum optimizer-update budget below. An explicit | Algorithm | Optimizer updates per role | | --- | --- | -| Imitation | epochs × ceil(training examples / (batch_size × accumulation_steps)) | -| IQL / DPO | updates | -| PPO | optimization_epochs × sum(ceil(rollout size / (minibatch_size × accumulation_steps))) | -| GRPO | optimization_epochs × sum(ceil(group trajectories / (batch_size × accumulation_steps))) | +| Imitation | epochs × ceil(training examples / (batch_size × accumulation_batches)) | +| IQL / DPO | iterations | +| PPO | optimization_epochs × sum(ceil(rollout size / (batch_size × accumulation_batches))) | +| GRPO | optimization_epochs × sum(ceil(group trajectories / (batch_size × accumulation_batches))) | -PPO `rollout_length` counts collected transitions; `minibatch_size` counts +PPO `rollout_env_steps` counts collected transitions; `batch_size` counts transitions in each optimization microbatch. They are distinct budgets. IQL uses `batch_size: 64` by default and one update per sampled batch; it does not accept accumulation. DPO counts preference pairs; GRPO counts trajectories. -PPO defaults to `minibatch_size: 64`; other batched algorithms default to 32. +PPO defaults to `batch_size: 64`; other batched algorithms default to 32. Early stopping, skipped updates or KL limits can consume less than the maximum. The authored schema is separate from resolved training phases. Checkpoints and @@ -614,7 +658,7 @@ contains execution/distribution, resume, data/window and model contracts only. A sequential phase exhausts each source in listed order. An interleaved phase uses `round_robin` or `weighted_random`. Each selection consumes at most `batches_per_selection`; every source is still exhausted once per phase epoch. -Weights affect order and burst frequency, not the amount of data consumed. +Source `selection_weight` values affect order and burst frequency, not the amount of data consumed. Gradient accumulation may cross part or source boundaries only while the dataset stays the same. A dataset transition flushes a partial window using its @@ -626,17 +670,21 @@ With several evaluation sources, configure an explicit monitor when validation-based saving or early stopping is enabled: ```yaml -# Inside a training_plan.phases entry: evaluation: - sources: [{source: events}, {source: telemetry.main}] - monitor: {source: events, metric: loss, mode: min} + sources: + - source: events + - source: telemetry.main + monitor: + source: events + metric: loss + direction: min ``` -Checkpoint interval fields retain distinct behavior. Phase `checkpoint_interval`, -global `save_interval_batches`, and global `save_interval_minutes` create persistent +Checkpoint interval fields retain distinct behavior. Phase epoch/iteration intervals, +global `snapshot_interval_batches`, and global `snapshot_interval_minutes` create persistent epoch/batch snapshots and refresh the rolling `latest` checkpoint; -`save_latest_interval_minutes` refreshes only `latest`. -`save_interval_val_loss` controls whether timed or batch snapshots also run +`latest_checkpoint_interval_minutes` refreshes only `latest`. +`evaluate_on_snapshot` controls whether timed or batch snapshots also run validation and record its monitored loss. Every checkpoint is written after flushing any partial gradient-accumulation window. @@ -649,10 +697,9 @@ such as `global_training`, `model.interfaces`, and `dataset_training`; duplicate fields are rejected. CLI overrides are applied after composition and before metadata resolution. -Singleton fragments may likewise contribute disjoint fields under -`model.interface` or `dataset`. Normalization occurs after fragment -composition, so all fragments in one training config must consistently use the -singular or named spelling at each level. +Fragments use the same named paths as the entry file. Components use only +`{name, arguments}`; constructor arguments must be nested under `arguments`. +Initialization methods use mappings such as `{method: zeros}`. The training command accepts `--model-name`, `--seed`, and `--skip-metadata` as configuration overrides. Dataset paths, columns, metadata paths, and device @@ -690,8 +737,10 @@ pool reduces the images assigned to one `itemPosition` to one embedding. model: interfaces: prediction: - input_columns: [temperature] - target_columns: [nextAction] + input_columns: + - temperature + target_columns: + - nextAction preconditioning_image_encoder: type: frozen model: resnet18 @@ -701,23 +750,31 @@ model: branches: measurements: type: embedding - columns: [temperature] + columns: + - temperature output_dim: 32 cameras: type: images - views: [front, side] + views: + - front + - side output_dim: 128 - encoder: {type: resnet, depth: 18} - frame_pool: attention + encoder: + type: resnet + depth: 18 + frame_pooling: attention view_merge: concat - merge: {type: concat} - decoder: {type: linear, prediction_length: 1} + merge: + type: concat + decoder: + type: linear + prediction_length: 1 ``` Bind the selected views to their original folders in each dataset part. The -preprocessing metadata deliberately omits folder paths. A single selected view -may use a string `image_path`; multiple views require a name-to-path map. -Preconditioning images use `preconditioning_image_path` in the same way: +preprocessing metadata deliberately omits folder paths. `image_paths` is always +a view-to-folder mapping, including runs with a single selected view. +Preconditioning images use `preconditioning_image_paths` in the same way: ```yaml dataset_training: @@ -726,11 +783,14 @@ dataset_training: parts: main: metadata_config_path: configs/metadata_configs/camera_data.json - image_path: {front: data/front_images, side: data/side_videos} - preconditioning_image_path: + image_paths: + front: data/front_images + side: data/side_videos + preconditioning_image_paths: front: data/sequence_front_images side: data/sequence_side_images - criterion: {nextAction: CrossEntropyLoss} + criterion: + nextAction: CrossEntropyLoss ``` Image view names and structured image cell names cannot be target columns; @@ -749,7 +809,7 @@ The `frozen` encoder supports TorchVision `resnet18` and `resnet50` with an explicit weight version. Its backbone stays frozen and in evaluation mode; its output projection trains. The default image size and normalization match the V1 ImageNet weight transforms. ResNet-50 V2 requires `resize_size: 232` -on its source; model resolution checks this. `frame_pool` is `mean` (default) or +on its source; model resolution checks this. `frame_pooling` is `mean` (default) or `attention`; `view_merge` is `concat` (default) or `mean`. Image embeddings may also occupy cells in an existing structured layout. The @@ -760,18 +820,30 @@ tabular cells: ```yaml feature_layout: sensors: - axes: {source: [camera, measurement]} + axes: + source: + - camera + - measurement columns: - camera_cell: {source: camera} - temperature: {source: measurement} + camera_cell: + source: camera + temperature: + source: measurement ingestion: type: structured layout: sensors output_dim: 128 - image_cells: {camera_cell: front} - image_encoder: {type: frozen, model: resnet50, weights: IMAGENET1K_V1} + image_cells: + camera_cell: front + image_encoder: + type: frozen + model: resnet50 + weights: IMAGENET1K_V1 processing_blocks: - - {type: axis_pool, axes: [source], mode: mean} + - type: axis_pool + axes: + - source + pooling: mean ``` The preconditioning source is independent of temporal views. One or more @@ -782,7 +854,7 @@ that one prefix position. ## Depth encoders and nested composites Use `global_training.read_format: pt` for preprocessed depth inputs. Layout -capacities come from dataset metadata. The following singleton model fragment +capacities come from dataset metadata. The following named model fragment combines one shallow branch with a depth encoder; its metadata comes from the preprocessing example in [preprocess.md](preprocess.md#named-depth-layouts). @@ -796,45 +868,76 @@ model: backbone: initialization_seed: 20260910 architecture: - dim_model: 128 + model_dim: 128 max_context_length: 128 num_layers: 4 - attention: {type: mha, n_heads: 8} - feed_forward: {dim: 512, activation: swiglu} - position_encoding: {type: rope} - interface: - input_columns: [accountType, subitemType, subitemAmount] - target_columns: [nextAction] - ingestion: - type: composite - branches: - item: - type: embedding - columns: [accountType] - output_dim: 32 - subitems: - type: depth_transformer - layout: subitems - columns: [subitemType, subitemAmount] - feature_embedding_dims: {subitemType: 48, subitemAmount: 16} - output_dim: 96 - dropout: 0.1 - initialization_seed: 12345 - initialization: - attention.qkv: - weight: {method: xavier_uniform, gain: 1.0} - free_parameter: - weight: {method: normal, mean: 0.0, std: 0.02} - architecture: - dim_model: 64 - num_layers: 2 - attention: {type: gqa, n_heads: 4, n_kv_heads: 2} - feed_forward: {dim: 256, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: learned} + attention: + type: mha + num_heads: 8 + feed_forward: + activation: swiglu + hidden_dim: 512 + position_encoding: + type: rope + interfaces: + default: + input_columns: + - accountType + - subitemType + - subitemAmount + target_columns: + - nextAction + ingestion: + type: composite + branches: + item: + type: embedding + columns: + - accountType + output_dim: 32 + subitems: + type: depth_transformer + layout: subitems + columns: + - subitemType + - subitemAmount + feature_embedding_dims: + subitemType: 48 + subitemAmount: 16 + output_dim: 96 dropout: 0.1 - merge: {type: concat} - decoder: {type: linear, prediction_length: 1} + initialization_seed: 12345 + initialization: + attention.qkv: + weight: + method: xavier_uniform + gain: 1.0 + free_parameter: + weight: + method: normal + mean: 0.0 + std: 0.02 + architecture: + model_dim: 64 + num_layers: 2 + attention: + type: gqa + num_heads: 4 + num_kv_heads: 2 + feed_forward: + activation: swiglu + hidden_dim: 256 + normalization: + type: rmsnorm + norm_first: true + position_encoding: + type: learned + dropout: 0.1 + merge: + type: concat + decoder: + type: linear + prediction_length: 1 ``` For a `gated` or `attention` merge, `merge.softmax` accepts `standard` @@ -848,16 +951,16 @@ GQA; LayerNorm or RMSNorm; the shared feed-forward activations and layer-sharing configuration. `architecture.dropout` controls depth position and transformer sites. The ingestion-level `dropout` controls the pooled output. Mixed categorical/real features require explicit feature widths; homogeneous features -can divide `architecture.dim_model` using the ordinary ingestion width rules. +can divide `architecture.model_dim` using the ordinary ingestion width rules. For categorical-only depth features, automatic allocation gives each column at -least two dimensions, so `architecture.dim_model` must be at least twice the +least two dimensions, so `architecture.model_dim` must be at least twice the number of columns. To use smaller shares, set positive widths for every column in `feature_embedding_dims`. For example, with two categorical columns, `feature_embedding_dims: {first: 1, second: 1}` permits one dimension each. -Explicit widths may sum to a value different from `architecture.dim_model`; +Explicit widths may sum to a value different from `architecture.model_dim`; the depth input projection handles the difference. The pooled projection handles -differences between `architecture.dim_model` and `output_dim`. +differences between `architecture.model_dim` and `output_dim`. CLS occupies position zero, and physical slot `s` occupies position `s+1`. Empty collections have a learnable CLS-only representation. Depth columns @@ -888,18 +991,32 @@ greedily inside the decoder. Categorical and real targets can be mixed. decoder: type: autoregressive_transformer prediction_length: 1 - support: 1 - target_columns: [patch_length, byte_0, byte_1, magnitude] + target_columns: + - patch_length + - byte_0 + - byte_1 + - magnitude architecture: - dim_model: 256 + model_dim: 256 num_layers: 2 - attention: {type: mha, n_heads: 4} - feed_forward: {dim: 768, activation: swiglu} - normalization: {type: rmsnorm, norm_first: true} - position_encoding: {type: rope, theta: 10000} + attention: + type: mha + num_heads: 4 + feed_forward: + activation: swiglu + hidden_dim: 768 + normalization: + type: rmsnorm + norm_first: true + position_encoding: + type: rope + theta: 10000 dropout: 0.0 - shared_categorical_target_groups: [[byte_0, byte_1]] + shared_categorical_target_groups: + - - byte_0 + - byte_1 tie_input_output_embeddings: true + support_length: 1 ``` Shared groups must contain categorical targets with identical decoder-ID @@ -931,22 +1048,29 @@ using the run RNG and checkpointed per-rank state. Dataset freezing can follow the same tree: ```yaml -dataset: - part: {metadata_config_path: configs/metadata_configs/raw-items.json} - criterion: {nextAction: CrossEntropyLoss} - freeze: - backbone: - freeze: [attention.qkv] - ingestion: - branches: - subitems: - freezing_except: [free_parameter, ingestion.output_projection] - item: - freeze: [] +dataset_training: + default: + criterion: + nextAction: CrossEntropyLoss + freeze: + backbone: + freeze_groups: + - attention.qkv + ingestion: + branches: + subitems: + freeze_except_groups: + - free_parameter + - ingestion.output_projection + item: + freeze_groups: [] + parts: + default: + metadata_config_path: configs/metadata_configs/raw-items.json ``` A local child selector replaces the inherited decision, including selective -unfreezing. `freeze: []` unfreezes the scope; `freezing_except: []` freezes it. +unfreezing. `freeze_groups: []` unfreezes the scope; `freeze_except_groups: []` freezes it. A node containing only `branches` delegates without changing inherited decisions. Merge parameters retain their parent's policy; the adapter has its own dataset selector. Unknown branches, policies below leaves, and contradictory alias @@ -960,8 +1084,7 @@ gradients are absent. ## Policy and mixed plans All algorithms use `training_plan.phases`. Declare policy sequence/preference -parts in `dataset_training`, and attach actions with `policy_binding` (one -interface) or `policy_bindings` (named interfaces). Run-wide settings remain in +parts in `dataset_training`, and attach actions with `policy_bindings` keyed by interface. Run-wide settings remain in `global_training`; no alternate run discriminator is needed. See the complete [supervised template](../../src/sequifier/templates/train-supervised.yaml), diff --git a/documentation/demos/self-contained-example.ipynb b/documentation/demos/self-contained-example.ipynb index 6d4f7949..13a32a60 100644 --- a/documentation/demos/self-contained-example.ipynb +++ b/documentation/demos/self-contained-example.ipynb @@ -100,7 +100,7 @@ " \"write_format\": \"parquet\",\n", " \"selected_columns\": [\"targetToken\"],\n", " \"split_ratios\": [0.8, 0.1, 0.1], # Train, Val, Test\n", - " \"window_length\": 11, # 10-token model context plus one future target\n", + " \"window_lengths\": [11, 11, 11], # 10-token model context plus one future target\n", " \"max_target_offset\": 1,\n", " \"window_stride\": 1, # Dense sampling\n", " \"max_rows\": None,\n", @@ -127,45 +127,67 @@ " \"training_objective\": \"causal\",\n", " \"context_length\": 10,\n", " \"inference_batch_size\": 10,\n", - " \"batch_size\": 32,\n", - " \"learning_rate\": 0.001,\n", - " \"optimizer\": {\"name\": \"Adam\"},\n", - " \"scheduler\": {\"name\": \"StepLR\", \"step_size\": 1, \"gamma\": 0.99},\n", - " \"log_interval\": 10000,\n", - " \"save_interval_epochs\": 5,\n", + " \"log_interval_batches\": 10000,\n", " \"torch_compile\": \"none\", # Keep the demo independent of a host C++ toolchain.\n", " },\n", " \"model\": {\n", " \"backbone\": {\n", " \"architecture\": {\n", - " \"dim_model\": 16,\n", + " \"model_dim\": 16,\n", " \"max_context_length\": 10,\n", - " \"num_layers\": 1, # Tiny model for speed\n", - " \"attention\": {\"n_heads\": 2},\n", - " \"feed_forward\": {\"dim\": 32},\n", + " \"num_layers\": 1,\n", + " \"attention\": {\"num_heads\": 2},\n", + " \"feed_forward\": {\"hidden_dim\": 32},\n", " \"dropout\": 0.0,\n", " },\n", " },\n", - " \"interface\": {\n", - " \"input_columns\": [\"targetToken\"],\n", - " \"target_columns\": [\"targetToken\"],\n", - " \"ingestion\": {\n", - " \"type\": \"embedding\",\n", - " \"output_dim\": 16,\n", - " \"feature_embedding_dims\": {\"targetToken\": 16},\n", + " \"interfaces\": {\n", + " \"default\": {\n", + " \"input_columns\": [\"targetToken\"],\n", + " \"target_columns\": [\"targetToken\"],\n", + " \"ingestion\": {\n", + " \"type\": \"embedding\",\n", + " \"output_dim\": 16,\n", + " \"feature_embedding_dims\": {\"targetToken\": 16},\n", + " },\n", + " \"decoder\": {\"type\": \"linear\", \"prediction_length\": 1},\n", " },\n", - " \"decoder\": {\"type\": \"linear\", \"prediction_length\": 1},\n", " },\n", " },\n", - " \"dataset\": {\n", - " # Metadata is generated by preprocessing from the input basename.\n", - " \"part\": {\n", - " \"metadata_config_path\": \"configs/metadata_configs/synthetic-sequences.json\"\n", + " \"dataset_training\": {\n", + " \"default\": {\n", + " \"parts\": {\n", + " \"default\": {\n", + " \"metadata_config_path\": \"configs/metadata_configs/synthetic-sequences.json\"\n", + " },\n", + " },\n", + " \"criterion\": {\"targetToken\": \"CrossEntropyLoss\"},\n", " },\n", - " \"criterion\": {\"targetToken\": \"CrossEntropyLoss\"},\n", " },\n", - " \"training_plan\": {\"epochs\": 5},\n", - " \"evaluation\": True,\n", + " \"training_plan\": {\n", + " \"phases\": [\n", + " {\n", + " \"name\": \"supervised\",\n", + " \"algorithm\": \"supervised\",\n", + " \"epochs\": 5,\n", + " \"mode\": \"sequential\",\n", + " \"sources\": [{\"source\": \"default.default\"}],\n", + " \"batch_size\": 32,\n", + " \"checkpoint_interval_epochs\": 5,\n", + " \"optimization\": {\n", + " \"model\": {\n", + " \"learning_rate\": 0.001,\n", + " \"optimizer\": {\"name\": \"Adam\", \"arguments\": {}},\n", + " \"scheduler\": {\n", + " \"name\": \"StepLR\",\n", + " \"arguments\": {\"step_size\": 1, \"gamma\": 0.99},\n", + " },\n", + " },\n", + " },\n", + " \"evaluation\": {\"sources\": [{\"source\": \"default.default\"}]},\n", + " }\n", + " ],\n", + " },\n", " \"export_generative_model\": True,\n", " \"export_embedding_model\": False,\n", " \"export_onnx\": True,\n", @@ -185,9 +207,9 @@ "source": [ "infer_config = {\n", " \"project_root\": PROJECT_ROOT,\n", - " \"model_type\": \"generative\",\n", + " \"task\": \"forecast\",\n", " # Training exports best and last artifacts with the final epoch in the filename.\n", - " \"model_path\": \"models/demo-cycle-model-best-5.onnx\",\n", + " \"model_paths\": [\"models/demo-cycle-model-best-5.onnx\"],\n", " \"training_config_path\": tr_conf_path,\n", " \"dataset\": \"default\",\n", " # Use the 'test' split created during preprocessing\n", @@ -197,7 +219,7 @@ " \"decode_categories\": True, # Map integer classes back to original values (e.g., \"1\", \"2\")\n", " \"device\": \"cpu\",\n", " \"inference_batch_size\": 10,\n", - " \"autoregressive\": True,\n", + " \"autoregressive_generation\": True,\n", " \"generation_steps\": 5, # Predict 5 steps into the future\n", "}\n", "\n", diff --git a/documentation/example_inputs/configs/import-policy-sequences.yaml b/documentation/example_inputs/configs/import-policy-sequences.yaml index 3b99ed68..15b0babf 100644 --- a/documentation/example_inputs/configs/import-policy-sequences.yaml +++ b/documentation/example_inputs/configs/import-policy-sequences.yaml @@ -1,12 +1,19 @@ operation: import_sequences -project_root: . +project_root: "." source: - paths: [data/demonstrations.csv] + paths: + - data/demonstrations.csv columns: - observations: {position: position, velocity: velocity} + observations: + position: position + velocity: velocity static_inputs: {} output_path: data/imported-demonstrations episodes_per_shard: 64 policy_binding: actions: - action: {type: categorical, values: [left, right]} + action: + type: categorical + values: + - left + - right diff --git a/documentation/example_inputs/configs/infer-policy.yaml b/documentation/example_inputs/configs/infer-policy.yaml index 2be2220f..92fcd94e 100644 --- a/documentation/example_inputs/configs/infer-policy.yaml +++ b/documentation/example_inputs/configs/infer-policy.yaml @@ -1,10 +1,11 @@ -project_root: . -model_path: models/policy/model.pt +project_root: "." data_path: data/policy-observations.csv -model_type: generative read_format: csv write_format: csv device: cpu desired_return: 10.0 enforce_determinism: true action_selection: deterministic +task: policy +model_paths: +- models/policy/model.pt diff --git a/documentation/example_inputs/configs/infer-ppo-toy.yaml b/documentation/example_inputs/configs/infer-ppo-toy.yaml index 0f78ff9d..6d331b65 100644 --- a/documentation/example_inputs/configs/infer-ppo-toy.yaml +++ b/documentation/example_inputs/configs/infer-ppo-toy.yaml @@ -1,10 +1,11 @@ -project_root: . -model_path: models/online-ppo/model.pt +project_root: "." data_path: documentation/example_inputs/ppo-observations.csv -model_type: generative read_format: csv write_format: csv device: cpu output_path: outputs/ppo-actions.csv enforce_determinism: true action_selection: deterministic +task: policy +model_paths: +- models/online-ppo/model.pt diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index 2f66632d..8959d470 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -1,7 +1,5 @@ -project_root: . -model_path: models/online-ppo/model.pt +project_root: "." data_path: documentation/example_inputs/ppo-observations.csv -model_type: generative read_format: csv write_format: csv device: cpu @@ -12,7 +10,11 @@ rollout: max_steps: 8 environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 enforce_determinism: true action_selection: deterministic +task: policy +model_paths: +- models/online-ppo/model.pt diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml index 4ef969d5..5385b85f 100644 --- a/documentation/example_inputs/configs/train-bc-toy.yaml +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -1,43 +1,66 @@ -project_root: . +project_root: "." model_name: toy-bc device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 128 + position_encoding_scope: global interfaces: actor: - input_columns: [position, __policy_previous_action] - target_columns: [action] + input_columns: + - position + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} -global_training: {context_length: 16, validation_fraction: 0.2} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [left, right]} + feature_embedding_dims: + position: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 16 + validation_fraction: 0.2 dataset_training: demonstrations: parts: - recorded: {type: sequence, paths: [data/offline-transitions.csv]} + recorded: + type: sequence + paths: + - data/offline-transitions.csv model_interface: actor training_plan: phases: - - name: behavior_cloning - algorithm: policy_imitation - sources: - - {source: demonstrations.recorded} - epochs: 30 - batch_size: 64 + - name: behavior_cloning + algorithm: policy_imitation + sources: + - source: demonstrations.recorded + epochs: 30 + batch_size: 64 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index 74abfb5b..fa19f4c4 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -1,42 +1,51 @@ -project_root: . +project_root: "." model_name: offline-iql device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 128 + position_encoding_scope: global interfaces: actor: - input_columns: [position, velocity, __policy_previous_action] - target_columns: [action] + input_columns: + - position + - velocity + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 1, velocity: 1, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + position: 1 + velocity: 1 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 16 validation_fraction: 0.2 - resume: {policy: if_exists} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [left, right]} + resume: + policy: if_exists dataset_training: demonstrations: parts: recorded: type: sequence - paths: [data/offline-transitions.parquet] + paths: + - data/offline-transitions.parquet columns: sequence_id: sequenceId item_position: itemPosition @@ -45,20 +54,35 @@ dataset_training: terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: {position: next_position, velocity: next_velocity} + successors: + position: next_position + velocity: next_velocity model_interface: actor training_plan: phases: - - name: offline_improvement - algorithm: iql - sources: - - {source: demonstrations.recorded} - updates: 10000 - batch_size: 64 - gamma: 0.99 - expectile: 0.7 - inverse_temperature: 3.0 - max_weight: 100.0 - target_update_rate: 0.005 - evaluation_interval: 100 - checkpoint_interval: 100 + - name: offline_improvement + algorithm: iql + sources: + - source: demonstrations.recorded + batch_size: 64 + gamma: 0.99 + expectile: 0.7 + inverse_temperature: 3.0 + max_advantage_weight: 100.0 + target_update_rate: 0.005 + iterations: 10000 + evaluation_interval_iterations: 100 + checkpoint_interval_iterations: 100 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml index 35bcb94c..e2cdd27f 100644 --- a/documentation/example_inputs/configs/train-iql-toy.yaml +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -1,4 +1,4 @@ -project_root: . +project_root: "." model_name: toy-iql device: cpu seed: 1010 @@ -6,52 +6,73 @@ initialize_from: models/toy-bc/actor.pt model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 128 + position_encoding_scope: global interfaces: actor: - input_columns: [position, __policy_previous_action] - target_columns: [action] + input_columns: + - position + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + position: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 16 validation_fraction: 0.2 - resume: {policy: if_exists} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [left, right]} + resume: + policy: if_exists dataset_training: demonstrations: parts: recorded: type: sequence - paths: [data/offline-transitions.csv] + paths: + - data/offline-transitions.csv columns: reward: reward terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: {position: next_position} + successors: + position: next_position model_interface: actor training_plan: phases: - - name: offline_improvement - algorithm: iql - sources: - - {source: demonstrations.recorded} - updates: 10000 - batch_size: 64 - evaluation_interval: 100 - checkpoint_interval: 100 + - name: offline_improvement + algorithm: iql + sources: + - source: demonstrations.recorded + batch_size: 64 + iterations: 10000 + evaluation_interval_iterations: 100 + checkpoint_interval_iterations: 100 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index 546a5fa8..e675ecec 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -1,111 +1,149 @@ -project_root: . +project_root: "." model_name: multiphase-control device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 1 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 64 + position_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [cue, __policy_previous_action] - target_columns: [action] + input_columns: + - cue + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + cue: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 4 - normalization: {source: identity} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [left, right]} + normalization: + source: identity dataset_training: decisions: parts: default: type: sequence - paths: [data/control-decisions.csv] + paths: + - data/control-decisions.csv columns: episode_id: episodeId reward: reward terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: {cue: next_cue} + successors: + cue: next_cue group: contextId model_interface: actor rankings: parts: - default: {type: preferences, manifest: data/control-preferences.json} + default: + type: preferences + manifest_path: data/control-preferences.json training_plan: phases: - - name: imitation - algorithm: policy_imitation - sources: - - {source: decisions.default} - epochs: 4 - - name: offline_improvement - algorithm: iql - sources: - - {source: decisions.default} - updates: 100 - - name: preferences - algorithm: dpo - preferences: {source: rankings.default} - updates: 100 - beta: 0.1 - - name: group_improvement - algorithm: grpo - total_groups: 32 - groups_per_update: 2 - group_size: 4 - max_steps: 8 + - name: imitation + algorithm: policy_imitation + sources: + - source: decisions.default + epochs: 4 + - name: offline_improvement + algorithm: iql + sources: + - source: decisions.default + iterations: 100 + - name: preferences + algorithm: dpo + preferences: + source: rankings.default + beta: 0.1 + iterations: 100 + - name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_iteration: 2 + group_size: 4 + max_steps: 8 + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 + evaluation: environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 - evaluation: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 - monitor: {metric: score_mean, mode: max} - - name: ppo_improvement - algorithm: ppo + seeds: + - 17 + - 29 + - 43 + - 71 + max_episode_steps: 8 + timeout_seconds: 30 + monitor: + metric: score_mean + direction: max + - name: ppo_improvement + algorithm: ppo + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 + total_env_steps: 8192 + rollout_env_steps: 256 + optimization_epochs: 4 + accumulation_batches: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + evaluation: environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 + seeds: + - 17 + - 29 + - 43 + - 71 + max_episode_steps: 8 + timeout_seconds: 30 + batch_size: 64 + evaluation_interval_iterations: 2 + checkpoint_interval_iterations: 2 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml index 34d85593..6d76be98 100644 --- a/documentation/example_inputs/configs/train-policy-from-table.yaml +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -1,45 +1,71 @@ -project_root: . +project_root: "." model_name: table-policy device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 128 + position_encoding_scope: global interfaces: actor: - input_columns: [position, velocity, __policy_previous_action] - target_columns: [action] + input_columns: + - position + - velocity + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {position: 1, velocity: 1, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} -global_training: {context_length: 16, validation_fraction: 0.2} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [left, right]} + feature_embedding_dims: + position: 1 + velocity: 1 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] +global_training: + context_length: 16 + validation_fraction: 0.2 dataset_training: demonstrations: parts: - recorded: {type: sequence, paths: [data/demonstrations.parquet]} + recorded: + type: sequence + paths: + - data/demonstrations.parquet model_interface: actor training_plan: phases: - - name: behavior_cloning - algorithm: policy_imitation - epochs: 80 - sources: - - {source: demonstrations.recorded} - batch_size: 64 - optimization: - actor: {learning_rate: 0.005} + - name: behavior_cloning + algorithm: policy_imitation + epochs: 80 + sources: + - source: demonstrations.recorded + batch_size: 64 + optimization: + actor: + learning_rate: 0.005 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index d259e484..c0a350b6 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -1,53 +1,81 @@ -project_root: . +project_root: "." model_name: policy device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 2 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + activation: swiglu + hidden_dim: 128 + position_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [OBSERVATION_COLUMN, __policy_previous_action, __policy_return] - target_columns: [action] + input_columns: + - OBSERVATION_COLUMN + - __policy_previous_action + - __policy_return + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} -model_path: models/policy/actor.pt + feature_embedding_dims: + OBSERVATION_COLUMN: 1 + __policy_previous_action: 6 + __policy_return: 1 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] report_path: outputs/policy/training-report.json global_training: context_length: 4 validation_fraction: 0.2 episode_cache_size: 8 - resume: {policy: never} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: true - returns: {gamma: 1.0, scale: 1.0} - actions: - action: {type: categorical, values: [0.0, 1.0]} + resume: + policy: never dataset_training: demonstrations: parts: - recorded: {type: sequence, paths: [data/demonstrations.parquet]} + recorded: + type: sequence + paths: + - data/demonstrations.parquet model_interface: actor training_plan: phases: - - name: offline_policy - algorithm: policy_imitation - epochs: 80 - sources: - - {source: demonstrations.recorded} - batch_size: 64 - optimization: - actor: {learning_rate: 0.005} + - name: offline_policy + algorithm: policy_imitation + epochs: 80 + sources: + - source: demonstrations.recorded + batch_size: 64 + optimization: + actor: + learning_rate: 0.005 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: true + returns: + gamma: 1.0 + scale: 1.0 + actions: + action: + type: categorical + values: + - 0.0 + - 1.0 +model_path: models/policy/actor.pt diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 5a4567aa..90346719 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -11,17 +11,17 @@ actions and mixed supervised/policy plans. ## Authoring and optimization -`policy_binding` selects actions for one interface; `policy_bindings` maps named -interfaces. `global_training` owns context length, frozen normalization, validation +`policy_bindings` maps interface names to action declarations, including runs +with one interface. `global_training` owns context length, frozen normalization, validation fraction, episode cache size, determinism and resume. Phases own batching, `optimization.`, evaluation/checkpoint intervals and `handoff: last | best`. | Algorithm | Required budget/data | Trainable roles | | --- | --- | --- | | `policy_imitation` | `epochs`, `sources` | `actor` | -| `iql` | `updates`, transition `sources` | `actor`, `value`, `critic` | -| `dpo` | `updates`, `preferences` | `actor` | -| `ppo` | `total_env_steps`, `rollout_length`, `environment`, `evaluation` | `actor`, `value` | +| `iql` | `iterations`, transition `sources` | `actor`, `value`, `critic` | +| `dpo` | `iterations`, `preferences` | `actor` | +| `ppo` | `total_env_steps`, `rollout_env_steps`, `environment`, `evaluation` | `actor`, `value` | | `grpo` | `total_groups`, `max_steps`, `environment`, `evaluation` | `actor` | Each role defaults to Adam, learning rate `0.0003`, gradient clip `1.0`, and @@ -30,8 +30,8 @@ arguments; grouped optimizer plans set their own rates. Policy schedulers advanc on optimizer steps; plateau/compound schedulers are unsupported. `OneCycleLR.total_steps` is derived from the phase budget; an explicitly supplied value must match that budget. Unknown roles fail validation. -Imitation, DPO and GRPO default to `batch_size: 32`, `accumulation_steps: 1`, -`evaluation_interval: 1` and `checkpoint_interval: 1`. Accumulation uses actual +Imitation, DPO and GRPO default to `batch_size: 32`, `accumulation_batches: 1`, +`evaluation_interval_iterations: 1` and `checkpoint_interval_iterations: 1`. Accumulation uses actual example counts, including partial batches. Network stochastic layers are disabled while gradients remain enabled. Best handoff keeps the earliest finite best on ties; last handoff keeps the latest transaction. Final updates always evaluate. @@ -62,8 +62,8 @@ Parquet shards and a checksummed manifest; see the Imitation minimizes joint action NLL. Return conditioning requires rewards and a known tail: true termination or an explicit finite return horizon. The label at `t` includes `r_t`; unknown tails are excluded. Offline monitors are fixed to -`{metric: validation_action_nll, mode: min}` for imitation/IQL and -`{metric: validation_dpo_loss, mode: min}` for DPO. Only +`{metric: validation_action_nll, direction: min}` for imitation/IQL and +`{metric: validation_dpo_loss, direction: min}` for DPO. Only `evaluation.sources` and `evaluation.monitor` are accepted; alternate spellings are rejected. @@ -71,9 +71,9 @@ IQL requires complete reward, terminated, truncated and bootstrap columns, plus true successors for bootstrapped transitions. It samples uniformly with replacement, fits expectile V and twin Q-functions, updates target Q-functions with Polyak averaging, and weights actor NLL by -`min(exp(inverse_temperature * (min(target_Q1,target_Q2)-V)), max_weight)`. +`min(exp(inverse_temperature * (min(target_Q1,target_Q2)-V)), max_advantage_weight)`. Defaults: `batch_size: 64`, `gamma: 0.99`, `expectile: 0.7`, -`inverse_temperature: 3`, `max_weight: 100`, `target_update_rate: 0.005`, +`inverse_temperature: 3`, `max_advantage_weight: 100`, `target_update_rate: 0.005`, evaluation/checkpoint intervals `100`. Held-out action NLL selects the actor; critic diagnostics are separate. Critics are fresh per phase and are not exported. @@ -126,10 +126,10 @@ is a complete local example. PPO seals one rollout generation, computes boundary-aware FP32 GAE, then optimizes clipped actor and independent V objectives. Truncation bootstraps from the final successor without carrying GAE across reset. Collection horizons retain live history. -Defaults: `optimization_epochs: 4`, `minibatch_size: 64`, `accumulation_steps: 1`, +Defaults: `optimization_epochs: 4`, `batch_size: 64`, `accumulation_batches: 1`, `gamma: 0.99`, `gae_lambda: 0.95`, `clip_epsilon: 0.2`, `value_coefficient: 0.5`, `entropy_coefficient: 0.01`, `normalize_advantages: true`, `reward_scale: 1`. -Optional `value_clip` clips V changes; `target_kl` stops old-policy drift. +Optional `value_clip_delta` clips V changes; `old_policy_target_kl` stops old-policy drift. `allow_partial_rollout: true` executes the remaining budget exactly; false requires divisibility. Optional `archive_directory` writes independent collection records; `archive_scores` defaults to false. Archives are not update sources. @@ -138,10 +138,10 @@ GRPO seals complete same-start groups before updating an actor-only model. Scores sum scaled rewards through termination, truncation or `max_steps`. `A_i = (R_i-mean(R))/(population_std(R)+advantage_epsilon)`. Its loss averages equally over trajectories, then over their decisions: -`-min(r*A, clip(r)*A) + kl_coefficient*(exp(d)-d-1)`, where +`-min(r*A, clip(r)*A) + reference_kl_coefficient*(exp(d)-d-1)`, where `r=exp(log_pi-log_pi_old)` and `d=log_pi_ref-log_pi`. -Defaults: `group_size: 4`, `groups_per_update: 1`, `optimization_epochs: 4`, -`advantage_epsilon: 1e-6`, `clip_epsilon: 0.2`, `kl_coefficient: 0.04`, +Defaults: `group_size: 4`, `groups_per_iteration: 1`, `optimization_epochs: 4`, +`advantage_epsilon: 1e-6`, `clip_epsilon: 0.2`, `reference_kl_coefficient: 0.04`, `reward_scale: 1`, `scoring_chunk_size: 64`. Optional `old_policy_target_kl` limits old-policy drift separately. The sampled reference penalty is not exact current-policy KL. `max_env_steps` uses a conservative complete-group bound. Failed/incomplete groups diff --git a/documentation/rollout.md b/documentation/rollout.md index 3cf2a1e8..3bf60ff5 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -10,12 +10,12 @@ Forecast rollout supports CPU, real scalar temporal/static inputs, and one-step `causal`/`final_value` models with `target_offset: 1`, using PT or schema-bearing ONNX: ```yaml -model_type: generative +task: forecast rollout: max_steps: 100 seed_window_index: 0 trajectory_path: outputs/trajectory.json - environment: {factory: application.adapters:make_environment, config: {}} + environment: {factory: "application.adapters:make_environment", config: {}} bindings: - {input: x, source: prediction, key: x} - {input: disturbance, source: environment, key: disturbance} diff --git a/documentation/supervised-policy-learning.md b/documentation/supervised-policy-learning.md index 1b4b5857..b01a84e3 100644 --- a/documentation/supervised-policy-learning.md +++ b/documentation/supervised-policy-learning.md @@ -7,17 +7,26 @@ ordinary supervised PT artifact or supported run checkpoint: ```yaml initialize_from: models/supervised/model.pt -policy_binding: - actions: - mode: {type: categorical, values: ["off", "on"]} - price: {type: real, scale: 0.5} +policy_bindings: + default: + actions: + mode: + type: categorical + values: + - 'off' + - 'on' + price: + type: real + scale: + fixed: 0.5 ``` Add datasets/phases as in the [policy guide](policy-learning.md). A separately declared model must agree with the artifact. Omitted categorical `values` select the whole decoder vocabulary; an explicit subset is reported as an adaptation. -Real `scale` is required and measured in physical units. `learn_scale` enables a -global log scale bounded by `min_scale`/`max_scale`. `lower`/`upper` use bijections; +Real `scale` explicitly selects `model` or `{fixed: ...}` in physical units. +Learned scales belong to the interface's `prediction_distributions`, where +`scale` declares `initial`, `trainable`, `min`, and `max`. `lower`/`upper` use bijections; `atoms` assigns exact point masses (positive probabilities summing to less than one). Bounds change the deterministic prediction and count as adaptation. Samples retain transform latents; external endpoints need declared atoms. `active_when` references diff --git a/documentation/training/multi-gpu-training.md b/documentation/training/multi-gpu-training.md index c6bc0b6c..c2fb0ae4 100644 --- a/documentation/training/multi-gpu-training.md +++ b/documentation/training/multi-gpu-training.md @@ -33,15 +33,14 @@ In your `train.yaml`, configure the canonical `global_training` block: ```yaml global_training: - read_format: pt # or parquet for beta sharded Parquet loading + read_format: pt distributed: true - data_parallelism: fsdp # or ddp - fsdp_cpu_offload: false # omit if using ddp; set true to offload FSDP parameters to CPU RAM - layer_type_dtypes: null # required for FSDP; use layer_autocast for mixed precision - torch_compile: inner # use inner or none for FSDP; use outer or none for DDP - world_size: 32 # The TOTAL number of GPUs across all nodes (e.g., 8 nodes * 4 GPUs = 32) - backend: nccl # 'nccl' is the standard and most efficient backend for NVIDIA GPUs - + data_parallelism: fsdp + fsdp_cpu_offload: false + layer_type_dtypes: null + torch_compile: inner + world_size: 32 + backend: nccl ``` When shards do not divide evenly across ranks, Sequifier automatically pads shorter ranks with repeated samples for step alignment. Those repeats are masked out of loss calculation, so each real sample contributes once. @@ -84,6 +83,6 @@ srun torchrun \ ### Important Considerations for Multi-Node -* **Batch Size:** The `batch_size` in your `train.yaml` is the **per-process** batch size. If `batch_size` is 100 and `world_size` is 32, each synchronized backward pass covers 3,200 samples. With gradient accumulation, the samples per optimizer update are `batch_size * world_size * accumulation_steps` (apart from a final partial accumulation window). +* **Batch Size:** The `batch_size` in your `train.yaml` is the **per-process** batch size. If `batch_size` is 100 and `world_size` is 32, each synchronized backward pass covers 3,200 samples. With gradient accumulation, the samples per optimizer update are `batch_size * world_size * accumulation_batches` (apart from a final partial accumulation window). * **Learning Rate:** You may need to scale your `learning_rate` up if you drastically increase your global batch size via distributed training. * **Data Access:** All nodes must have access to the same shared filesystem (e.g., NFS, GPFS) where the `project_root` and the sharded preprocessing output are stored. diff --git a/release-notes/unreleased-canonical-config-names.md b/release-notes/unreleased-canonical-config-names.md new file mode 100644 index 00000000..c64aa562 --- /dev/null +++ b/release-notes/unreleased-canonical-config-names.md @@ -0,0 +1,23 @@ +# Canonical public configuration names + +- Use named training interfaces, datasets, parts, bindings, and phases for every + cardinality. Remove singleton aliases and boolean evaluation shorthand. +- Standardize dimensions, counts, batch/optimizer/iteration clocks, selectors, + checkpoint intervals, paths, monitoring direction, and probability names. +- Require explicit inference `task` and list-valued `model_paths`. Named media + paths are mappings; preprocessing widths are always `window_lengths`. +- Nest registered component constructor settings under `arguments` and declare + initialization methods as mappings. Remove scalar, method, and dtype aliases. +- Give hyperparameter search explicit `$choices`, `$fixed`, `$range`, and + `$variants` operators with canonical training paths and independent sampler seed. +- Resolve run `seed: random` once. CLI flags mirror the canonical keys, with no + short aliases or abbreviations and explicit boolean override values. +- Update runtime consumers, fingerprints, scaffolding, templates, and guides. + +This is a breaking authoring change without an alias compatibility layer. See +[the complete naming reference](../documentation/configs/naming.md). Configurations +embedded in artifacts/checkpoints must also use the canonical reconstruction +schema; old embedded declarations are not silently translated. + +No tests were written or run for this change, as requested. Verification is limited +to static source syntax, reference searches, and diff inspection. diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index a84ab155..fdfaf280 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -22,7 +22,7 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: "window_view": True, "policy": True, "image_sources": {"__all__": {"path"}}, - "preconditioning_image": {"path"}, + "preconditioning_image": {"paths"}, }, ) values["execution_schema"] = ExecutionSchema.from_interface( diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index 56ee833d..979f6c34 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -17,21 +17,11 @@ from decimal import Decimal from typing import Any, Literal -from pydantic import ( - BaseModel, - ConfigDict, - Field, - PrivateAttr, - ValidationError, - model_validator, -) +from pydantic import ConfigDict, Field, PrivateAttr, ValidationError, model_validator from sequifier.config.composition import load_composed_yaml_config -from sequifier.config.train_config import ( - SequifierConfig, - load_train_config_with_source, - normalize_train_config_parameter_surface, -) +from sequifier.config.naming import ConfigModel, RunSeed +from sequifier.config.train_config import SequifierConfig, load_train_config_with_source from sequifier.typechecking import beartype ConfigPath = tuple[str | int, ...] @@ -551,26 +541,39 @@ def _list_index_mapping(expression: dict[Any, Any]) -> dict[int, Any] | None: @beartype def _compile_value(expression: Any, base: Any, path: ConfigPath) -> _CompiledValue: - if ( - isinstance(expression, dict) - and set(expression) in ({"choices"}, {"$choices"}) - and not (isinstance(base, dict) and next(iter(expression)) in base) - ): - key = "choices" if "choices" in expression else "$choices" - return _categorical_space(path, expression[key]) - if ( - isinstance(expression, dict) - and set(expression) in ({"fixed"}, {"$fixed"}) - and not (isinstance(base, dict) and next(iter(expression)) in base) - ): - key = "fixed" if "fixed" in expression else "$fixed" - return _LiteralValue(expression[key]) - if ( - isinstance(expression, dict) - and not isinstance(base, dict) - and _is_distribution(expression) - ): - return _distribution_space(path, expression, base) + if isinstance(expression, dict): + if set(expression) == {"$choices"}: + return _categorical_space(path, expression["$choices"]) + if set(expression) == {"$fixed"}: + if not isinstance(expression["$fixed"], (dict, list)): + raise ValueError(f"{_path_name(path)}: use primitive scalars directly") + return _LiteralValue(expression["$fixed"]) + if set(expression) == {"$range"}: + spec = expression["$range"] + if not isinstance(spec, dict) or not _is_distribution(spec): + raise ValueError( + f"{_path_name(path)}.$range requires low, high and optional step/log/type" + ) + return _distribution_space(path, spec, base) + if not isinstance(base, dict): + for old, new in ( + ("choices", "$choices"), + ("fixed", "$fixed"), + ("variants", "$variants"), + ): + if old in expression: + raise ValueError(f"{_path_name(path)}.{old} was removed; use {new}") + if _is_distribution(expression) and not isinstance(base, dict): + raise ValueError( + f"{_path_name(path)}: wrap numeric distributions in $range" + ) + unsupported = { + key for key in expression if isinstance(key, str) and key.startswith("$") + } - {"$variants"} + if unsupported: + raise ValueError( + f"{_path_name(path)}: invalid search operator combination {sorted(unsupported)}" + ) if isinstance(base, list) and isinstance(expression, dict): indexed = _list_index_mapping(expression) @@ -592,12 +595,7 @@ def _compile_value(expression: Any, base: Any, path: ConfigPath) -> _CompiledVal base_mapping = copy.deepcopy(base) if isinstance(base, dict) else {} if isinstance(base, dict) and _changes_discriminator(base, expression, path): base_mapping = {} - variant_keys = {"variants", "$variants"} & set(expression) - if len(variant_keys) > 1: - raise ValueError( - f"{_path_name(path)} cannot configure both variants and $variants" - ) - variant_key = next(iter(variant_keys), None) + variant_key = "$variants" if "$variants" in expression else None child_expressions = { key: value for key, value in expression.items() if key != variant_key } @@ -631,22 +629,14 @@ def _compile_value(expression: Any, base: Any, path: ConfigPath) -> _CompiledVal return _MappingValue(path, base_mapping, mapping_children) if isinstance(expression, list): - if base is _MISSING: - return _LiteralValue(expression) - if isinstance(base, list): - # A list of lists is the established shorthand for selecting a - # complete list-valued field (for example input_columns). Other - # list-valued parameters are fixed replacements; ``choices`` is the - # unambiguous form for sampling arbitrary list values. - if expression and all(isinstance(value, list) for value in expression): - return _categorical_space(path, expression) - return _LiteralValue(expression) - return _categorical_space(path, expression) + raise ValueError( + f"{_path_name(path)}: use $choices to sample or $fixed to replace a list" + ) return _LiteralValue(expression) -class CanonicalHyperparameterSearchConfig(BaseModel): +class CanonicalHyperparameterSearchConfig(ConfigModel): """Search controls and a recursive sampler over one canonical train config.""" model_config = ConfigDict( @@ -660,17 +650,17 @@ class CanonicalHyperparameterSearchConfig(BaseModel): project_root: str = Field(min_length=1) name: str = Field(min_length=1) - method: Literal["bayesian", "sample", "grid"] = "bayesian" - global_seed: int | None = None + method: Literal["bayesian", "random", "grid"] = "bayesian" + sampler_seed: RunSeed | None = None trials: int | None = Field(default=None, gt=0) prune_trials: bool = True pruning_warmup_epochs: int | None = Field(default=None, ge=0) pruning_warmup_batches: int | None = Field(default=None, ge=0) - model_config_write_path: str = Field(min_length=1) + trial_config_directory: str = Field(min_length=1) - evaluation_inference_config: str | None = None - evaluation_script: str | None = None - evaluation_metric_directions: list[Literal["minimize", "maximize"]] | None = None + evaluation_inference_config_path: str | None = None + evaluation_script_path: str | None = None + evaluation_metric_directions: list[Literal["min", "max"]] | None = None evaluation_metrics: list[str] | None = None _compiled_config: _CompiledValue = PrivateAttr() @@ -694,9 +684,9 @@ def validate_search_controls(self): if self.evaluation_metrics is not None: if not self.evaluation_metrics: raise ValueError("evaluation_metrics cannot be empty") - if self.evaluation_script is None: + if self.evaluation_script_path is None: raise ValueError( - "evaluation_script must be provided if evaluation_metrics " + "evaluation_script_path must be provided if evaluation_metrics " "is defined." ) if self.evaluation_metric_directions is None: @@ -709,26 +699,26 @@ def validate_search_controls(self): "evaluation_metrics and evaluation_metric_directions must have " "the same number of values" ) - if self.evaluation_inference_config is None: + if self.evaluation_inference_config_path is None: warnings.warn( - "Please provide evaluation_inference_config if your " - "evaluation_script requires inference outputs", + "Please provide evaluation_inference_config_path if your " + "evaluation_script_path requires inference outputs", stacklevel=2, ) - if self.evaluation_script is not None and not os.path.exists( - os.path.join(self.project_root, self.evaluation_script) + if self.evaluation_script_path is not None and not os.path.exists( + os.path.join(self.project_root, self.evaluation_script_path) ): raise ValueError( - f"evaluation_script {self.evaluation_script!r} does not exist " + f"evaluation_script_path {self.evaluation_script_path!r} does not exist " f"under project_root {self.project_root!r}" ) - if self.evaluation_inference_config is not None: - inference_path = self.evaluation_inference_config + if self.evaluation_inference_config_path is not None: + inference_path = self.evaluation_inference_config_path if not os.path.isabs(inference_path): inference_path = os.path.join(self.project_root, inference_path) if not os.path.exists(inference_path): raise ValueError( - f"evaluation_inference_config {self.evaluation_inference_config!r} " + f"evaluation_inference_config_path {self.evaluation_inference_config_path!r} " "does not exist" ) return self @@ -800,14 +790,14 @@ def _materialized_values(self, trial: Any | None, run_index: int) -> dict[str, A if ( isinstance(architecture_override, dict) and "position_encoding" in architecture_override - and "positional_encoding_scope" not in architecture_override + and "position_encoding_scope" not in architecture_override ): architecture = values["model"]["backbone"]["architecture"] if architecture["position_encoding"]["type"] in { "range", "range_concat", }: - architecture["positional_encoding_scope"] = "global" + architecture["position_encoding_scope"] = "global" values["project_root"] = self.project_root trial_count = self.trials if self.trials is not None else self.grid_size() values["model_name"] = f"{self.name}-run-{run_index:0{len(str(trial_count))}d}" @@ -882,10 +872,6 @@ def compile_canonical_hyperparameter_search_config( search_values.setdefault("project_root", base_config.project_root) try: base_values = base_config.model_dump(mode="python") - search_values["parameters"] = normalize_train_config_parameter_surface( - search_values.get("parameters"), - base_values, - ) search_config = CanonicalHyperparameterSearchConfig.model_validate( search_values ) diff --git a/src/sequifier/config/components.py b/src/sequifier/config/components.py index 8b025d21..7c2bbaf1 100644 --- a/src/sequifier/config/components.py +++ b/src/sequifier/config/components.py @@ -1,12 +1,10 @@ """Reusable configuration components shared by training and artifacts.""" import math -from collections.abc import Mapping from itertools import product from typing import Annotated, Any, Literal, Optional, TypeAlias, Union from pydantic import ( - BaseModel, ConfigDict, Field, RootModel, @@ -18,6 +16,7 @@ ) from sequifier.config.initialization_config import ModelInitializationConfig +from sequifier.config.naming import ConfigModel from sequifier.config.probabilities import ProbabilityDistribution from sequifier.typechecking import beartype @@ -25,7 +24,7 @@ NextOccurrenceTargetValue: TypeAlias = StrictInt | StrictStr -class CartesianLayoutModel(BaseModel): +class CartesianLayoutModel(ConfigModel): """Reusable coordinate annotation for flat feature columns.""" model_config = ConfigDict(extra="forbid") @@ -94,7 +93,7 @@ def __getitem__(self, key: str) -> CartesianLayoutModel: return self.root[key] -class IngestionComponentBase(BaseModel): +class IngestionComponentBase(ConfigModel): """Settings shared by every feature-ingestion component.""" model_config = ConfigDict(extra="forbid") @@ -125,7 +124,7 @@ def serialize_optional_seed(self, serializer): return values -class MultiHashEmbeddingConfig(BaseModel): +class MultiHashEmbeddingConfig(ConfigModel): """Embed one categorical variable through independent hash tables.""" model_config = ConfigDict(extra="forbid") @@ -136,7 +135,7 @@ class MultiHashEmbeddingConfig(BaseModel): seed: StrictInt = Field(default=0, ge=0, le=2**63 - 1) -class QRHashEmbeddingConfig(BaseModel): +class QRHashEmbeddingConfig(ConfigModel): """Embed one categorical variable with quotient and remainder tables.""" model_config = ConfigDict(extra="forbid") @@ -276,7 +275,7 @@ class TemporalConvIngestionConfig(IngestionComponentBase): causal: bool = True activation: Literal["relu", "gelu", "silu"] = "gelu" dropout: float = Field(0.0, ge=0.0, lt=1.0) - post_conv_norm: Literal["layer_norm", "rmsnorm", "none"] = "layer_norm" + post_conv_normalization: Literal["layer_norm", "rmsnorm", "none"] = "layer_norm" orientation: Literal["within_column", "within_item_position"] = ( "within_item_position" ) @@ -333,7 +332,7 @@ def dilation_schedule(self) -> list[int]: return [self.dilation] * self.num_layers -class AxisProjectionBlockModel(BaseModel): +class AxisProjectionBlockModel(ConfigModel): """Flatten configured cartesian axes and project them with a linear layer.""" model_config = ConfigDict(extra="forbid") @@ -351,7 +350,7 @@ def validate_axes_unique(self): return self -class AxisConvBlockModel(BaseModel): +class AxisConvBlockModel(ConfigModel): """Sweep a native 1D/2D/3D convolution over configured cartesian axes.""" model_config = ConfigDict(extra="forbid") @@ -372,7 +371,7 @@ def validate_axes_unique(self): return self -class AxisAttentionBlockModel(BaseModel): +class AxisAttentionBlockModel(ConfigModel): """Apply self-attention over one or more configured cartesian axes.""" model_config = ConfigDict(extra="forbid") @@ -380,7 +379,7 @@ class AxisAttentionBlockModel(BaseModel): type: Literal["axis_attention"] axes: list[str] = Field(..., min_length=1) output_dim: int = Field(..., gt=0) - n_heads: int = Field(1, gt=0) + num_heads: int = Field(1, gt=0) dropout: float = Field(0.0, ge=0.0, lt=1.0) unshared_axes: list[str] = Field(default_factory=list) @@ -389,19 +388,19 @@ class AxisAttentionBlockModel(BaseModel): def validate_axes_unique(self): _validate_axis_list_unique(self.axes, "axes") _validate_axis_list_unique(self.unshared_axes, "unshared_axes") - if self.output_dim % self.n_heads != 0: - raise ValueError("axis_attention output_dim must be divisible by n_heads") + if self.output_dim % self.num_heads != 0: + raise ValueError("axis_attention output_dim must be divisible by num_heads") return self -class AxisPoolBlockModel(BaseModel): +class AxisPoolBlockModel(ConfigModel): """Reduce configured cartesian axes without changing the channel dimension.""" model_config = ConfigDict(extra="forbid") type: Literal["axis_pool"] axes: list[str] = Field(..., min_length=1) - mode: Literal["mean", "sum", "max"] = "mean" + pooling: Literal["mean", "sum", "max"] = "mean" @model_validator(mode="after") @beartype @@ -458,7 +457,7 @@ def _validate_feature_embedding_dims( raise ValueError(f"{field_name} values must be positive: {invalid_dims}") -class AxisEmbeddingModel(BaseModel): +class AxisEmbeddingModel(ConfigModel): """Optional positional encoding for cartesian layout axes.""" model_config = ConfigDict(extra="forbid") @@ -503,7 +502,7 @@ class StructuredIngestionConfig(IngestionComponentBase): processing_blocks: list[StructuredProcessingBlock] = Field(default_factory=list) image_cells: dict[str, str] = Field(default_factory=dict) image_encoder: "ImageEncoderConfig | None" = None - image_frame_pool: Literal["mean", "attention"] = "mean" + image_frame_pooling: Literal["mean", "attention"] = "mean" @field_validator("axis_embeddings", mode="before") @classmethod @@ -524,7 +523,7 @@ def validate_image_cells(self): return self -class ImageEncoderConfig(BaseModel): +class ImageEncoderConfig(ConfigModel): model_config = ConfigDict(extra="forbid") type: Literal["resnet", "frozen"] = "resnet" @@ -586,7 +585,7 @@ class ImageIngestionConfig(IngestionComponentBase): views: list[str] = Field(..., min_length=1) output_dim: int = Field(..., gt=0) encoder: ImageEncoderConfig = Field(default_factory=ImageEncoderConfig) - frame_pool: Literal["mean", "attention"] = "mean" + frame_pooling: Literal["mean", "attention"] = "mean" view_merge: Literal["mean", "concat"] = "concat" @field_validator("views") @@ -599,7 +598,7 @@ def validate_views(cls, views): StructuredIngestionConfig.model_rebuild() -class LinearDecodingConfig(BaseModel): +class LinearDecodingConfig(ConfigModel): """Project each support window directly to target outputs.""" model_config = ConfigDict(extra="forbid") @@ -616,7 +615,7 @@ def validate_target_columns(cls, v): return v -class MLPDecodingConfig(BaseModel): +class MLPDecodingConfig(ConfigModel): """Flatten support windows and decode targets with a shared MLP branch.""" model_config = ConfigDict(extra="forbid") @@ -652,7 +651,7 @@ def validate_hidden_dims(cls, v): return v -class AutoregressiveTransformerDecodingConfig(BaseModel): +class AutoregressiveTransformerDecodingConfig(ConfigModel): """Decode ordered targets autoregressively within each temporal position.""" model_config = ConfigDict(extra="forbid") @@ -710,13 +709,13 @@ def validate_decoder(self): DecodingSpecConfig = BranchDecodingConfig | dict[str, BranchDecodingConfig] -class DecoderComponentBase(BaseModel): +class DecoderComponentBase(ConfigModel): """Settings shared by every target-decoder component.""" model_config = ConfigDict(extra="forbid") prediction_length: int = Field(..., gt=0) - support: int = Field(1, gt=0) + support_length: int = Field(1, gt=0) initialization: ModelInitializationConfig = Field( default_factory=ModelInitializationConfig ) @@ -762,60 +761,60 @@ def validate_branch_names(cls, branches): ] -class BackboneAttentionConfig(BaseModel): +class BackboneAttentionConfig(ConfigModel): model_config = ConfigDict(extra="forbid") type: Literal["mha", "mqa", "gqa"] = "mha" - n_heads: int = Field(..., gt=0) - n_kv_heads: Optional[int] = Field(default=None, gt=0) + num_heads: int = Field(..., gt=0) + num_kv_heads: Optional[int] = Field(default=None, gt=0) output_projection: bool = True @model_validator(mode="after") @beartype def validate_heads(self): - if self.n_kv_heads is None: + if self.num_kv_heads is None: if self.type == "gqa": - raise ValueError("gqa requires n_kv_heads") - self.n_kv_heads = 1 if self.type == "mqa" else self.n_heads - if self.n_heads % self.n_kv_heads != 0: + raise ValueError("gqa requires num_kv_heads") + self.num_kv_heads = 1 if self.type == "mqa" else self.num_heads + if self.num_heads % self.num_kv_heads != 0: raise ValueError( - f"n_heads {self.n_heads} must be divisible by n_kv_heads " - f"{self.n_kv_heads}" + f"num_heads {self.num_heads} must be divisible by num_kv_heads " + f"{self.num_kv_heads}" ) - if self.type == "mha" and self.n_kv_heads != self.n_heads: - raise ValueError("mha requires n_kv_heads to equal n_heads") - if self.type == "mqa" and self.n_kv_heads != 1: - raise ValueError("mqa requires n_kv_heads to equal 1") + if self.type == "mha" and self.num_kv_heads != self.num_heads: + raise ValueError("mha requires num_kv_heads to equal num_heads") + if self.type == "mqa" and self.num_kv_heads != 1: + raise ValueError("mqa requires num_kv_heads to equal 1") return self -class BackboneFeedForwardConfig(BaseModel): +class BackboneFeedForwardConfig(ConfigModel): model_config = ConfigDict(extra="forbid") - dim: int = Field(..., gt=0) + hidden_dim: int = Field(..., gt=0) activation: Literal["relu", "gelu", "swiglu"] = "swiglu" -class BackboneNormalizationConfig(BaseModel): +class BackboneNormalizationConfig(ConfigModel): model_config = ConfigDict(extra="forbid") type: Literal["layer_norm", "rmsnorm"] = "rmsnorm" norm_first: bool = True -class BackbonePositionEncodingConfig(BaseModel): +class BackbonePositionEncodingConfig(ConfigModel): model_config = ConfigDict(extra="forbid") type: Literal["learned", "rope", "range", "range_concat", "sinusoidal"] = "learned" theta: float = Field(10000.0, gt=0.0) -class TransformerEncoderArchitectureConfig(BaseModel): +class TransformerEncoderArchitectureConfig(ConfigModel): """Capacity-independent transformer architecture shared by both encoders.""" model_config = ConfigDict(extra="forbid") - dim_model: int = Field(..., gt=0) + model_dim: int = Field(..., gt=0) num_layers: int = Field(..., gt=0) attention: BackboneAttentionConfig feed_forward: BackboneFeedForwardConfig @@ -831,13 +830,13 @@ class TransformerEncoderArchitectureConfig(BaseModel): @model_validator(mode="after") @beartype def validate_architecture(self): - n_heads = self.attention.n_heads - if self.dim_model % n_heads != 0: + num_heads = self.attention.num_heads + if self.model_dim % num_heads != 0: raise ValueError( - f"dim_model {self.dim_model} must be divisible by n_heads {n_heads}" + f"model_dim {self.model_dim} must be divisible by num_heads {num_heads}" ) if self.position_encoding.type == "rope": - head_dim = self.dim_model // n_heads + head_dim = self.model_dim // num_heads if head_dim % 2 != 0: raise ValueError( f"RoPE requires an even head dimension, got {head_dim}" @@ -870,7 +869,7 @@ def validate_architecture(self): class BackboneArchitectureConfig(TransformerEncoderArchitectureConfig): max_context_length: int = Field(..., gt=0) - positional_encoding_scope: Literal["per_feature", "global"] = "per_feature" + position_encoding_scope: Literal["per_feature", "global"] = "per_feature" @model_validator(mode="before") @classmethod @@ -886,24 +885,24 @@ def default_global_position_scope(cls, values): ) if ( position_type in {"range", "range_concat", "sinusoidal"} - and "positional_encoding_scope" not in values + and "position_encoding_scope" not in values ): values = dict(values) - values["positional_encoding_scope"] = "global" + values["position_encoding_scope"] = "global" return values @model_validator(mode="after") def validate_temporal_positions(self): if ( self.position_encoding.type in {"range", "range_concat", "sinusoidal"} - and self.positional_encoding_scope != "global" + and self.position_encoding_scope != "global" ): raise ValueError( f"position_encoding type {self.position_encoding.type!r} requires " - "positional_encoding_scope 'global'" + "position_encoding_scope 'global'" ) - if self.position_encoding.type == "range_concat" and self.dim_model < 2: - raise ValueError("range_concat requires dim_model to be at least 2") + if self.position_encoding.type == "range_concat" and self.model_dim < 2: + raise ValueError("range_concat requires model_dim to be at least 2") return self @@ -938,7 +937,7 @@ def validate_depth(self): return self -class IngestionMergeConfig(BaseModel): +class IngestionMergeConfig(ConfigModel): """How composite branch outputs are merged.""" model_config = ConfigDict(extra="forbid") @@ -997,7 +996,7 @@ def validate_branch_names(cls, branches): IngestionSpecConfig = IngestionComponentConfig | dict[str, IngestionComponentConfig] -class BackboneRepositoryConfig(BaseModel): +class BackboneRepositoryConfig(ConfigModel): model_config = ConfigDict(extra="forbid") backbone_id: StrictStr = Field(..., min_length=1) @@ -1007,7 +1006,7 @@ class BackboneRepositoryConfig(BaseModel): conflict_policy: Literal["compare_and_swap"] = "compare_and_swap" -class BackboneComponentConfig(BaseModel): +class BackboneComponentConfig(ConfigModel): model_config = ConfigDict(extra="forbid") initialization_seed: Optional[StrictInt] = Field(default=None, ge=0, le=2**63 - 1) @@ -1027,7 +1026,7 @@ def serialize_optional_seed(self, serializer): return values -class ComponentSpec(BaseModel): +class ComponentSpec(ConfigModel): """A named component and its constructor arguments.""" model_config = ConfigDict(extra="forbid", frozen=True) @@ -1035,28 +1034,8 @@ class ComponentSpec(BaseModel): name: str = Field(min_length=1) arguments: dict[str, Any] = Field(default_factory=dict) - @model_validator(mode="before") - @classmethod - @beartype - def parse_flattened(cls, value): - if isinstance(value, cls): - return value - if not isinstance(value, Mapping): - raise TypeError("component configuration must be a mapping") - if "arguments" in value: - return value - return { - "name": value.get("name"), - "arguments": {key: item for key, item in value.items() if key != "name"}, - } - - @model_serializer - @beartype - def serialize_flattened(self) -> dict[str, Any]: - return {"name": self.name, **self.arguments} - -class ReplacementDistribution(BaseModel): +class ReplacementDistribution(ConfigModel): model_config = ConfigDict(extra="forbid") masked: float = Field(..., ge=0.0, le=1.0) @@ -1074,7 +1053,7 @@ def validate_sum(self): return self -class BERTSpecModel(BaseModel): +class BERTSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") masking_probability: float = Field(..., gt=0.0, le=1.0) @@ -1082,14 +1061,14 @@ class BERTSpecModel(BaseModel): span_masking: ProbabilityDistribution -class NextOccurrenceConfigModel(BaseModel): +class NextOccurrenceConfigModel(ConfigModel): model_config = ConfigDict(extra="forbid") column_name: str target_values: list[NextOccurrenceTargetValue] = Field(..., min_length=1) -class ResumeConfig(BaseModel): +class ResumeConfig(ConfigModel): """Policy for resuming the same run from its complete checkpoint.""" model_config = ConfigDict(extra="forbid") diff --git a/src/sequifier/config/composition.py b/src/sequifier/config/composition.py index ee713d8b..f472f18c 100644 --- a/src/sequifier/config/composition.py +++ b/src/sequifier/config/composition.py @@ -34,13 +34,7 @@ def _is_atomic(path: ConfigPath, atomic_paths: frozenset[ConfigPath]) -> bool: "model", "interfaces", ) - singleton_interface_component = len(path) == 3 and path[:2] == ( - "model", - "interface", - ) - return (named_interface_component or singleton_interface_component) and path[ - -1 - ] in {"ingestion", "decoder"} + return named_interface_component and path[-1] in {"ingestion", "decoder"} @beartype @@ -185,20 +179,18 @@ def _normalize_additional_config_paths( ) -> list[str]: if value is None: return [] - if isinstance(value, str): - paths = [value] - elif isinstance(value, list): + if isinstance(value, list): paths = value else: raise ValueError( f"Config '{config_path}' field '{ADDITIONAL_CONFIG_PATHS_KEY}' must " - "be a non-empty string, a list of non-empty strings, or null." + "be a list of non-empty strings, or null." ) if any(not isinstance(path, str) or not path.strip() for path in paths): raise ValueError( f"Config '{config_path}' field '{ADDITIONAL_CONFIG_PATHS_KEY}' must " - "be a non-empty string, a list of non-empty strings, or null." + "be a list of non-empty strings, or null." ) return paths @@ -325,7 +317,7 @@ def _deep_merge_dicts( def _changes_discriminator( base: Mapping[str, Any], override: Mapping[str, Any] ) -> bool: - for discriminator in ("type", "name", "algorithm", "model_type"): + for discriminator in ("type", "name", "algorithm", "task"): if ( discriminator in base and discriminator in override diff --git a/src/sequifier/config/depth_layout.py b/src/sequifier/config/depth_layout.py index 47094157..0040070b 100644 --- a/src/sequifier/config/depth_layout.py +++ b/src/sequifier/config/depth_layout.py @@ -2,19 +2,21 @@ from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, RootModel, model_validator +from pydantic import ConfigDict, Field, RootModel, model_validator from torch import nn +from sequifier.config.naming import ConfigModel + def depth_mask_metadata_key(layout: str) -> str: return f"depth_valid_mask:{layout}" -class DepthLayoutModel(BaseModel): +class DepthLayoutModel(ConfigModel): model_config = ConfigDict(extra="forbid") position_column: str columns: list[str] = Field(min_length=1) - context_length: int = Field(gt=0) + max_depth_length: int = Field(gt=0) position_base: Literal[0, 1] = 0 allow_gaps: bool = False diff --git a/src/sequifier/config/freezing_config.py b/src/sequifier/config/freezing_config.py index 927fd870..ffae7899 100644 --- a/src/sequifier/config/freezing_config.py +++ b/src/sequifier/config/freezing_config.py @@ -3,7 +3,6 @@ from typing import Optional from pydantic import ( - BaseModel, ConfigDict, Field, field_validator, @@ -12,18 +11,19 @@ ) from sequifier.config.layer_groups import LayerGroup +from sequifier.config.naming import ConfigModel from sequifier.typechecking import beartype -class LayerFreezingConfigFields(BaseModel): +class LayerFreezingConfigFields(ConfigModel): """Mutually exclusive semantic layer-freezing selections.""" model_config = ConfigDict(extra="forbid") - freeze: Optional[list[LayerGroup]] = None - freezing_except: Optional[list[LayerGroup]] = None + freeze_groups: Optional[list[LayerGroup]] = None + freeze_except_groups: Optional[list[LayerGroup]] = None - @field_validator("freeze", "freezing_except") + @field_validator("freeze_groups", "freeze_except_groups") @classmethod @beartype def validate_unique_groups( @@ -36,9 +36,9 @@ def validate_unique_groups( @model_validator(mode="after") @beartype def validate_mutually_exclusive(self): - if self.freeze is not None and self.freezing_except is not None: + if self.freeze_groups is not None and self.freeze_except_groups is not None: raise ValueError( - "freeze and freezing_except are mutually exclusive; " + "freeze_groups and freeze_except_groups are mutually exclusive; " "only one may be non-null" ) return self @@ -49,16 +49,16 @@ def serialize_freezing_fields(self, serializer): """Omit inactive fields from canonical dataset freezing policies.""" values = serializer(self) - if self.freeze is None: - values.pop("freeze", None) - if self.freezing_except is None: - values.pop("freezing_except", None) + if self.freeze_groups is None: + values.pop("freeze_groups", None) + if self.freeze_except_groups is None: + values.pop("freeze_except_groups", None) return values @property @beartype def has_freezing_policy(self) -> bool: - return self.freeze is not None or self.freezing_except is not None + return self.freeze_groups is not None or self.freeze_except_groups is not None class IngestionFreezingConfig(LayerFreezingConfigFields): @@ -75,10 +75,10 @@ def has_freezing_policy(self) -> bool: @model_serializer(mode="wrap") def serialize_ingestion_policy(self, serializer): values = serializer(self) - if self.freeze is None: - values.pop("freeze", None) - if self.freezing_except is None: - values.pop("freezing_except", None) + if self.freeze_groups is None: + values.pop("freeze_groups", None) + if self.freeze_except_groups is None: + values.pop("freeze_except_groups", None) if not self.branches: values.pop("branches", None) return values diff --git a/src/sequifier/config/image_config.py b/src/sequifier/config/image_config.py index 4149d887..51ae56b8 100644 --- a/src/sequifier/config/image_config.py +++ b/src/sequifier/config/image_config.py @@ -1,11 +1,13 @@ """Image source and frame-alignment configuration.""" -from typing import Any, Literal, Optional, Union +from typing import Any, Literal, Optional -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import ConfigDict, Field, model_validator +from sequifier.config.naming import ConfigModel -class FrameAlignment(BaseModel): + +class FrameAlignment(ConfigModel): model_config = ConfigDict(extra="forbid") frames: int = Field(default=1, gt=0) @@ -46,7 +48,7 @@ def max_frames_per_item(self) -> int: return (self.frames + self.items - 1) // self.items -class ImageSourceConfig(BaseModel): +class ImageSourceConfig(ConfigModel): model_config = ConfigDict(extra="forbid") path: str | None = None @@ -79,10 +81,10 @@ def validate_size(self): return self -class PreconditioningImageConfig(BaseModel): +class PreconditioningImageConfig(ConfigModel): model_config = ConfigDict(extra="forbid") - path: str | dict[str, str] | None = None + paths: dict[str, str] | None = None view_names: list[str] = Field(default_factory=list) resize_size: int = Field(default=256, gt=0) crop_size: int = Field(default=224, gt=0) @@ -92,76 +94,61 @@ class PreconditioningImageConfig(BaseModel): def validate_size(self): if self.crop_size > self.resize_size: raise ValueError("crop_size must not exceed resize_size") - if isinstance(self.path, dict) and ( - not self.path + if self.paths is not None and ( + not self.paths or any( - not name.isidentifier() or not path for name, path in self.path.items() + not name.isidentifier() or not path for name, path in self.paths.items() ) ): raise ValueError("Preconditioning views require names and folder paths") - if isinstance(self.path, str) and not self.path: - raise ValueError("Preconditioning image path cannot be empty") - if self.path is None and not self.view_names: + if self.paths is None and not self.view_names: raise ValueError("Preconditioning image views are required") return self @property def views(self) -> dict[str, str]: - if isinstance(self.path, dict): - return self.path - if isinstance(self.path, str): - return {"default": self.path} - return {name: "" for name in self.view_names} + return ( + self.paths + if self.paths is not None + else {name: "" for name in self.view_names} + ) def compatibility_signature(self) -> dict: return { - **self.model_dump(exclude={"path", "signature", "view_names"}), + **self.model_dump(exclude={"paths", "signature", "view_names"}), "views": sorted(self.views), } -MediaPath = Optional[Union[str, dict[str, str]]] +MediaPath = Optional[dict[str, str]] def bind_image_paths( sources: dict[str, ImageSourceConfig], preconditioning: PreconditioningImageConfig | None, - image_path: MediaPath, - preconditioning_image_path: MediaPath, + image_paths: MediaPath, + preconditioning_image_paths: MediaPath, ) -> tuple[dict[str, ImageSourceConfig], PreconditioningImageConfig | None]: - """Attach explicit dataset media locations to path-free metadata.""" - if isinstance(image_path, str): - if len(sources) != 1: - raise ValueError( - "image_path must map view names when there are multiple views" - ) - paths = {next(iter(sources)): image_path} - else: - paths = image_path or {} + """Attach explicit named media locations to path-free metadata.""" + paths = image_paths or {} if set(paths) != set(sources) or any(not path for path in paths.values()): - raise ValueError("image_path must provide exactly the selected image views") + raise ValueError("image_paths must provide exactly the selected image views") bound = { name: source.model_copy(update={"path": paths[name]}) for name, source in sources.items() } if preconditioning is None: - if preconditioning_image_path is not None: - raise ValueError("preconditioning_image_path has no metadata source") + if preconditioning_image_paths is not None: + raise ValueError("preconditioning_image_paths has no metadata source") return bound, None - names = set(preconditioning.views) - if isinstance(preconditioning_image_path, str): - if names != {"default"}: - raise ValueError("preconditioning_image_path must map named views") - prefix_paths = preconditioning_image_path - else: - prefix_paths = preconditioning_image_path or {} - if set(prefix_paths) != names or any( - not path for path in prefix_paths.values() - ): - raise ValueError( - "preconditioning_image_path must provide exactly the selected views" - ) - return bound, preconditioning.model_copy(update={"path": prefix_paths}) + prefix_paths = preconditioning_image_paths or {} + if set(prefix_paths) != set(preconditioning.views) or any( + not path for path in prefix_paths.values() + ): + raise ValueError( + "preconditioning_image_paths must provide exactly the selected views" + ) + return bound, preconditioning.model_copy(update={"paths": prefix_paths}) def image_views(ingestion: Any) -> set[str]: diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 33de83c3..b97d9033 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -1,7 +1,7 @@ import json import os import re -from typing import Any, Generic, Literal, Optional, TypeVar, Union +from typing import Any, Generic, Literal, Optional, TypeVar import numpy as np from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator @@ -30,7 +30,6 @@ StoredWindowLayout, canonicalize_polars_dtype_name, derive_target_column_types, - metadata_config_path_from_preprocessing_data_path, normalize_path, resolve_window_view, try_catch_excess_keys, @@ -154,7 +153,7 @@ def _execution_source( "context_length": global_spec.context_length, "target_offset": global_spec.target_offset, "prediction_length": interface.decoder.prediction_length, - "decoder_support": interface.decoder.support, + "decoder_support_length": interface.decoder.support_length, }, DatasetMetadata( depth_layouts=interface.depth_layouts, @@ -406,7 +405,7 @@ def load_inferer_config( raise ValueError("skip_metadata is not supported for policy inference") return policy_config forecast_values, _ = extract_inline_metadata(authored_values) - InferenceConfig.model_validate(forecast_values) + authored_values["seed"] = InferenceConfig.model_validate(forecast_values).seed authored_values, extracted_metadata_values = extract_inline_metadata( authored_values ) @@ -420,9 +419,7 @@ def load_inferer_config( image_selections: list[tuple[set[str], bool]] = [] dataset_name = authored_values.get("dataset") - model_paths = authored_values.get("model_path") - if isinstance(model_paths, str): - model_paths = [model_paths] + model_paths = authored_values.get("model_paths") training_config_path = authored_values.get("training_config_path") route_selected = ( dataset_name is not None or authored_values.get("model_interface") is not None @@ -431,7 +428,9 @@ def load_inferer_config( route_selected or ( isinstance(model_paths, list) - and ("model_path" not in cli_values or "training_config_path" in cli_values) + and ( + "model_paths" not in cli_values or "training_config_path" in cli_values + ) and any( isinstance(path, str) and path.lower().endswith(".pt") for path in model_paths @@ -502,7 +501,7 @@ def load_inferer_config( if model_config is None: continue if ( - "model_path" in cli_values + "model_paths" in cli_values and "training_config_path" not in cli_values and not route_selected ): @@ -511,7 +510,7 @@ def load_inferer_config( model_path, model_config, interface_name=authored_values.get("model_interface"), - task="forecast", + task=authored_values["task"], ) training = selection.resolve(str(authored_values.get("device", "cpu"))) interface_name = selection.interface_name @@ -577,7 +576,7 @@ def load_inferer_config( else: raise ValueError( f"Inference config '{config_path}' must define metadata_config_path " - "or preprocessing_data_path unless the PT artifact contains model " + "unless a selected model artifact contains model " "metadata." ) @@ -612,10 +611,6 @@ def load_inferer_config( def _effective_metadata_config_path(config: "InferenceConfig") -> Optional[str]: if config.metadata_config_path: return config.metadata_config_path - if config.preprocessing_data_path: - return metadata_config_path_from_preprocessing_data_path( - config.preprocessing_data_path - ) return None @@ -711,7 +706,7 @@ def resolve_inference_config( if selected_layouts: if config.read_format != "pt": raise ValueError("Preprocessed depth inputs require read_format: pt") - if config.autoregressive: + if config.autoregressive_generation: raise ValueError("Depth inputs do not support autoregressive inference") target_column_types = config.target_column_types or derive_target_column_types( config.target_columns, column_data_types @@ -812,7 +807,7 @@ def resolve_inference_config( f"Aligned inference requires window width at least {required}" ) if aligned_data_path: - if config.autoregressive: + if config.autoregressive_generation: raise ValueError( "Prediction aligned inference does not support autoregressive output" ) @@ -875,11 +870,9 @@ class _InferenceConfigBase( model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - preprocessing_data_path: Optional[str] = None metadata_config_path: Optional[str] = None - model_path: Union[str, list[str]] - model_type: str - task: Literal["forecast"] | None = None + model_paths: list[str] = Field(min_length=1) + task: Literal["forecast", "embedding"] training_objective: _ObjectiveT data_path: _DataPathT training_config_path: Optional[str] = None @@ -889,8 +882,8 @@ class _InferenceConfigBase( input_columns: _InputColumnsT static_columns: list[str] = Field(default_factory=list) static_features_path: Optional[str] = None - image_path: MediaPath = None - preconditioning_image_path: MediaPath = None + image_paths: MediaPath = None + preconditioning_image_paths: MediaPath = None target_columns: list[str] column_data_types: _ColumnTypesT target_column_types: _ColumnTypesT @@ -901,20 +894,18 @@ class _InferenceConfigBase( target_offset: int = Field(default=1, ge=0) window_stride: Optional[int] = Field(default=None, gt=0) prediction_length: Optional[int] = None - decoder_support: int = Field(default=1, gt=0) + decoder_support_length: int = Field(default=1, gt=0) prediction_alignment_steps: int = Field(default=1, gt=0) sample_from_distribution_columns: Optional[list[str]] = Field(default=None) infer_with_dropout: bool = Field(default=False) rollout: RolloutConfig | None = None - autoregressive: bool = Field(default=False) + autoregressive_generation: bool = Field(default=False) generation_steps: Optional[int] = Field(default=None) @model_validator(mode="after") @beartype def validate_route_selection(self): - if self.model_type == "embedding" and self.task is not None: - raise ValueError("task is only supported for generative inference") for usage, value in ( ("dataset", self.dataset), ("part", self.part), @@ -926,14 +917,6 @@ def validate_route_selection(self): raise ValueError("part selection requires dataset selection") return self - @field_validator("input_columns", mode="before") - @classmethod - @beartype - def normalize_single_input_column(cls, value): - if isinstance(value, str): - return [value] - return value - @field_validator("training_objective") @classmethod @beartype @@ -944,14 +927,6 @@ def validate_authored_training_objective(cls, value: str | None) -> str | None: ) return value - @field_validator("model_type") - @classmethod - @beartype - def validate_authored_model_type(cls, value: str) -> str: - if value not in {"embedding", "generative"}: - raise ValueError("forecast model_type must be generative or embedding") - return value - @model_validator(mode="after") @beartype def validate_authored_paths(self): @@ -984,11 +959,11 @@ def validate_embedding_options(self): "sample_from_distribution_columns", "decode_categories", "output_probabilities", - "autoregressive", + "autoregressive_generation", "generation_steps", "rollout", } - if self.model_type == "embedding" and ( + if self.task == "embedding" and ( invalid := self.model_fields_set & generative_fields ): raise ValueError(f"embedding inference does not support {sorted(invalid)}") @@ -1021,11 +996,11 @@ class ResolvedInferenceConfig( def validate_numeric_rollout(self): if self.rollout is None: return self - if self.autoregressive or self.generation_steps is not None: + if self.autoregressive_generation or self.generation_steps is not None: raise ValueError( "rollout uses max_steps, not autoregressive/generation_steps" ) - if self.device != "cpu" or self.model_type != "generative": + if self.device != "cpu" or self.task != "forecast": raise ValueError("numeric rollout requires a CPU generative model") if ( self.training_objective not in {"causal", "final_value"} @@ -1039,11 +1014,9 @@ def validate_numeric_rollout(self): raise ValueError( "numeric rollout does not support categorical probabilities or sampling" ) - paths = ( - self.model_path if isinstance(self.model_path, list) else [self.model_path] - ) + paths = self.model_paths if len(paths) != 1: - raise ValueError("rollout requires exactly one model_path") + raise ValueError("rollout requires exactly one entry in model_paths") fields = self.input_columns + self.static_columns + self.target_columns if not self.input_columns or any( "float" not in self.column_data_types[column].lower() for column in fields @@ -1074,8 +1047,8 @@ def image_store(self): sources, preconditioning = bind_image_paths( self.dataset_metadata.image_sources, self.dataset_metadata.preconditioning_image, - self.image_path, - self.preconditioning_image_path, + self.image_paths, + self.preconditioning_image_paths, ) self._image_store = ImageStore( sources, @@ -1102,13 +1075,6 @@ def derive_optional_config_values(cls, values): values["target_offset"] = window_view.get("target_offset", 1) elif "target_offset" not in values and isinstance(window_view, ModelWindowView): values["target_offset"] = window_view.target_offset - preprocessing_data_path = values.get("preprocessing_data_path") - if values.get("metadata_config_path") is None and preprocessing_data_path: - values["metadata_config_path"] = ( - metadata_config_path_from_preprocessing_data_path( - preprocessing_data_path - ) - ) if ( values.get("target_column_types") is None and values.get("column_data_types") is not None @@ -1142,11 +1108,13 @@ def normalize_prediction_length(self): self.window_view.context_length ) if objective_class.forward_looking: - decoded_length = self.window_view.context_length - self.decoder_support + 1 + decoded_length = ( + self.window_view.context_length - self.decoder_support_length + 1 + ) if not 1 <= self.prediction_length <= decoded_length: raise ValueError( "Forward inference prediction_length must be between 1 and " - f"context_length - decoder_support + 1 ({decoded_length})" + f"context_length - decoder_support_length + 1 ({decoded_length})" ) objective_class.validate_prediction_length( self.prediction_length, @@ -1165,24 +1133,11 @@ def validate_training_objective(cls, v): raise ValueError(f"Only {OBJECTIVE_NAME_MESSAGE} are allowed, found {v}") return v - @field_validator("model_type") - @classmethod - @beartype - def validate_model_type(cls, v: str) -> str: - if v not in [ - "embedding", - "generative", - ]: - raise ValueError( - f"model_type must be one of 'embedding' and 'generative, {v} isn't" - ) - return v - @field_validator("output_probabilities") @classmethod @beartype def validate_output_probabilities(cls, v: bool, info: Any) -> bool: - if v and info.data.get("model_type") == "embedding": + if v and info.data.get("task") == "embedding": raise ValueError( "For embedding models, 'output_probabilities' must be set to false" ) @@ -1191,9 +1146,7 @@ def validate_output_probabilities(cls, v: bool, info: Any) -> bool: @model_validator(mode="after") @beartype def validate_training_config_path(self): - model_paths = ( - self.model_path if isinstance(self.model_path, list) else [self.model_path] - ) + model_paths = self.model_paths if not any(path.lower().endswith(".pt") for path in model_paths): return self @@ -1207,16 +1160,16 @@ def validate_training_config_path(self): @classmethod @beartype def validate_generation_steps(cls, v: Optional[int], info: Any) -> Optional[int]: - if v is None and info.data.get("autoregressive") is True: + if v is None and info.data.get("autoregressive_generation") is True: raise ValueError( - "If autoregressive==True, 'generation_steps' needs to be set to an integer value." + "If autoregressive_generation==True, 'generation_steps' needs to be set to an integer value." ) if v is not None and v < 1: raise ValueError("generation_steps must by >= 1.") if v is not None and v > 1: - if not info.data.get("autoregressive"): + if not info.data.get("autoregressive_generation"): raise ValueError( - f"'generation_steps' can only be larger than 1 if 'autoregressive' is true: {info.data.get('autoregressive')}" + f"'generation_steps' can only be larger than 1 if 'autoregressive_generation' is true: {info.data.get('autoregressive_generation')}" ) if info.data.get("rollout") is None and not np.all( @@ -1229,11 +1182,11 @@ def validate_generation_steps(cls, v: Optional[int], info: Any) -> Optional[int] return v - @field_validator("autoregressive") + @field_validator("autoregressive_generation") @classmethod @beartype def validate_autoregressive(cls, v: bool, info: Any): - if v and info.data.get("model_type") == "embedding": + if v and info.data.get("task") == "embedding": raise ValueError( "Autoregressive inference is not possible for embedding models" ) diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index 01fb37f9..2d885e53 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -4,10 +4,12 @@ from dataclasses import dataclass from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import ConfigDict, Field, field_validator +from sequifier.config.naming import ConfigModel, RunSeed -class InferenceExecutionConfig(BaseModel): + +class InferenceExecutionConfig(ConfigModel): model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) project_root: str = "." @@ -15,27 +17,26 @@ class InferenceExecutionConfig(BaseModel): read_format: str = "parquet" write_format: str = "csv" device: str = "cpu" - seed: int = 1010 + seed: RunSeed = 1010 enforce_determinism: bool = False inference_batch_size: int = Field(default=1, gt=0) -class InferenceRouteConfig(BaseModel): +class InferenceRouteConfig(ConfigModel): """Validate artifact and interface selection before route-specific options.""" model_config = ConfigDict(extra="ignore") project_root: str = "." - model_path: str | list[str] - model_type: Literal["generative", "embedding"] + model_paths: list[str] = Field(min_length=1) model_interface: str | None = None - task: Literal["policy", "forecast"] | None = None + task: Literal["forecast", "policy", "embedding"] - @field_validator("model_path") + @field_validator("model_paths") @classmethod def validate_paths(cls, value): - paths = [value] if isinstance(value, str) else value + paths = value if not paths or any(not path.strip() for path in paths): - raise ValueError("model_path requires at least one nonempty artifact path") + raise ValueError("model_paths requires at least one nonempty artifact path") return value @field_validator("model_interface") @@ -45,12 +46,6 @@ def validate_interface(cls, value): raise ValueError("model_interface must be a valid identifier without '.'") return value - @model_validator(mode="after") - def validate_task(self): - if self.model_type == "embedding" and self.task is not None: - raise ValueError("task is only supported for generative inference") - return self - @dataclass(frozen=True) class SelectedInferenceModel: diff --git a/src/sequifier/config/initialization_config.py b/src/sequifier/config/initialization_config.py index a8e7778b..77971d29 100644 --- a/src/sequifier/config/initialization_config.py +++ b/src/sequifier/config/initialization_config.py @@ -3,24 +3,17 @@ import math from typing import Annotated, Any, Literal, TypeAlias, Union -from pydantic import ( - BaseModel, - BeforeValidator, - ConfigDict, - Field, - RootModel, - field_serializer, - model_validator, -) +from pydantic import ConfigDict, Field, RootModel, field_serializer, model_validator from sequifier.config.layer_groups import LayerGroup +from sequifier.config.naming import ConfigModel from sequifier.typechecking import beartype ParameterKind: TypeAlias = Literal["weight", "bias"] InitializationTarget: TypeAlias = tuple[LayerGroup, ParameterKind] -class _InitializationMethod(BaseModel): +class _InitializationMethod(ConfigModel): """Base class for strictly validated initialization methods.""" model_config = ConfigDict(extra="forbid") @@ -52,12 +45,7 @@ def validate_bounds(self): class XavierUniformInitialization(_InitializationMethod): - method: Literal[ - "xavier", - "glorot", - "xavier_uniform", - "glorot_uniform", - ] + method: Literal["xavier_uniform"] gain: float = Field(1.0, ge=0.0) fan_mode: Literal["per_tensor", "joint"] = "per_tensor" @@ -68,7 +56,7 @@ def serialize_method(self, method): class XavierNormalInitialization(_InitializationMethod): - method: Literal["xavier_normal", "glorot_normal"] + method: Literal["xavier_normal"] gain: float = Field(1.0, ge=0.0) fan_mode: Literal["per_tensor", "joint"] = "per_tensor" @@ -113,11 +101,6 @@ class IdentityPlusNormalInitialization(_InitializationMethod): std: float = Field(0.02, ge=0.0) -@beartype -def _expand_method_shorthand(value: Any) -> Any: - return {"method": value} if isinstance(value, str) else value - - InitializationMethodConfig: TypeAlias = Annotated[ Union[ PreserveInitialization, @@ -133,11 +116,10 @@ def _expand_method_shorthand(value: Any) -> Any: IdentityPlusNormalInitialization, ], Field(discriminator="method"), - BeforeValidator(_expand_method_shorthand), ] -class LayerGroupInitialization(BaseModel): +class LayerGroupInitialization(ConfigModel): """Optional weight and bias overrides for one semantic layer group.""" model_config = ConfigDict(extra="forbid") @@ -172,7 +154,7 @@ def configured_targets(self) -> set[InitializationTarget]: } -class InitializationMethodCandidates(BaseModel): +class InitializationMethodCandidates(ConfigModel): """Candidate initialization methods for one weight or bias group.""" model_config = ConfigDict(extra="forbid") @@ -186,7 +168,7 @@ class InitializationMethodCandidates(BaseModel): ] -class LayerGroupInitializationSampling(BaseModel): +class LayerGroupInitializationSampling(ConfigModel): """Fixed or sampled weight and bias initialization for one layer group.""" model_config = ConfigDict(extra="forbid") diff --git a/src/sequifier/config/metadata.py b/src/sequifier/config/metadata.py index 04793885..29c9bbd0 100644 --- a/src/sequifier/config/metadata.py +++ b/src/sequifier/config/metadata.py @@ -6,17 +6,11 @@ import json from typing import Any -from pydantic import ( - AliasChoices, - BaseModel, - ConfigDict, - Field, - field_validator, - model_validator, -) +from pydantic import ConfigDict, Field, field_validator, model_validator from sequifier.config.depth_layout import DepthLayoutRegistryModel from sequifier.config.image_config import ImageSourceConfig, PreconditioningImageConfig +from sequifier.config.naming import ConfigModel from sequifier.helpers import ModelWindowView, StoredWindowLayout from sequifier.special_tokens import SPECIAL_TOKEN_IDS, validate_special_token_ids from sequifier.typechecking import beartype @@ -44,7 +38,7 @@ } -class DatasetMetadata(BaseModel): +class DatasetMetadata(ConfigModel): """The stable subset of preprocessing metadata consumed by other commands.""" model_config = ConfigDict(extra="allow", populate_by_name=True) @@ -70,7 +64,6 @@ def reject_legacy_placement(cls, values: Any) -> Any: split_index: int | None = Field(default=None, ge=0) column_data_types: dict[str, str] = Field( default_factory=dict, - validation_alias=AliasChoices("column_data_types", "column_types"), ) static_columns: list[str] = Field(default_factory=list) static_features_path: str | None = None diff --git a/src/sequifier/config/naming.py b/src/sequifier/config/naming.py new file mode 100644 index 00000000..ce1c7d47 --- /dev/null +++ b/src/sequifier/config/naming.py @@ -0,0 +1,130 @@ +"""Canonical authored names and run-seed resolution. + +Removed names are diagnosed, never translated into accepted aliases. Arbitrary +user dictionaries (columns, constructor arguments, and environment config) are +not inspected as configuration schemas. +""" + +import secrets +from typing import Annotated, Any + +from pydantic import BaseModel, BeforeValidator, model_validator + +RENAMED_FIELDS = { + "interface": "interfaces.", + "dataset": "dataset_training.", + "part": "parts.", + "policy_binding": "policy_bindings.", + "dim_model": "model_dim", + "n_heads": "num_heads", + "n_kv_heads": "num_kv_heads", + "embedding_size": "total_embedding_dim", + "positional_encoding_scope": "position_encoding_scope", + "post_conv_norm": "post_conv_normalization", + "frame_pool": "frame_pooling", + "image_frame_pool": "image_frame_pooling", + "support": "support_length", + "decoder_support": "decoder_support_length", + "device_max_concat_length": "max_concat_tensors", + "accumulation_steps": "accumulation_batches", + "minibatch_size": "batch_size", + "updates": "iterations", + "groups_per_update": "groups_per_iteration", + "rollout_length": "rollout_env_steps", + "log_interval": "log_interval_batches", + "early_stopping_epochs": "early_stopping_patience_epochs", + "gradient_clip": "gradient_clip_norm", + "value_clip": "value_clip_delta", + "target_kl": "old_policy_target_kl", + "kl_coefficient": "reference_kl_coefficient", + "max_weight": "max_advantage_weight", + "weight": "selection_weight", + "save_interval_batches": "snapshot_interval_batches", + "save_interval_minutes": "snapshot_interval_minutes", + "save_latest_interval_minutes": "latest_checkpoint_interval_minutes", + "save_interval_val_loss": "evaluate_on_snapshot", + "calculate_validation_loss_on_initialization": "evaluate_before_training", + "min_freq": "min_count", + "top_k": "max_categories", + "k": "max_categories", + "buckets": "num_buckets", + "column_types": "column_data_types", + "standard_deviation": "std", + "n_cores": "num_workers", + "image_path": "image_paths", + "preconditioning_image_path": "preconditioning_image_paths", + "manifest": "manifest_path", + "freezing_except": "freeze_except_groups", + "model_type": "task: forecast | policy | embedding", + "model_path": "model_paths (a list)", + "autoregressive": "autoregressive_generation", + "global_seed": "sampler_seed", + "model_config_write_path": "trial_config_directory", + "evaluation_script": "evaluation_script_path", + "evaluation_inference_config": "evaluation_inference_config_path", +} + + +class ConfigModel(BaseModel): + @model_validator(mode="before") + @classmethod + def reject_removed_names(cls, values: Any) -> Any: + if not isinstance(values, dict): + return values + fields = cls.model_fields + replacements = dict(RENAMED_FIELDS) + for key in ("evaluation_interval", "checkpoint_interval"): + unit = "iterations" if f"{key}_iterations" in fields else "epochs" + replacements[key] = f"{key}_{unit}" + if "direction" in fields: + replacements["mode"] = "direction: min | max" + if "pooling" in fields: + replacements["mode"] = "pooling" + if "hidden_dim" in fields: + replacements["dim"] = "hidden_dim" + if "max_depth_length" in fields: + replacements["context_length"] = "max_depth_length" + if "freeze_groups" in fields: + replacements["freeze"] = "freeze_groups" + elif "freeze_modules" in fields: + replacements["freeze"] = "freeze_modules" + if "paths" in fields and cls.__name__ == "PreconditioningImageConfig": + replacements["path"] = "paths (a view-to-path mapping)" + if "interface_metadata_paths" in fields: + replacements["metadata_config_path"] = "interface_metadata_paths" + if cls.__name__ == "PreprocessorModel": + replacements["window_length"] = "window_lengths (one value per split)" + if cls.__name__ in {"InferenceConfig", "ResolvedInferenceConfig"}: + replacements["preprocessing_data_path"] = "metadata_config_path" + if cls.__name__ == "SequifierConfig": + replacements["evaluation"] = "training_plan.phases[].evaluation" + for key in values: + if key not in fields and key in replacements: + replacement = replacements[key] + target = replacement.split(":")[0].split(".")[0].split()[0] + if target in fields: + raise ValueError( + f"{cls.__name__}.{key} was removed; use {replacement}" + ) + if cls.__name__ == "TrainingPlanModel" and "phases" not in values: + raise ValueError("Declare training phases under training_plan.phases") + if cls.__name__ == "ComponentSpec" and (extra := values.keys() - fields.keys()): + raise ValueError( + f"Put constructor arguments {sorted(extra)} under arguments" + ) + if "_project_root" in cls.__private_attributes__ and "project_root" in values: + raise ValueError( + "Source project_root was removed; use the run's project_root" + ) + return values + + +def resolve_run_seed(value: Any) -> int: + if value == "random": + return secrets.randbelow(1_000_000_000) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("seed must be an integer or 'random'") + return value + + +RunSeed = Annotated[int, BeforeValidator(resolve_run_seed)] diff --git a/src/sequifier/config/optimizer_config.py b/src/sequifier/config/optimizer_config.py index 04f8d610..e0d27bbc 100644 --- a/src/sequifier/config/optimizer_config.py +++ b/src/sequifier/config/optimizer_config.py @@ -4,12 +4,13 @@ from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import ConfigDict, Field, model_validator from sequifier.config.components import ComponentSpec +from sequifier.config.naming import ConfigModel -class OptimizerSelector(BaseModel): +class OptimizerSelector(ConfigModel): """Match parameter catalog descriptors; supplied conditions are conjunctive.""" model_config = ConfigDict(extra="forbid", frozen=True) @@ -41,7 +42,7 @@ def validate_selector(self) -> "OptimizerSelector": return self -class OptimizerGroupSpec(BaseModel): +class OptimizerGroupSpec(ConfigModel): """One named route and the optimizer applied to its parameters.""" model_config = ConfigDict(extra="forbid", frozen=True) @@ -53,7 +54,7 @@ class OptimizerGroupSpec(BaseModel): base_learning_rate: float | None = Field(default=None, ge=0) -class OptimizerPlan(BaseModel): +class OptimizerPlan(ConfigModel): """Ordered, exhaustive optimizer routes with a final fallback.""" model_config = ConfigDict(extra="forbid", frozen=True) diff --git a/src/sequifier/config/policy_config.py b/src/sequifier/config/policy_config.py index 1716a87f..49b9d0cc 100644 --- a/src/sequifier/config/policy_config.py +++ b/src/sequifier/config/policy_config.py @@ -3,14 +3,15 @@ import math from typing import Annotated, Any, Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import ConfigDict, Field, model_validator +from sequifier.config.naming import ConfigModel from sequifier.io.values import DomainValue NonemptyStr = Annotated[str, Field(min_length=1)] -class PolicyConfigBase(BaseModel): +class PolicyConfigBase(ConfigModel): model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 7973c21c..7b550f5c 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -25,9 +25,8 @@ class PolicyRolloutConfig(PolicyConfigBase): class PolicyInferenceConfig(InferenceExecutionConfig): - model_path: str - model_type: Literal["generative"] - task: Literal["policy"] = "policy" + model_paths: list[str] = Field(min_length=1, max_length=1) + task: Literal["policy"] data_path: str output_path: str | None = None @@ -50,7 +49,7 @@ class PolicyInferenceConfig(InferenceExecutionConfig): context_length: int | None = Field(default=None, gt=0) target_offset: int | None = Field(default=None, ge=0) prediction_length: int | None = Field(default=None, gt=0) - decoder_support: int | None = Field(default=None, gt=0) + decoder_support_length: int | None = Field(default=None, gt=0) @model_validator(mode="after") def validate_execution(self): @@ -76,11 +75,9 @@ def _policy_inference_route( route: InferenceRouteConfig, artifacts: dict[str, ModelArtifact] ) -> SelectedInferenceModel | None: """Inspect artifact interfaces before applying either inference schema.""" - if route.model_type == "embedding": + if route.task != "policy": return None - paths = ( - [route.model_path] if isinstance(route.model_path, str) else route.model_path - ) + paths = route.model_paths bindings = [] selection = None for path in paths: @@ -95,15 +92,6 @@ def _policy_inference_route( task=route.task, ) bindings.append(selection.policy_bound) - task = route.task - if task is None: - if any(bindings) and not all(bindings): - raise ValueError( - "Selected artifacts mix policy and forecast interfaces; select task explicitly" - ) - task = "policy" if all(bindings) else "forecast" - if task == "forecast": - return None if len(paths) != 1 or not all(bindings): raise ValueError( "task: policy requires one artifact with a selected policy-bound interface" @@ -163,7 +151,7 @@ def policy_inference_config( in artifacts[model_path].model_config.values["interfaces"][selected], ) authored = request.model_dump() - authored["model_path"] = model_path + authored["model_paths"] = [model_path] authored["model_interface"] = selected config = ResolvedPolicyInferenceConfig.model_validate(authored) config._artifacts = artifacts diff --git a/src/sequifier/config/policy_optimization.py b/src/sequifier/config/policy_optimization.py index c40726ea..fbe79499 100644 --- a/src/sequifier/config/policy_optimization.py +++ b/src/sequifier/config/policy_optimization.py @@ -3,9 +3,10 @@ import inspect from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import ConfigDict, Field, model_validator from sequifier.config.components import ComponentSpec +from sequifier.config.naming import ConfigModel from sequifier.config.optimizer_config import ( OptimizerPlan, resolve_plan_scheduler_arguments, @@ -21,7 +22,7 @@ def default_scheduler(): return ComponentSpec(name="ConstantLR", arguments={"factor": 1.0, "total_iters": 1}) -class RoleOptimizationConfig(BaseModel): +class RoleOptimizationConfig(ConfigModel): """One authored optimizer/scheduler configuration per trainable role.""" model_config = ConfigDict(extra="forbid", allow_inf_nan=False) @@ -30,7 +31,7 @@ class RoleOptimizationConfig(BaseModel): ) scheduler: ComponentSpec = Field(default_factory=default_scheduler) learning_rate: float | None = Field(default=None, gt=0) - gradient_clip: float | None = Field(default=1.0, gt=0) + gradient_clip_norm: float | None = Field(default=1.0, gt=0) @model_validator(mode="after") def validate_components(self): @@ -78,7 +79,7 @@ def continuation_settings(self): """Compare optimizer settings independently of optional budget assertions.""" values = self.model_dump(mode="python") if self.scheduler.name == "OneCycleLR": - values["scheduler"].pop("total_steps", None) + values["scheduler"]["arguments"].pop("total_steps", None) return values def validate_budget(self, step_budget): @@ -159,7 +160,7 @@ def validate_policy_scheduler(self): class SupervisedOptimizationConfig(RoleOptimizationConfig): """Epoch scheduling includes metric-driven ReduceLROnPlateau.""" - scheduler_step_on: Literal["epoch", "batch"] = "epoch" + scheduler_step_on: Literal["epoch", "optimizer_step"] = "epoch" scheduler: ComponentSpec = Field( default_factory=lambda: ComponentSpec( name="StepLR", arguments={"step_size": 1, "gamma": 0.99} diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 8475c2d9..6b9c8036 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -3,48 +3,49 @@ import math from typing import Annotated, ClassVar, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import ConfigDict, Field, field_validator, model_validator from sequifier.config.identifiers import identifier +from sequifier.config.naming import ConfigModel from sequifier.config.policy_optimization import OptimizationConfig from sequifier.config.sources import TrainingSourceSpecModel -class OfflineEvaluationMonitor(BaseModel): +class OfflineEvaluationMonitor(ConfigModel): model_config = ConfigDict(extra="forbid") metric: Literal["validation_action_nll"] = "validation_action_nll" - mode: Literal["min"] = "min" + direction: Literal["min"] = "min" -class DPOEvaluationMonitor(BaseModel): +class DPOEvaluationMonitor(ConfigModel): model_config = ConfigDict(extra="forbid") metric: Literal["validation_dpo_loss"] = "validation_dpo_loss" - mode: Literal["min"] = "min" + direction: Literal["min"] = "min" -class OfflineEvaluationConfig(BaseModel): +class OfflineEvaluationConfig(ConfigModel): model_config = ConfigDict(extra="forbid") sources: list[TrainingSourceSpecModel] | None = Field(default=None, min_length=1) monitor: OfflineEvaluationMonitor = Field(default_factory=OfflineEvaluationMonitor) -class DPOEvaluationConfig(BaseModel): +class DPOEvaluationConfig(ConfigModel): model_config = ConfigDict(extra="forbid") sources: list[TrainingSourceSpecModel] | None = Field(default=None, min_length=1) monitor: DPOEvaluationMonitor = Field(default_factory=DPOEvaluationMonitor) -class PhaseConfig(BaseModel): +class PhaseConfig(ConfigModel): model_config = ConfigDict(extra="forbid", allow_inf_nan=False) name: str optimizer_state: Literal["reset", "continue"] = "reset" handoff: Literal["best", "last"] = "last" model_interface: str | None = None - freeze: list[str] = Field(default_factory=list) + freeze_modules: list[str] = Field(default_factory=list) optimization_roles: ClassVar[tuple[str, ...]] = ("actor",) optimization: dict[str, OptimizationConfig] = Field(default_factory=dict) - evaluation_interval: int = Field(default=1, gt=0) - checkpoint_interval: int = Field(default=1, gt=0) + evaluation_interval_iterations: int = Field(default=1, gt=0) + checkpoint_interval_iterations: int = Field(default=1, gt=0) def optimizer_step_budget(self, training_size=None): raise NotImplementedError("built-in phase must declare its optimizer budget") @@ -72,7 +73,7 @@ def phase_name(cls, value): class BatchedPhaseConfig(PhaseConfig): batch_size: int = Field(default=32, gt=0) - accumulation_steps: int = Field(default=1, gt=0) + accumulation_batches: int = Field(default=1, gt=0) class OfflinePhaseConfig(PhaseConfig): @@ -87,7 +88,7 @@ class PolicyImitationPhaseConfig(BatchedPhaseConfig, OfflinePhaseConfig): def optimizer_step_budget(self, training_size=None): if training_size is None: return None - size = self.batch_size * self.accumulation_steps + size = self.batch_size * self.accumulation_batches return self.epochs * ((training_size + size - 1) // size) @@ -95,21 +96,21 @@ class IQLPhaseConfig(OfflinePhaseConfig): optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value", "critic") algorithm: Literal["iql"] name: str = Field(default="offline_iql", min_length=1) - updates: int = Field(gt=0) + iterations: int = Field(gt=0) batch_size: int = Field(default=64, gt=0) gamma: float = Field(default=0.99, gt=0, le=1, allow_inf_nan=False) expectile: float = Field(default=0.7, gt=0, lt=1, allow_inf_nan=False) inverse_temperature: float = Field(default=3.0, gt=0, allow_inf_nan=False) - max_weight: float = Field(default=100.0, ge=1, allow_inf_nan=False) + max_advantage_weight: float = Field(default=100.0, ge=1, allow_inf_nan=False) target_update_rate: float = Field(default=0.005, gt=0, le=1, allow_inf_nan=False) - evaluation_interval: int = Field(default=100, gt=0) - checkpoint_interval: int = Field(default=100, gt=0) + evaluation_interval_iterations: int = Field(default=100, gt=0) + checkpoint_interval_iterations: int = Field(default=100, gt=0) def optimizer_step_budget(self, training_size=None): - return self.updates + return self.iterations -class OnlineEnvironmentConfig(BaseModel): +class OnlineEnvironmentConfig(ConfigModel): model_config = ConfigDict(extra="forbid") factory: str config: dict = Field(default_factory=dict) @@ -131,7 +132,7 @@ def validate_factory(cls, value): return value -class FrozenNormalizationConfig(BaseModel): +class FrozenNormalizationConfig(ConfigModel): model_config = ConfigDict(extra="forbid") source: Literal["identity", "statistics"] statistics: dict[str, dict[str, float]] = Field(default_factory=dict) @@ -152,7 +153,7 @@ def validate_statistics(self): return self -class EvaluationEpisodesConfig(BaseModel): +class EvaluationEpisodesConfig(ConfigModel): model_config = ConfigDict(extra="forbid") environment: OnlineEnvironmentConfig seeds: list[int] = Field(min_length=1) @@ -168,13 +169,13 @@ def validate_seeds(cls, values): return values -class OnlineEvaluationMonitor(BaseModel): +class OnlineEvaluationMonitor(ConfigModel): model_config = ConfigDict(extra="forbid") metric: Literal[ "mean_completed_return", "success_rate", "mean_completed_length", "score_mean" ] = "mean_completed_return" - mode: Literal["min", "max"] = "max" + direction: Literal["min", "max"] = "max" class OnlineEvaluationConfig(EvaluationEpisodesConfig): @@ -189,18 +190,18 @@ class PPOPhaseConfig(PhaseConfig): evaluation: OnlineEvaluationConfig seed_policy: Literal["derived_per_reset"] = "derived_per_reset" total_env_steps: int = Field(gt=0) - rollout_length: int = Field(gt=0) + rollout_env_steps: int = Field(gt=0) allow_partial_rollout: bool = True optimization_epochs: int = Field(default=4, gt=0) - minibatch_size: int = Field(default=64, gt=0) - accumulation_steps: int = Field(default=1, gt=0) + batch_size: int = Field(default=64, gt=0) + accumulation_batches: int = Field(default=1, gt=0) gamma: float = Field(default=0.99, ge=0, le=1, allow_inf_nan=False) gae_lambda: float = Field(default=0.95, ge=0, le=1, allow_inf_nan=False) clip_epsilon: float = Field(default=0.2, gt=0, lt=1, allow_inf_nan=False) - value_clip: float | None = Field(default=None, gt=0, allow_inf_nan=False) + value_clip_delta: float | None = Field(default=None, gt=0, allow_inf_nan=False) value_coefficient: float = Field(default=0.5, gt=0, allow_inf_nan=False) entropy_coefficient: float = Field(default=0.01, ge=0, allow_inf_nan=False) - target_kl: float | None = Field(default=None, gt=0, allow_inf_nan=False) + old_policy_target_kl: float | None = Field(default=None, gt=0, allow_inf_nan=False) normalize_advantages: bool = True reward_scale: float = Field(default=1.0, gt=0, allow_inf_nan=False) archive_directory: str | None = Field(default=None, min_length=1) @@ -208,10 +209,10 @@ class PPOPhaseConfig(PhaseConfig): max_buffer_bytes: int = Field(default=536870912, gt=0) def optimizer_step_budget(self, training_size=None): - full, remainder = divmod(self.total_env_steps, self.rollout_length) - size = self.minibatch_size * self.accumulation_steps + full, remainder = divmod(self.total_env_steps, self.rollout_env_steps) + size = self.batch_size * self.accumulation_batches return self.optimization_epochs * ( - full * ((self.rollout_length + size - 1) // size) + full * ((self.rollout_env_steps + size - 1) // size) + ((remainder + size - 1) // size if remainder else 0) ) @@ -219,15 +220,15 @@ def optimizer_step_budget(self, training_size=None): def validate_plans(self): if ( not self.allow_partial_rollout - and self.total_env_steps % self.rollout_length + and self.total_env_steps % self.rollout_env_steps ): raise ValueError( - "rollout_length must divide total_env_steps unless allow_partial_rollout" + "rollout_env_steps must divide total_env_steps unless allow_partial_rollout" ) return self -class ReferenceConfig(BaseModel): +class ReferenceConfig(ConfigModel): model_config = ConfigDict(extra="forbid") source: Literal["phase_start", "artifact"] = "phase_start" path: str | None = None @@ -243,13 +244,13 @@ class DPOPhaseConfig(BatchedPhaseConfig): algorithm: Literal["dpo"] preferences: TrainingSourceSpecModel evaluation: DPOEvaluationConfig = Field(default_factory=DPOEvaluationConfig) - updates: int = Field(gt=0) + iterations: int = Field(gt=0) beta: float = Field(default=0.1, gt=0) scoring_chunk_size: int = Field(default=64, gt=0) reference: ReferenceConfig = Field(default_factory=ReferenceConfig) def optimizer_step_budget(self, training_size=None): - return self.updates + return self.iterations class GRPOEvaluationConfig(OnlineEvaluationConfig): @@ -265,7 +266,7 @@ class GRPOPhaseConfig(BatchedPhaseConfig): evaluation: GRPOEvaluationConfig reference: ReferenceConfig = Field(default_factory=ReferenceConfig) total_groups: int = Field(gt=0) - groups_per_update: int = Field(default=1, gt=0) + groups_per_iteration: int = Field(default=1, gt=0) group_size: int = Field(default=4, ge=2) max_steps: int = Field(gt=0) max_env_steps: int | None = Field(default=None, gt=0) @@ -273,16 +274,16 @@ class GRPOPhaseConfig(BatchedPhaseConfig): optimization_epochs: int = Field(default=4, gt=0) clip_epsilon: float = Field(default=0.2, gt=0, lt=1) advantage_epsilon: float = Field(default=1e-6, gt=0) - kl_coefficient: float = Field(default=0.04, ge=0) + reference_kl_coefficient: float = Field(default=0.04, ge=0) old_policy_target_kl: float | None = Field(default=None, gt=0) reward_scale: float = Field(default=1.0, gt=0) archive_directory: str | None = None def optimizer_step_budget(self, training_size=None): - full, remainder = divmod(self.total_groups, self.groups_per_update) - size = self.batch_size * self.accumulation_steps + full, remainder = divmod(self.total_groups, self.groups_per_iteration) + size = self.batch_size * self.accumulation_batches return self.optimization_epochs * ( - full * ((self.groups_per_update * self.group_size + size - 1) // size) + full * ((self.groups_per_iteration * self.group_size + size - 1) // size) + ((remainder * self.group_size + size - 1) // size if remainder else 0) ) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 9bea2e04..3a38c5c7 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Literal -from pydantic import Field, model_validator +from pydantic import Field, PrivateAttr, model_validator from sequifier.config.policy_config import NonemptyStr, PolicyConfigBase @@ -41,10 +41,10 @@ def validate_action_mappings(self): class SequenceSourceConfig(PolicyConfigBase): - project_root: str = "." + _project_root: str = PrivateAttr(default=".") units: dict[str, str] | None = None paths: list[NonemptyStr] = Field(default_factory=list) - manifest: NonemptyStr | None = None + manifest_path: NonemptyStr | None = None format: Literal["auto", "trajectory", "table", "prepared"] = "auto" columns: PolicyTableColumns = Field(default_factory=PolicyTableColumns) # A file boundary alone says nothing about task completion. @@ -56,12 +56,17 @@ class SequenceSourceConfig(PolicyConfigBase): split_index: int | None = Field(default=None, ge=0) action_semantics: Literal["policy", "executed_predictions"] = "policy" + @property + def project_root(self) -> str: + """Run-owned media root, excluded from the authored and serialized schema.""" + return self._project_root + @model_validator(mode="after") def validate_location(self): metadata_split = ( self.metadata_config_path is not None and self.split_index is not None ) - if sum((bool(self.paths), self.manifest is not None, metadata_split)) != 1: + if sum((bool(self.paths), self.manifest_path is not None, metadata_split)) != 1: raise ValueError("trajectory source requires either paths or one manifest") if ( self.split_index is not None @@ -78,9 +83,8 @@ def resolve_sequence_source( source: SequenceSourceConfig, root: Path ) -> SequenceSourceConfig: values = source.model_dump() - values["project_root"] = str(root) values["paths"] = [str((root / path).resolve()) for path in source.paths] - for key in ("manifest", "metadata_config_path", "static_data_path"): + for key in ("manifest_path", "metadata_config_path", "static_data_path"): if values[key] is not None: values[key] = str((root / values[key]).resolve()) if source.metadata_config_path is not None and ( @@ -116,4 +120,6 @@ def resolve_sequence_source( format="prepared", split_index=None, ) - return SequenceSourceConfig.model_validate(values) + resolved = SequenceSourceConfig.model_validate(values) + resolved._project_root = str(root) + return resolved diff --git a/src/sequifier/config/prediction_config.py b/src/sequifier/config/prediction_config.py index 849aa75a..7bf8bc77 100644 --- a/src/sequifier/config/prediction_config.py +++ b/src/sequifier/config/prediction_config.py @@ -2,10 +2,12 @@ from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import ConfigDict, Field, model_validator +from sequifier.config.naming import ConfigModel -class PredictionConfigBase(BaseModel): + +class PredictionConfigBase(ConfigModel): model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) diff --git a/src/sequifier/config/preprocess_config.py b/src/sequifier/config/preprocess_config.py index a8ba01c9..38ac3e52 100644 --- a/src/sequifier/config/preprocess_config.py +++ b/src/sequifier/config/preprocess_config.py @@ -4,14 +4,7 @@ from typing import Any, Optional, Union import numpy as np -from pydantic import ( - AliasChoices, - BaseModel, - ConfigDict, - Field, - field_validator, - model_validator, -) +from pydantic import ConfigDict, Field, field_validator, model_validator from sequifier.config.composition import ( load_composed_yaml_config, @@ -19,6 +12,7 @@ ) from sequifier.config.depth_layout import DepthLayoutRegistryModel from sequifier.config.image_config import ImageSourceConfig, PreconditioningImageConfig +from sequifier.config.naming import ConfigModel, RunSeed from sequifier.config.sequence_import_config import SequenceImportConfig from sequifier.helpers import ( canonicalize_polars_dtype_name, @@ -43,49 +37,50 @@ def load_preprocessor_config( return try_catch_excess_keys(config_path, PreprocessorModel, config_values) -class CardinalityHashingModel(BaseModel): +class CardinalityHashingModel(ConfigModel): """Hash non-retained categorical values into a bounded set of buckets.""" model_config = ConfigDict(extra="forbid", populate_by_name=True) num_buckets: int = Field( ge=1, - validation_alias=AliasChoices("num_buckets", "buckets"), ) seed: int = 0 -class CardinalityLimitModel(BaseModel): +class CardinalityLimitModel(ConfigModel): """Bound the vocabulary produced for one categorical column.""" model_config = ConfigDict(extra="forbid", populate_by_name=True) - min_freq: Optional[int] = Field( + min_count: Optional[int] = Field( default=None, ge=1, - validation_alias=AliasChoices("min_freq", "min_count"), ) - top_k: Optional[int] = Field( + max_categories: Optional[int] = Field( default=None, ge=1, - validation_alias=AliasChoices("top_k", "k", "max_categories"), ) hashing: Optional[CardinalityHashingModel] = None @model_validator(mode="after") def validate_options(self) -> "CardinalityLimitModel": - if self.min_freq is not None and self.top_k is not None: + if self.min_count is not None and self.max_categories is not None: raise ValueError( - "top_k and min_freq are mutually exclusive cardinality limits" + "max_categories and min_count are mutually exclusive cardinality limits" ) - if self.min_freq is None and self.top_k is None and self.hashing is None: + if ( + self.min_count is None + and self.max_categories is None + and self.hashing is None + ): raise ValueError( - "cardinality configuration requires top_k, min_freq, or hashing" + "cardinality configuration requires max_categories, min_count, or hashing" ) return self -class PreprocessorModel(BaseModel): +class PreprocessorModel(ConfigModel): """Top-level preprocessing config.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") @@ -117,8 +112,7 @@ class PreprocessorModel(BaseModel): split_method: str = Field(default="within_sequence") split_column: Optional[str] = None split_values: Optional[list[Union[int, str, date, datetime]]] = None - window_length: Optional[int] = Field(default=None, gt=0) - window_lengths: Optional[list[int]] = None + window_lengths: list[int] = Field(min_length=1) max_target_offset: int = Field(default=1, ge=0) window_stride: Optional[int] = Field(default=None, gt=0) prediction_aligned_splits: list[int] = Field(default_factory=list) @@ -126,8 +120,8 @@ class PreprocessorModel(BaseModel): prediction_alignment_steps: int = Field(default=1, gt=0) target_offset: Optional[int] = Field(default=None, ge=0) max_rows: Optional[int] = None - seed: int = 1010 - n_cores: Optional[int] = None + seed: RunSeed = 1010 + num_workers: Optional[int] = None batches_per_file: int = 1024 process_by_file: bool = True continue_preprocessing: bool = False @@ -531,20 +525,7 @@ def validate_mask_column_requires_metadata(self) -> "PreprocessorModel": if self.split_values is not None else 0 ) - if (self.window_length is None) == (self.window_lengths is None): - raise ValueError("Specify exactly one of window_length or window_lengths") - if self.prediction_alignment_steps > 1 and self.window_lengths is None: - raise ValueError( - "window_lengths must be set when prediction_alignment_steps > 1" - ) - if self.window_lengths is not None: - widths = self.window_lengths - else: - if self.window_length is None: - raise ValueError( - "window_length is required when window_lengths is absent" - ) - widths = [self.window_length] * n_splits + widths = self.window_lengths if len(widths) != n_splits or any( isinstance(width, bool) or not isinstance(width, int) @@ -600,8 +581,5 @@ def validate_mask_column_requires_metadata(self) -> "PreprocessorModel": @beartype def __init__(self, **kwargs): if kwargs.get("window_stride") is None: - kwargs["window_stride"] = ( - kwargs.get("window_length") - or (kwargs.get("window_lengths") or [None])[0] - ) + kwargs["window_stride"] = (kwargs.get("window_lengths") or [None])[0] super().__init__(**kwargs) diff --git a/src/sequifier/config/probabilities.py b/src/sequifier/config/probabilities.py index 15d99e07..dafa37aa 100644 --- a/src/sequifier/config/probabilities.py +++ b/src/sequifier/config/probabilities.py @@ -5,6 +5,7 @@ import torch from pydantic import BaseModel, ConfigDict, Field +from sequifier.config.naming import ConfigModel from sequifier.typechecking import beartype @@ -24,7 +25,7 @@ def sample( pass -class GeometricDistribution(BaseModel, ProbabilityDistributionBaseClass): +class GeometricDistribution(ConfigModel, ProbabilityDistributionBaseClass): model_config = ConfigDict(extra="forbid") type: Literal["geometric"] = "geometric" @@ -46,12 +47,12 @@ def sample( ) -class NormalDistributionDiscretizedFloor(BaseModel, ProbabilityDistributionBaseClass): +class NormalDistributionDiscretizedFloor(ConfigModel, ProbabilityDistributionBaseClass): model_config = ConfigDict(extra="forbid") - type: Literal["normal_discretized_floor"] = "normal_discretized_floor" + type: Literal["clipped_rounded_normal_plus_one"] = "clipped_rounded_normal_plus_one" mean: float - standard_deviation: float = Field(..., gt=0.0) + std: float = Field(..., gt=0.0) @beartype def sample( @@ -61,8 +62,7 @@ def sample( generator: Optional[torch.Generator] = None, ) -> torch.Tensor: val = ( - torch.randn(shape, device=device, generator=generator) - * self.standard_deviation + torch.randn(shape, device=device, generator=generator) * self.std + self.mean ) return torch.clamp(torch.round(val), min=0).long() + 1 @@ -73,9 +73,9 @@ class LogNormalDistributionDiscretizedFloor( ): model_config = ConfigDict(extra="forbid") - type: Literal["log_normal_discretized_floor",] = "log_normal_discretized_floor" + type: Literal["rounded_log_normal_plus_one",] = "rounded_log_normal_plus_one" mean: float - standard_deviation: float = Field(..., gt=0.0) + std: float = Field(..., gt=0.0) @beartype def sample( @@ -85,18 +85,17 @@ def sample( generator: Optional[torch.Generator] = None, ) -> torch.Tensor: normal = ( - torch.randn(shape, device=device, generator=generator) - * self.standard_deviation + torch.randn(shape, device=device, generator=generator) * self.std + self.mean ) val = torch.exp(normal) return torch.round(val).long() + 1 -class PoissonDistributionFloor(BaseModel, ProbabilityDistributionBaseClass): +class PoissonDistributionFloor(ConfigModel, ProbabilityDistributionBaseClass): model_config = ConfigDict(extra="forbid") - type: Literal["poisson_floor"] = "poisson_floor" + type: Literal["poisson_plus_one"] = "poisson_plus_one" rate: float = Field(..., gt=0.0) @beartype diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index 9d86e4d2..7e7bdad0 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -2,10 +2,12 @@ from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import ConfigDict, Field, field_validator, model_validator +from sequifier.config.naming import ConfigModel -class InputBinding(BaseModel): + +class InputBinding(ConfigModel): model_config = ConfigDict(extra="forbid", frozen=True) input: str = Field(min_length=1) @@ -13,7 +15,7 @@ class InputBinding(BaseModel): key: str = Field(min_length=1) -class AdapterConfig(BaseModel): +class AdapterConfig(ConfigModel): model_config = ConfigDict(extra="forbid") factory: str @@ -34,7 +36,7 @@ def validate_factory(cls, value: str) -> str: return value -class RolloutConfig(BaseModel): +class RolloutConfig(ConfigModel): model_config = ConfigDict(extra="forbid") max_steps: int = Field(gt=0) diff --git a/src/sequifier/config/sources.py b/src/sequifier/config/sources.py index df168c85..8a41ffa9 100644 --- a/src/sequifier/config/sources.py +++ b/src/sequifier/config/sources.py @@ -2,17 +2,18 @@ from typing import Optional -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import ConfigDict, Field, field_validator from sequifier.config.identifiers import identifier as _identifier +from sequifier.config.naming import ConfigModel from sequifier.typechecking import beartype -class TrainingSourceSpecModel(BaseModel): +class TrainingSourceSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") source: str - weight: Optional[float] = Field(default=None, gt=0) + selection_weight: Optional[float] = Field(default=None, gt=0) batches_per_selection: Optional[int] = Field(default=None, gt=0) @field_validator("source") diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index e2edca08..83580058 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -1,9 +1,7 @@ -"""Canonical Sequifier training configuration and resolution. +"""Canonical named training configuration and resolution. -The concise singleton authoring surface is normalized to the canonical named -schema before validation. This module intentionally contains no migration -adapter for the historical flat single-dataset YAML schema. Runtime consumers -select a dataset, part, or model interface explicitly. +All cardinalities use named interfaces, datasets, parts and ordered phases. +Removed spellings are diagnosed rather than accepted as aliases. """ from __future__ import annotations @@ -20,7 +18,6 @@ import torch from pydantic import ( - BaseModel, ConfigDict, Field, PrivateAttr, @@ -55,6 +52,7 @@ image_views, ) from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata +from sequifier.config.naming import ConfigModel, RunSeed from sequifier.config.policy_config import PolicyBinding, ResolvedPolicyBinding from sequifier.config.policy_optimization import SupervisedOptimizationConfig from sequifier.config.policy_plan import ( @@ -121,292 +119,70 @@ def _unique_columns(value: list[str], usage: str) -> list[str]: return value -_SINGLETON_CONFIG_NAME = "default" -_SINGLE_PHASE_NAME = "train" -_PHASE_FIELD_NAMES = frozenset( - { - "algorithm", - "name", - "epochs", - "mode", - "selection", - "sources", - "updates", - "total_env_steps", - "total_groups", - "evaluation", - "evaluation_interval", - "checkpoint_interval", - "batch_size", - "accumulation_steps", - "optimization", - "handoff", - "optimizer_state", - } -) -_TRAINING_PLAN_POLICY_FIELDS = frozenset({"curriculum_training", "curriculum_column"}) - - -@beartype -def _normalize_dataset_part_surface(datasets: Any) -> None: - if not isinstance(datasets, dict): - return - for dataset_name, dataset in datasets.items(): - if not isinstance(dataset, dict): - continue - if "part" in dataset and "parts" in dataset: - raise ValueError( - f"Dataset {dataset_name!r} cannot define both 'part' and 'parts'." - ) - if "part" in dataset: - dataset["parts"] = {_SINGLETON_CONFIG_NAME: dataset.pop("part")} - - -@beartype -def _implicit_source(values: dict[str, Any]) -> str | None: - datasets = values.get("dataset_training") - if not isinstance(datasets, dict) or len(datasets) != 1: - return None - dataset_name, dataset = next(iter(datasets.items())) - if not isinstance(dataset, dict): - return None - parts = dataset.get("parts") - if not isinstance(parts, dict) or not parts: - return None - if len(parts) == 1: - return f"{dataset_name}.{next(iter(parts))}" - return dataset_name - - @beartype def normalize_train_config_surface(values: Any) -> Any: - """Expand the concise singleton authoring surface to canonical mappings.""" - + """Apply defaults without accepting alternate authoring structures.""" if not isinstance(values, dict): return values normalized = copy.deepcopy(values) - model = normalized.get("model") + training = normalized.get("global_training", {}) if isinstance(model, dict): backbone = model.get("backbone") - global_training = normalized.get("global_training") - if isinstance(backbone, dict) and isinstance(global_training, dict): - architecture = backbone.get("architecture") - if isinstance(architecture, dict) and "context_length" in global_training: + architecture = ( + backbone.get("architecture") if isinstance(backbone, dict) else None + ) + if isinstance(architecture, dict) and isinstance(training, dict): + if training.get("context_length") is not None: architecture.setdefault( - "max_context_length", global_training["context_length"] + "max_context_length", training["context_length"] ) - if "interface" in model and "interfaces" in model: - raise ValueError("model cannot define both 'interface' and 'interfaces'.") - if "interface" in model: - model["interfaces"] = {_SINGLETON_CONFIG_NAME: model.pop("interface")} - - if "dataset" in normalized and "dataset_training" in normalized: - raise ValueError( - "Training config cannot define both 'dataset' and 'dataset_training'." - ) - if "dataset" in normalized: - normalized["dataset_training"] = { - _SINGLETON_CONFIG_NAME: normalized.pop("dataset") - } - - datasets = normalized.get("dataset_training") - _normalize_dataset_part_surface(datasets) + datasets = normalized.get("dataset_training", {}) + interfaces = model.get("interfaces", {}) if isinstance(model, dict) else {} if isinstance(datasets, dict): for dataset in datasets.values(): - if isinstance(dataset, dict): - for part in dataset.get("parts", {}).values(): + if not isinstance(dataset, dict): + continue + if isinstance(interfaces, dict) and len(interfaces) == 1: + dataset.setdefault("model_interface", next(iter(interfaces))) + parts = dataset.get("parts", {}) + if isinstance(parts, dict): + for part in parts.values(): if isinstance(part, dict): part.setdefault("type", "prepared") - - interfaces = model.get("interfaces") if isinstance(model, dict) else None - if ( - isinstance(interfaces, dict) - and len(interfaces) == 1 - and isinstance(datasets, dict) - ): - interface_name = next(iter(interfaces)) - for dataset in datasets.values(): - if isinstance(dataset, dict): - dataset.setdefault("model_interface", interface_name) - - if "policy_binding" in normalized and "policy_bindings" in normalized: - raise ValueError("Use policy_binding or policy_bindings, not both") - if ( - "policy_binding" in normalized - and isinstance(interfaces, dict) - and len(interfaces) == 1 - ): - normalized["policy_bindings"] = { - next(iter(interfaces)): normalized.pop("policy_binding") - } - - implicit_source = _implicit_source(normalized) - training_plan = normalized.get("training_plan") - if isinstance(training_plan, dict): - direct_fields = _PHASE_FIELD_NAMES & set(training_plan) - if "phases" in training_plan and direct_fields: - raise ValueError( - "training_plan cannot combine 'phases' with direct phase fields." - ) - if "phases" not in training_plan and direct_fields: - policy = { - key: training_plan[key] - for key in _TRAINING_PLAN_POLICY_FIELDS & set(training_plan) - } - phase = { - key: value - for key, value in training_plan.items() - if key not in _TRAINING_PLAN_POLICY_FIELDS - } - phase.setdefault("name", _SINGLE_PHASE_NAME) - if phase.get("algorithm", "supervised") == "supervised": - phase.setdefault("mode", "sequential") - normalized["training_plan"] = {**policy, "phases": [phase]} - training_plan = normalized["training_plan"] - - phases = training_plan.get("phases") - if isinstance(phases, list): - for phase in phases: - if isinstance(phase, dict): - phase.setdefault("algorithm", "supervised") - if implicit_source is not None and phase["algorithm"] in { - "supervised", - "policy_imitation", - "iql", - }: - phase.setdefault("sources", [{"source": implicit_source}]) - - if isinstance(training_plan, dict): - for phase in training_plan.get("phases", []): - if not isinstance(phase, dict) or phase.get("algorithm") != "supervised": - continue - evaluation = phase.get("evaluation") - if isinstance(evaluation, bool): - if not evaluation: - phase["evaluation"] = None - else: - sources = phase.get("sources", []) - if len(sources) != 1: - raise ValueError( - "evaluation=true requires exactly one phase source" - ) - phase["evaluation"] = { - "sources": [{"source": sources[0]["source"]}] - } - - return normalized - - -@beartype -def _only_base_name(values: Any, usage: str) -> str: - if not isinstance(values, dict) or len(values) != 1: - raise ValueError( - f"The singleton {usage} override requires exactly one {usage} " - "in the base training config." - ) - return next(iter(values)) - - -@beartype -def normalize_train_config_parameter_surface( - parameters: Any, - base_values: dict[str, Any], -) -> Any: - """Translate singleton search parameters to canonical base paths.""" - - if not isinstance(parameters, dict): - return parameters - normalized = copy.deepcopy(parameters) - - model_override = normalized.get("model") - if isinstance(model_override, dict) and "interface" in model_override: - if "interfaces" in model_override: - raise ValueError( - "model parameters cannot define both 'interface' and 'interfaces'." - ) - interface_name = _only_base_name( - base_values.get("model", {}).get("interfaces"), - "model interface", - ) - model_override["interfaces"] = {interface_name: model_override.pop("interface")} - - if "dataset" in normalized: - if "dataset_training" in normalized: - raise ValueError( - "Training parameters cannot define both 'dataset' and " - "'dataset_training'." - ) - dataset_name = _only_base_name( - base_values.get("dataset_training"), - "dataset", - ) - normalized["dataset_training"] = {dataset_name: normalized.pop("dataset")} - - dataset_overrides = normalized.get("dataset_training") - base_datasets = base_values.get("dataset_training") - if isinstance(dataset_overrides, dict): - for dataset_name, dataset_override in dataset_overrides.items(): - if not isinstance(dataset_override, dict): - continue - if "part" in dataset_override and "parts" in dataset_override: - raise ValueError( - f"Dataset override {dataset_name!r} cannot define both " - "'part' and 'parts'." - ) - if "part" not in dataset_override: - continue - base_dataset = ( - base_datasets.get(dataset_name) - if isinstance(base_datasets, dict) - else None - ) - part_name = _only_base_name( - base_dataset.get("parts") if isinstance(base_dataset, dict) else None, - f"part for dataset {dataset_name!r}", - ) - dataset_override["parts"] = {part_name: dataset_override.pop("part")} - - plan_override = normalized.get("training_plan") - if isinstance(plan_override, dict): - direct_fields = _PHASE_FIELD_NAMES & set(plan_override) - if "phases" in plan_override and direct_fields: - raise ValueError( - "training_plan parameters cannot combine 'phases' with direct " - "phase fields." - ) - if "phases" not in plan_override and direct_fields: - base_phases = base_values.get("training_plan", {}).get("phases") - if not isinstance(base_phases, list) or len(base_phases) != 1: - raise ValueError( - "Direct training_plan parameters require exactly one phase " - "in the base training config." + implicit_source = None + if isinstance(datasets, dict) and len(datasets) == 1: + dataset_name, dataset = next(iter(datasets.items())) + if isinstance(dataset, dict) and isinstance(dataset.get("parts"), dict): + parts = dataset["parts"] + if parts: + implicit_source = ( + f"{dataset_name}.{next(iter(parts))}" + if len(parts) == 1 + else dataset_name ) - policy = { - key: plan_override[key] - for key in _TRAINING_PLAN_POLICY_FIELDS & set(plan_override) - } - phase_override = { - key: value - for key, value in plan_override.items() - if key not in _TRAINING_PLAN_POLICY_FIELDS - } - normalized["training_plan"] = { - **policy, - "phases": {0: phase_override}, - } - + plan = normalized.get("training_plan") + if isinstance(plan, dict) and isinstance(plan.get("phases"), list): + for phase in plan["phases"]: + if isinstance(phase, dict): + phase.setdefault("algorithm", "supervised") + if implicit_source and phase["algorithm"] in { + "supervised", + "policy_imitation", + "iql", + }: + phase.setdefault("sources", [{"source": implicit_source}]) return normalized -class GlobalTrainingSpecModel(BaseModel): +class GlobalTrainingSpecModel(ConfigModel): """Run-wide data, precision, execution and distribution settings.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") validation_fraction: float = Field(default=0.2, gt=0, lt=1) episode_cache_size: int = Field(default=8, gt=0) - metadata_config_path: str | dict[str, str] | None = None + interface_metadata_paths: dict[str, str] = Field(default_factory=dict) normalization: FrozenNormalizationConfig | None = None read_format: Literal["csv", "parquet", "pt"] = "parquet" training_objective: str | None = None @@ -418,14 +194,14 @@ class GlobalTrainingSpecModel(BaseModel): bert_spec: Optional[BERTSpecModel] = None next_occurrence_config: Optional[NextOccurrenceConfigModel] = None - device_max_concat_length: int = Field(default=12, gt=0) - log_interval: int = Field(default=10, gt=0) - early_stopping_epochs: Optional[int] = Field(default=None, gt=0) - save_latest_interval_minutes: Optional[float] = None - save_interval_minutes: Optional[float] = None - save_interval_batches: Optional[int] = None - save_interval_val_loss: bool = True - calculate_validation_loss_on_initialization: bool = True + max_concat_tensors: int = Field(default=12, gt=0) + log_interval_batches: int = Field(default=10, gt=0) + early_stopping_patience_epochs: Optional[int] = Field(default=None, gt=0) + latest_checkpoint_interval_minutes: Optional[float] = None + snapshot_interval_minutes: Optional[float] = None + snapshot_interval_batches: Optional[int] = None + evaluate_on_snapshot: bool = True + evaluate_before_training: bool = True resume: Optional[ResumeConfig] = None enforce_determinism: bool = False distributed: bool = False @@ -489,9 +265,9 @@ def validate_distribution(self): if self.data_parallelism == "ddp" and self.torch_compile == "inner": raise ValueError("DDP requires torch_compile 'none' or 'outer'") for name in ( - "save_latest_interval_minutes", - "save_interval_minutes", - "save_interval_batches", + "latest_checkpoint_interval_minutes", + "snapshot_interval_minutes", + "snapshot_interval_batches", ): value = getattr(self, name) if ( @@ -503,10 +279,10 @@ def validate_distribution(self): return self -class StaticConditioningConfig(BaseModel): +class StaticConditioningConfig(ConfigModel): model_config = ConfigDict(extra="forbid") columns: list[str] = Field(min_length=1) - embedding_size: int | None = Field(default=None, gt=0) + total_embedding_dim: int | None = Field(default=None, gt=0) feature_embedding_dims: dict[str, int] | None = None @field_validator("columns") @@ -527,16 +303,16 @@ def validate_embedding_dims(self): if any(dim <= 0 for dim in dims.values()): raise ValueError("Static feature embedding dimensions must be positive") if ( - self.embedding_size is not None - and sum(dims.values()) != self.embedding_size + self.total_embedding_dim is not None + and sum(dims.values()) != self.total_embedding_dim ): raise ValueError( - "Static feature embedding dimensions must sum to embedding_size" + "Static feature embedding dimensions must sum to total_embedding_dim" ) return self -class ModelInterfaceSpecModel(BaseModel): +class ModelInterfaceSpecModel(ConfigModel): """Architecture and selected-column contract for one named model route.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") @@ -661,7 +437,7 @@ def image_cell_names(ingestion): return self -class ModelSpecModel(BaseModel): +class ModelSpecModel(ConfigModel): """One shared backbone and one or more named interfaces.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") @@ -678,13 +454,13 @@ def validate_interface_names(cls, value): return value -class DatasetPartSpecModel(BaseModel): +class DatasetPartSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") type: Literal["prepared"] = "prepared" metadata_config_path: str - image_path: MediaPath = None - preconditioning_image_path: MediaPath = None + image_paths: MediaPath = None + preconditioning_image_paths: MediaPath = None file_order: Literal["shuffled", "name"] = "shuffled" @@ -692,10 +468,10 @@ class SequencePartSpecModel(SequenceSourceConfig): type: Literal["sequence"] -class PreferencePartSpecModel(BaseModel): +class PreferencePartSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") type: Literal["preferences"] - manifest: str = Field(min_length=1) + manifest_path: str = Field(min_length=1) DatasetPart = Annotated[ @@ -704,7 +480,7 @@ class PreferencePartSpecModel(BaseModel): ] -class DatasetFreezingSpecModel(BaseModel): +class DatasetFreezingSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") backbone: LayerFreezingConfigFields = Field( @@ -725,7 +501,7 @@ def active(self) -> bool: ) -class DatasetTrainingSpecModel(BaseModel): +class DatasetTrainingSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") model_interface: str | None = None @@ -782,15 +558,15 @@ def validate_loss_weights(cls, value): return value -class EvaluationMonitorSpecModel(BaseModel): +class EvaluationMonitorSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") source: str metric: str = "loss" - mode: Literal["min", "max"] = "min" + direction: Literal["min", "max"] = "min" -class EvaluationSpecModel(BaseModel): +class EvaluationSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) @@ -800,11 +576,12 @@ class EvaluationSpecModel(BaseModel): @beartype def validate_monitor(self): if any( - source.weight is not None or source.batches_per_selection is not None + source.selection_weight is not None + or source.batches_per_selection is not None for source in self.sources ): raise ValueError( - "Evaluation sources cannot configure weight or batches_per_selection" + "Evaluation sources cannot configure selection_weight or batches_per_selection" ) sources = [source.source for source in self.sources] if len(sources) != len(set(sources)): @@ -816,12 +593,12 @@ def validate_monitor(self): return self -class TrainingPhaseSpecModel(BaseModel): +class TrainingPhaseSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") evaluation: EvaluationSpecModel | None = None - evaluation_interval: int = Field(default=1, gt=0) - checkpoint_interval: int = Field(default=1, gt=0) + evaluation_interval_epochs: int = Field(default=1, gt=0) + checkpoint_interval_epochs: int = Field(default=1, gt=0) algorithm: Literal["supervised"] = "supervised" handoff: Literal["last", "best"] = "last" @@ -829,7 +606,7 @@ class TrainingPhaseSpecModel(BaseModel): optimization_roles: ClassVar[tuple[str, ...]] = ("model",) optimizer_state: Literal["reset", "continue"] = "reset" batch_size: int = Field(default=32, gt=0) - accumulation_steps: int = Field(default=1, gt=0) + accumulation_batches: int = Field(default=1, gt=0) optimization: dict[Literal["model"], SupervisedOptimizationConfig] = Field( default_factory=lambda: {"model": SupervisedOptimizationConfig()} ) @@ -859,14 +636,16 @@ def validate_mode_fields(self): raise ValueError( "batches_per_selection is invalid in sequential phases" ) - if any(source.weight is not None for source in self.sources): - raise ValueError("weight is invalid in sequential phases") + if any(source.selection_weight is not None for source in self.sources): + raise ValueError("selection_weight is invalid in sequential phases") else: self.selection = self.selection or "round_robin" if self.selection == "round_robin" and any( - source.weight is not None for source in self.sources + source.selection_weight is not None for source in self.sources ): - raise ValueError("weight is only valid for weighted_random selection") + raise ValueError( + "selection_weight is only valid for weighted_random selection" + ) return self @@ -881,7 +660,7 @@ def validate_mode_fields(self): ] -class TrainingPlanModel(BaseModel): +class TrainingPlanModel(ConfigModel): model_config = ConfigDict(extra="forbid") curriculum_training: bool = False @@ -924,7 +703,9 @@ def validate_unique_names(self): raise ValueError( "supervised optimizer continuation requires identical dataset ownership" ) - if getattr(previous, "freeze", None) != getattr(phase, "freeze", None): + if getattr(previous, "freeze_modules", None) != getattr( + phase, "freeze_modules", None + ): raise ValueError( "optimizer continuation requires identical parameter selection" ) @@ -948,22 +729,21 @@ def validate_unique_names(self): return self -class SequifierConfig(BaseModel): - """Training configuration with singleton authoring normalization.""" +class SequifierConfig(ConfigModel): + """Training configuration with canonical names and inferred defaults.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") project_root: str model_name: str device: str - seed: int = 1010 + seed: RunSeed = 1010 global_training: GlobalTrainingSpecModel = Field( default_factory=GlobalTrainingSpecModel ) model: ModelSpecModel | None = None initialize_from: str | None = None policy_bindings: dict[str, PolicyBinding] = Field(default_factory=dict) - policy_binding: PolicyBinding | None = Field(default=None, exclude=True) model_path: str = "" report_path: str = "" phase_directory: str = "" @@ -982,7 +762,7 @@ class SequifierConfig(BaseModel): @model_validator(mode="before") @classmethod @beartype - def normalize_singleton_surface(cls, values): + def apply_authored_defaults(cls, values): return normalize_train_config_surface(values) @field_validator("dataset_training") @@ -1017,7 +797,8 @@ def validate_relationships(self): for ref in refs: self._validate_source(ref.source, f"phase {phase.name!r}") if phase.algorithm != "supervised" and ( - ref.weight is not None or ref.batches_per_selection is not None + ref.selection_weight is not None + or ref.batches_per_selection is not None ): raise ValueError( "policy source references cannot configure supervised interleaving" @@ -1028,10 +809,9 @@ def validate_relationships(self): needs_monitor = ( phase.handoff == "best" or role.scheduler.name == "ReduceLROnPlateau" - or self.global_training.early_stopping_epochs is not None + or self.global_training.early_stopping_patience_epochs is not None or ( - evaluation is not None - and self.global_training.save_interval_val_loss + evaluation is not None and self.global_training.evaluate_on_snapshot ) ) if needs_monitor and (evaluation is None or evaluation.monitor is None): @@ -1040,15 +820,17 @@ def validate_relationships(self): ) if ( role.scheduler.name == "ReduceLROnPlateau" - and phase.evaluation_interval != 1 + and phase.evaluation_interval_epochs != 1 ): - raise ValueError("ReduceLROnPlateau requires evaluation_interval: 1") + raise ValueError( + "ReduceLROnPlateau requires evaluation_interval_epochs: 1" + ) if ( - self.global_training.early_stopping_epochs is not None - and phase.evaluation_interval != 1 + self.global_training.early_stopping_patience_epochs is not None + and phase.evaluation_interval_epochs != 1 ): raise ValueError( - "early_stopping_epochs requires evaluation_interval: 1" + "early_stopping_patience_epochs requires evaluation_interval_epochs: 1" ) if any(phase.algorithm != "supervised" for phase in self.training_plan.phases): if self.global_training.distributed: @@ -1152,11 +934,11 @@ def validate_relationships(self): "context_length plus static prefix exceeds backbone max_context_length" ) for name, interface in model.interfaces.items(): - if interface.decoder.support > context_length: + if interface.decoder.support_length > context_length: raise ValueError( f"Interface {name!r} decoder support exceeds context_length" ) - decoded_length = context_length - interface.decoder.support + 1 + decoded_length = context_length - interface.decoder.support_length + 1 if interface.decoder.prediction_length > decoded_length: raise ValueError( f"Interface {name!r} prediction_length exceeds decoded length" @@ -1184,21 +966,21 @@ def _validate_source(self, source: str, usage: str) -> None: raise ValueError(f"Unknown {usage} part source {source!r}") -class ResolvedDatasetPart(BaseModel): +class ResolvedDatasetPart(ConfigModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") name: str metadata_config_path: str metadata: DatasetMetadata - image_path: MediaPath = None - preconditioning_image_path: MediaPath = None + image_paths: MediaPath = None + preconditioning_image_paths: MediaPath = None training_data_path: str validation_data_path: Optional[str] = None storage_form: Literal["file", "folder"] file_order: Literal["shuffled", "name"] = "shuffled" -class ResolvedModelInterface(BaseModel): +class ResolvedModelInterface(ConfigModel): policy: ResolvedPolicyBinding | None = None prediction_distributions: dict[str, PredictionDistributionConfig] = Field( default_factory=dict @@ -1263,7 +1045,7 @@ def validate_prediction_targets(self): return self -class ResolvedDatasetTrainingSpec(BaseModel): +class ResolvedDatasetTrainingSpec(ConfigModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") name: str @@ -1277,30 +1059,30 @@ class ResolvedDatasetTrainingSpec(BaseModel): freeze: DatasetFreezingSpecModel -class ResolvedTrainingSource(BaseModel): +class ResolvedTrainingSource(ConfigModel): model_config = ConfigDict(extra="forbid") source: str dataset: str part: Optional[str] = None - weight: float = 1.0 + selection_weight: float = 1.0 batches_per_selection: int = 1 -class ResolvedTrainingPhase(BaseModel): +class ResolvedTrainingPhase(ConfigModel): model_config = ConfigDict(extra="forbid") evaluation_sources: list[ResolvedTrainingSource] = Field(default_factory=list) evaluation_monitor: EvaluationMonitorSpecModel | None = None - evaluation_interval: int = Field(default=1, gt=0) - checkpoint_interval: int = Field(default=1, gt=0) + evaluation_interval_epochs: int = Field(default=1, gt=0) + checkpoint_interval_epochs: int = Field(default=1, gt=0) algorithm: Literal["supervised"] = "supervised" handoff: Literal["last", "best"] = "last" optimizer_state: Literal["reset", "continue"] = "reset" batch_size: int = Field(default=32, gt=0) - accumulation_steps: int = Field(default=1, gt=0) + accumulation_batches: int = Field(default=1, gt=0) optimization: dict[str, SupervisedOptimizationConfig] name: str @@ -1324,12 +1106,12 @@ def resolve_supervised_phases(authored_phases): evaluation_monitor=phase.evaluation.monitor if phase.evaluation is not None else None, - evaluation_interval=phase.evaluation_interval, - checkpoint_interval=phase.checkpoint_interval, + evaluation_interval_epochs=phase.evaluation_interval_epochs, + checkpoint_interval_epochs=phase.checkpoint_interval_epochs, handoff=phase.handoff, optimizer_state=phase.optimizer_state, batch_size=phase.batch_size, - accumulation_steps=phase.accumulation_steps, + accumulation_batches=phase.accumulation_batches, optimization={ role: value.model_copy(deep=True) for role, value in phase.optimization_configs().items() @@ -1339,8 +1121,8 @@ def resolve_supervised_phases(authored_phases): sources=[ _source( source.source, - phase.accumulation_steps, - weight=source.weight, + phase.accumulation_batches, + selection_weight=source.selection_weight, batches_per_selection=( None if phase.mode == "sequential" @@ -1374,7 +1156,7 @@ def resolve_supervised_phases(authored_phases): return phases -class ResolvedSequifierConfig(BaseModel): +class ResolvedSequifierConfig(ConfigModel): """Runtime configuration after all dataset parts and interfaces resolve.""" _prepared_inputs: Any = PrivateAttr(default=None) @@ -1526,16 +1308,16 @@ class LoadedTrainConfig: @beartype def _source( - source: str, accumulation_steps: Optional[int], **values: Any + source: str, accumulation_batches: Optional[int], **values: Any ) -> ResolvedTrainingSource: dataset, _, part = source.partition(".") return ResolvedTrainingSource( source=source, dataset=dataset, part=part or None, - weight=values.get("weight") or 1.0, + selection_weight=values.get("selection_weight") or 1.0, batches_per_selection=( - values.get("batches_per_selection") or accumulation_steps or 1 + values.get("batches_per_selection") or accumulation_batches or 1 ), ) @@ -2063,8 +1845,8 @@ def resolve_sequifier_config( or part_spec.metadata_config_path ), metadata=part_metadata, - image_path=part_spec.image_path, - preconditioning_image_path=part_spec.preconditioning_image_path, + image_paths=part_spec.image_paths, + preconditioning_image_paths=part_spec.preconditioning_image_paths, training_data_path=training_path, validation_data_path=validation_path, storage_form=form, @@ -2400,7 +2182,6 @@ def load_train_config_with_source( if ( config.initialize_from is not None or config.policy_bindings - or config.policy_binding is not None or any(phase.algorithm != "supervised" for phase in config.training_plan.phases) ): if sensitive or metadata_inline_values or skip_metadata: @@ -2598,8 +2379,8 @@ def dataset_part_view( bind_image_paths( selected_sources, selected_prefix, - part.image_path, - part.preconditioning_image_path, + part.image_paths, + part.preconditioning_image_paths, ) ) diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 548182b4..ba2df6aa 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -46,7 +46,7 @@ def resolve(self, *, prepared=True): sequences, preferences = {}, {} for ref, (_, part) in self.parts.items(): if isinstance(part, PreferencePartSpecModel): - path = self.root / part.manifest + path = self.root / part.manifest_path preferences[ref] = str(path.resolve() if prepared else path) continue if isinstance(part, SequencePartSpecModel): @@ -273,7 +273,6 @@ def _supervised_subset(config): }, "training_plan": config.training_plan.model_copy(update={"phases": phases}), "policy_bindings": {}, - "policy_binding": None, } ) @@ -319,13 +318,6 @@ def resolve_training_run(authored: SequifierConfig): ) setattr(config.global_training, field, actual) assert config.model is not None - if config.policy_binding is not None: - if len(config.model.interfaces) != 1: - raise ValueError("policy_binding requires exactly one model interface") - config.policy_bindings = { - next(iter(config.model.interfaces)): config.policy_binding - } - config.policy_binding = None for dataset in config.dataset_training.values(): if dataset.model_interface is None: if len(config.model.interfaces) != 1: @@ -352,13 +344,7 @@ def resolve_training_run(authored: SequifierConfig): # All declared metadata for an interface is an assertion of the same facts. metadata = {} - declarations = training.metadata_config_path - if isinstance(declarations, str): - if len(config.model.interfaces) != 1: - raise ValueError( - "global_training.metadata_config_path must map interface names when ambiguous" - ) - declarations = {next(iter(config.model.interfaces)): declarations} + declarations = training.interface_metadata_paths for name, path in (declarations or {}).items(): if name not in config.model.interfaces: raise ValueError(f"unknown metadata model_interface {name!r}") diff --git a/src/sequifier/docs.py b/src/sequifier/docs.py index 0c5f045f..c2fe5705 100644 --- a/src/sequifier/docs.py +++ b/src/sequifier/docs.py @@ -10,6 +10,7 @@ "documentation/configs/infer.md", "documentation/commands/visualize-training.md", "documentation/configs/hyperparameter-search.md", + "documentation/configs/naming.md", "documentation/training/multi-gpu-training.md", "documentation/policy-learning.md", "documentation/supervised-policy-learning.md", diff --git a/src/sequifier/helpers.py b/src/sequifier/helpers.py index 0cd3f19e..e1b0b5ab 100644 --- a/src/sequifier/helpers.py +++ b/src/sequifier/helpers.py @@ -70,11 +70,7 @@ "UInt8": pl.UInt8, } -POLARS_NUMERIC_DTYPE_ALIASES = { - alias: canonical - for canonical in POLARS_NUMERIC_DTYPES - for alias in (canonical, canonical.lower()) -} +POLARS_NUMERIC_DTYPE_ALIASES = {name: name for name in POLARS_NUMERIC_DTYPES} FLOAT_TYPE_ORDER = ("Float16", "Float32", "Float64") INTEGER_TYPE_ORDER = ( diff --git a/src/sequifier/hyperparameter_search.py b/src/sequifier/hyperparameter_search.py index affd9144..3fd48c97 100644 --- a/src/sequifier/hyperparameter_search.py +++ b/src/sequifier/hyperparameter_search.py @@ -59,15 +59,15 @@ def _monitored_dataset(run_config: Any) -> tuple[str | None, int]: @beartype def create_sampler(config: Any) -> optuna.samplers.BaseSampler: strategy = getattr(config, "method", "bayesian") - global_seed = getattr(config, "global_seed", None) - if strategy in ["sample"]: - return optuna.samplers.RandomSampler(seed=global_seed) + sampler_seed = getattr(config, "sampler_seed", None) + if strategy == "random": + return optuna.samplers.RandomSampler(seed=sampler_seed) if strategy == "grid": if hasattr(optuna.samplers, "BruteForceSampler"): - return optuna.samplers.BruteForceSampler(seed=global_seed) + return optuna.samplers.BruteForceSampler(seed=sampler_seed) raise RuntimeError("Grid search requires Optuna >= 3.1 for BruteForceSampler.") return optuna.samplers.TPESampler( - seed=global_seed, + seed=sampler_seed, multivariate=True, ) @@ -91,7 +91,7 @@ def objective( monitored_dataset, dataset_count = _monitored_dataset(run_config) config_path = os.path.join( - config.project_root, config.model_config_write_path, f"{run_name}.yaml" + config.project_root, config.trial_config_directory, f"{run_name}.yaml" ) os.makedirs(os.path.dirname(config_path), exist_ok=True) @@ -240,27 +240,27 @@ def consume_metrics( ) evaluation_id = os.path.splitext(os.path.basename(model_path))[0] - if config.evaluation_inference_config: - evaluation_inference_config = config.evaluation_inference_config - if not os.path.isabs(evaluation_inference_config) and not os.path.exists( - evaluation_inference_config + if config.evaluation_inference_config_path: + evaluation_inference_config_path = config.evaluation_inference_config_path + if not os.path.isabs(evaluation_inference_config_path) and not os.path.exists( + evaluation_inference_config_path ): - evaluation_inference_config = os.path.join( + evaluation_inference_config_path = os.path.join( config.project_root, - evaluation_inference_config, + evaluation_inference_config_path, ) subprocess.run( [ "sequifier", "infer", - f"--config-path={evaluation_inference_config}", - f"--model-path={model_path}", + f"--config-path={evaluation_inference_config_path}", + f"--model-paths={model_path}", ], check=True, ) - if config.evaluation_script and config.evaluation_metrics: - eval_script_path = config.evaluation_script + if config.evaluation_script_path and config.evaluation_metrics: + eval_script_path = config.evaluation_script_path cmd = [sys.executable, eval_script_path, evaluation_id] eval_process = subprocess.run( @@ -423,9 +423,13 @@ def hyperparameter_search(config_path: str, skip_metadata: bool) -> None: ) if is_multivariate: + assert config.evaluation_metric_directions is not None study = optuna.create_study( study_name=config.name, - directions=config.evaluation_metric_directions, + directions=[ + {"min": "minimize", "max": "maximize"}[value] + for value in config.evaluation_metric_directions + ], sampler=sampler, storage=f"sqlite:///{storage_path}", load_if_exists=True, @@ -437,11 +441,11 @@ def hyperparameter_search(config_path: str, skip_metadata: bool) -> None: config.evaluation_metric_directions and len(config.evaluation_metric_directions) == 1 ) - else "minimize" + else "min" ) study = optuna.create_study( study_name=config.name, - direction=direction, + direction={"min": "minimize", "max": "maximize"}[direction], sampler=sampler, storage=f"sqlite:///{storage_path}", load_if_exists=True, diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index c0664c9b..1baded50 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -637,11 +637,7 @@ def infer_worker( ) ] - model_paths = ( - config.model_path - if isinstance(config.model_path, list) - else [config.model_path] - ) + model_paths = config.model_paths for model_path in model_paths: target_decoder_ids = None route_args = dict(args_config) @@ -696,7 +692,7 @@ def infer_worker( ) inferer = Inferer( - config.model_type, + ("embedding" if config.task == "embedding" else "generative"), model_path, config.project_root, id_maps, @@ -725,9 +721,9 @@ def infer_worker( model_id = Path(model_path).stem logger.info(f"Inferring for {model_id}") - if config.model_type == "generative": + if config.task == "forecast": infer_generative(config, inferer, model_id, dataset, column_data_types) - if config.model_type == "embedding": + if config.task == "embedding": infer_embedding(config, inferer, model_id, dataset, column_data_types) logger.info("--- Inference Complete ---") @@ -1004,12 +1000,12 @@ def infer_generative( if data_path is None: raise ValueError("data_path must be provided or resolved from metadata") for data_id, data in enumerate(dataset): - if config.autoregressive and isinstance(data, pl.DataFrame): + if config.autoregressive_generation and isinstance(data, pl.DataFrame): data = _autoregressive_seed_dataframe(config, data) windowed = _windowed_inference_batch(config, data, column_data_types) if windowed.sequence_ids.numel() == 0: continue - if config.autoregressive and inferer.prediction_length != 1: + if config.autoregressive_generation and inferer.prediction_length != 1: raise ValueError( "prediction_length must be 1 for autoregressive inference, " f"got {inferer.prediction_length}" @@ -1017,7 +1013,7 @@ def infer_generative( total_steps = ( config.generation_steps - if config.autoregressive and config.generation_steps is not None + if config.autoregressive_generation and config.generation_steps is not None else 1 ) if ( @@ -1284,7 +1280,7 @@ def predict(history): history_validator=validate_history, metadata={ "model_id": model_id, - "model_path": config.model_path, + "model_paths": config.model_paths, "model_interface": config.model_interface, "training_objective": config.training_objective, "target_offset": config.window_view.target_offset, diff --git a/src/sequifier/integration/contexts.py b/src/sequifier/integration/contexts.py index b93646f1..8475614a 100644 --- a/src/sequifier/integration/contexts.py +++ b/src/sequifier/integration/contexts.py @@ -18,7 +18,7 @@ class StepIdentity: global_batch_step: int optimizer_step: int accumulation_index: int - accumulation_steps: int + accumulation_batches: int rank: int world_size: int diff --git a/src/sequifier/io/depth_preprocess.py b/src/sequifier/io/depth_preprocess.py index 022b9e2b..e834bddc 100644 --- a/src/sequifier/io/depth_preprocess.py +++ b/src/sequifier/io/depth_preprocess.py @@ -312,7 +312,7 @@ def _audit_parquet_row_group(job): ) -def _audit_parquet_groups(path, capacity, n_cores): +def _audit_parquet_groups(path, capacity, num_workers): """Count complete items in parallel, joining row groups at item boundaries.""" from sequifier.preprocess import _create_preprocess_pool @@ -320,10 +320,10 @@ def _audit_parquet_groups(path, capacity, n_cores): logger.info( "Depth audit: {} Parquet row groups across {} workers", group_count, - min(n_cores, group_count), + min(num_workers, group_count), ) jobs = ((path, index, capacity) for index in range(group_count)) - with _create_preprocess_pool(min(n_cores, group_count)) as pool: + with _create_preprocess_pool(min(num_workers, group_count)) as pool: audits = list(pool.imap(_audit_parquet_row_group, jobs)) counts = Counter() first_positions = {} @@ -383,7 +383,9 @@ def _validated_item_rows(item, columns, shallow, layout, curriculum_columns): layout.position_column, ) if not ( - layout.position_base <= child < layout.position_base + layout.context_length + layout.position_base + <= child + < layout.position_base + layout.max_depth_length ): raise ValueError(f"Depth position {child} is outside layout capacity") slot = child - layout.position_base @@ -457,7 +459,7 @@ def _fit_parquet_group(job): path, "parquet", required, - layout.context_length, + layout.max_depth_length, row_groups=row_groups, ): sid, pos = item.sequence_id, item.position @@ -574,7 +576,7 @@ def flush(split, index): path, "parquet", job["required"], - layout.context_length, + layout.max_depth_length, row_groups=job["row_groups"], sequence_id=sid, ) @@ -670,14 +672,16 @@ def advance_before(position): encode_until(absolute_start + width - 1) tensors = { c: torch.zeros( - (1, width, layout.context_length) + (1, width, layout.max_depth_length) if c in layout.columns else (1, width), dtype=PANDAS_TO_TORCH_TYPES[col_types[c]], ) for c in columns } - mask = torch.zeros((1, width, layout.context_length), dtype=torch.bool) + mask = torch.zeros( + (1, width, layout.max_depth_length), dtype=torch.bool + ) sample_position = None for time in range(window_pad, width): position = absolute_start + time @@ -889,7 +893,7 @@ def check_sources(): owner.read_format == "parquet" and len(files) == 1 and owner.max_rows is None - and owner.n_cores > 1 + and owner.num_workers > 1 and pq.ParquetFile(files[0]).num_row_groups > 1 ): ( @@ -897,7 +901,7 @@ def check_sources(): first_positions, parallel_groups, parallel_audits, - ) = _audit_parquet_groups(files[0], layout.context_length, owner.n_cores) + ) = _audit_parquet_groups(files[0], layout.max_depth_length, owner.num_workers) selected_count = sum(sequence_counts.values()) previous = parallel_audits[-1].last else: @@ -905,7 +909,7 @@ def check_sources(): files, owner.read_format, ["sequenceId", "itemPosition"], - layout.context_length, + layout.max_depth_length, ): if owner.max_rows is not None and selected_count >= owner.max_rows: continue @@ -926,7 +930,7 @@ def check_sources(): def selected_items(): check_sources() stream = _merged_items( - files, owner.read_format, required, layout.context_length, selected_stop + files, owner.read_format, required, layout.max_depth_length, selected_stop ) try: for _ in range(selected_count): @@ -1072,9 +1076,11 @@ def fit_jobs(): logger.info( "Depth metadata fit: {} source partitions across {} workers", len(parallel_groups), - min(owner.n_cores, len(parallel_groups)), + min(owner.num_workers, len(parallel_groups)), ) - with _create_preprocess_pool(min(owner.n_cores, len(parallel_groups))) as pool: + with _create_preprocess_pool( + min(owner.num_workers, len(parallel_groups)) + ) as pool: for local_stats, local_sets, local_counts in pool.imap( _fit_parquet_group, fit_jobs() ): @@ -1224,7 +1230,7 @@ def fit_jobs(): if ledger["version"] == 2 and not parallel_render: raise ValueError( "Depth parallel resume requires the original Parquet row groups " - "and n_cores > 1" + "and num_workers > 1" ) else: ledger = { @@ -1337,9 +1343,11 @@ def render_jobs(): logger.info( "Depth windows: {} sequences across {} workers", len(sequence_counts), - min(owner.n_cores, len(sequence_counts)), + min(owner.num_workers, len(sequence_counts)), ) - with _create_preprocess_pool(min(owner.n_cores, len(sequence_counts))) as pool: + with _create_preprocess_pool( + min(owner.num_workers, len(sequence_counts)) + ) as pool: for names, records in pool.imap_unordered( _render_parquet_sequence, render_jobs() ): @@ -1494,7 +1502,7 @@ def advance_before(sid, position): encode_until(sid, absolute_start + width - 1) tensors = { c: torch.zeros( - (1, width, layout.context_length) + (1, width, layout.max_depth_length) if c in layout.columns else (1, width), dtype=PANDAS_TO_TORCH_TYPES[col_types[c]], @@ -1502,7 +1510,7 @@ def advance_before(sid, position): for c in columns } mask = torch.zeros( - (1, width, layout.context_length), dtype=torch.bool + (1, width, layout.max_depth_length), dtype=torch.bool ) sample_position = None for time in range(window_pad, width): diff --git a/src/sequifier/io/images.py b/src/sequifier/io/images.py index 5efe7373..2d8929e9 100644 --- a/src/sequifier/io/images.py +++ b/src/sequifier/io/images.py @@ -424,7 +424,7 @@ def validate_sources( "video_index_paths": video_index_paths.get(name, {}), } if preconditioning is not None: - if preconditioning.path is None: + if preconditioning.paths is None: raise ValueError("Preconditioning images need a source path") for name, path in preconditioning.views.items(): index, _, _ = _index( @@ -443,7 +443,7 @@ def validate_sources( f"Cannot decode preconditioning image {frames[0]}: {error}" ) from error manifest["preconditioning_image"] = { - **preconditioning.model_dump(mode="json", exclude={"path"}), + **preconditioning.model_dump(mode="json", exclude={"paths"}), "view_names": sorted(preconditioning.views), "signature": _preconditioning_signature(preconditioning, project_root), } @@ -463,14 +463,14 @@ def __init__( self.preconditioning = preconditioning for name, source in sources.items(): if source.path is None: - raise ValueError(f"Image view {name!r} needs image_path") + raise ValueError(f"Image view {name!r} needs image_paths") if ( source.signature is not None and _signature(source.path, project_root) != source.signature ): raise ValueError(f"Image view {name!r} changed after preprocessing") - if preconditioning is not None and preconditioning.path is None: - raise ValueError("Preconditioning images need preconditioning_image_path") + if preconditioning is not None and preconditioning.paths is None: + raise ValueError("Preconditioning images need preconditioning_image_paths") if ( preconditioning is not None and preconditioning.signature is not None @@ -486,7 +486,7 @@ def __init__( self.item_times: dict[int, list[int]] | None = None for name, source in sources.items(): if source.path is None: - raise ValueError(f"Image view {name!r} needs image_path") + raise ValueError(f"Image view {name!r} needs image_paths") index, timing, mode = _index(source.path, project_root, kind=source.type) self.indices[name] = index if source.type == "videos": diff --git a/src/sequifier/io/pt_payload.py b/src/sequifier/io/pt_payload.py index a4c7a91e..b3a86e00 100644 --- a/src/sequifier/io/pt_payload.py +++ b/src/sequifier/io/pt_payload.py @@ -23,7 +23,7 @@ def validate_tensor_inputs( raise ValueError("Features require batch and temporal axes") for column, values in sequences.items(): layout = layouts.layout_for_column(column) - expected = (*shape, layout.context_length) if layout else shape + expected = (*shape, layout.max_depth_length) if layout else shape if tuple(values.shape) != tuple(expected): raise ValueError( f"{column}: expected {tuple(expected)}, got {tuple(values.shape)}" @@ -56,10 +56,10 @@ def validate_tensor_inputs( mask = masks[name] if mask.dtype != torch.bool or tuple(mask.shape) != ( *shape, - layout.context_length, + layout.max_depth_length, ): raise ValueError( - f"{name}: depth mask must be bool [batch, time, {layout.context_length}]" + f"{name}: depth mask must be bool [batch, time, {layout.max_depth_length}]" ) if not layout.allow_gaps and (mask[..., 1:] & ~mask[..., :-1]).any().item(): raise ValueError( diff --git a/src/sequifier/io/yaml.py b/src/sequifier/io/yaml.py index 7f6ba388..563db27f 100644 --- a/src/sequifier/io/yaml.py +++ b/src/sequifier/io/yaml.py @@ -10,11 +10,11 @@ def represent_sequifier_object(dumper, data): """Represent sequifier config objects as plain YAML mappings.""" values = data.model_dump() if isinstance(data, BaseModel) else vars(data).copy() - for field_name in ("freeze", "freezing_except"): + for field_name in ("freeze_groups", "freeze_except_groups"): if values.get(field_name) is None: values.pop(field_name, None) for component_name in ("ingestion", "backbone", "decoder"): - for suffix in ("freeze", "freezing_except"): + for suffix in ("freeze_groups", "freeze_except_groups"): field_name = f"{component_name}_{suffix}" if values.get(field_name) is None: values.pop(field_name, None) diff --git a/src/sequifier/make.py b/src/sequifier/make.py index cebb3da3..351f79b0 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -4,101 +4,112 @@ from sequifier.typechecking import beartype -preprocess_config_string = """project_root: . +preprocess_config_string = """project_root: "." preprocessing_data_path: PLEASE FILL read_format: csv write_format: parquet -selected_columns: [EXAMPLE_INPUT_COLUMN_NAME] # should include all target column, can include additional columns -column_data_types: null # optional map of selected columns to output dtypes, e.g. {EXAMPLE_INPUT_COLUMN_NAME: Float32} +selected_columns: +- EXAMPLE_INPUT_COLUMN_NAME +column_data_types: null mask_column: null - split_ratios: - 0.8 - 0.1 - 0.1 -split_method: within_sequence # one of within_sequence, between_sequence, value_cutoff -# For value_cutoff, set split_ratios: null and configure both fields below. -# split_column: eventTimestamp -# split_values: [2024-01-01T00:00:00Z, 2024-07-01T00:00:00Z] # creates 3 splits -# Timestamp split columns support Date/Datetime (or ISO strings), not Time/Duration, -# and cannot be used as training or inference inputs/targets. -window_length: 49 +split_method: within_sequence max_target_offset: 1 window_stride: 49 -# prediction_aligned_splits: [1, 2] -# prediction_length: 1 -# target_offset: 1 max_rows: null +window_lengths: +- 49 +- 49 +- 49 """ -train_config_string = """project_root: . +train_config_string = """project_root: "." model_name: PLEASE FILL device: cuda - global_training: training_objective: causal context_length: 48 inference_batch_size: 10 - model: backbone: architecture: - dim_model: 128 + model_dim: 128 num_layers: 3 - attention: {n_heads: 16} - feed_forward: {dim: 128} + attention: + num_heads: 16 + feed_forward: + hidden_dim: 128 dropout: 0.2 - interface: - input_columns: [EXAMPLE_INPUT_COLUMN_NAME] - target_columns: [EXAMPLE_TARGET_COLUMN_NAME] - ingestion: {type: embedding, output_dim: 128} - decoder: {type: linear, prediction_length: 1} - -dataset: - part: {metadata_config_path: PLEASE FILL} - criterion: {EXAMPLE_TARGET_COLUMN_NAME: MSELoss} - + interfaces: + default: + input_columns: + - EXAMPLE_INPUT_COLUMN_NAME + target_columns: + - EXAMPLE_TARGET_COLUMN_NAME + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 training_plan: phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential - sources: [{source: default.default}] - batch_size: 10 - accumulation_steps: 1 - optimization: - model: - learning_rate: 0.0001 - optimizer: {name: AdamW} - scheduler: {name: StepLR, arguments: {step_size: 1, gamma: 0.99}} - scheduler_step_on: epoch - -evaluation: true - + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - source: default.default + batch_size: 10 + accumulation_batches: 1 + optimization: + model: + learning_rate: 0.0001 + optimizer: + name: AdamW + arguments: {} + scheduler: + name: StepLR + arguments: + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + evaluation: + sources: + - source: default.default export_generative_model: true export_embedding_model: false +dataset_training: + default: + criterion: + EXAMPLE_TARGET_COLUMN_NAME: MSELoss + parts: + default: + metadata_config_path: PLEASE FILL """ -infer_config_string = """project_root: . -preprocessing_data_path: PLEASE FILL -model_type: PLEASE FILL # generative or embedding -model_path: PLEASE FILL - -input_columns: [EXAMPLE_INPUT_COLUMN_NAME] # should include all target column, can include additional columns -target_columns: [EXAMPLE_TARGET_COLUMN_NAME] +infer_config_string = """project_root: "." +input_columns: +- EXAMPLE_INPUT_COLUMN_NAME +target_columns: +- EXAMPLE_TARGET_COLUMN_NAME target_column_types: EXAMPLE_TARGET_COLUMN_NAME: real - training_objective: causal output_probabilities: false decode_categories: true device: cpu context_length: 48 inference_batch_size: 10 - -autoregressive: true generation_steps: 5 +task: forecast +model_paths: +- PLEASE FILL +metadata_config_path: configs/metadata_configs/PLEASE_FILL.json +autoregressive_generation: true """ gitignore_string = """models/ diff --git a/src/sequifier/model/backbone.py b/src/sequifier/model/backbone.py index 37db03ca..0ac3aa8f 100644 --- a/src/sequifier/model/backbone.py +++ b/src/sequifier/model/backbone.py @@ -14,7 +14,7 @@ class TransformerBackbone(TransformerEncoderStack): """Dataset-independent temporal transformer. The input and output contract is batch-first - ``[batch, time, input_dim]``. ``input_dim`` normally equals ``dim_model``; + ``[batch, time, input_dim]``. ``input_dim`` normally equals ``model_dim``; ``range_concat`` reserves the final model channel for its coordinate. Attention policy and padding are supplied by the caller as one broadcastable attention mask. @@ -26,11 +26,11 @@ def __init__(self, architecture: Any): architecture, architecture.max_context_length, defer_layers=True ) self.architecture = architecture - self.dim_model = architecture.dim_model + self.model_dim = architecture.model_dim self.max_context_length = architecture.max_context_length self.position_encoding_type = architecture.position_encoding.type - self.positional_encoding_scope = architecture.positional_encoding_scope - self.input_dim = self.dim_model - int( + self.position_encoding_scope = architecture.position_encoding_scope + self.input_dim = self.model_dim - int( self.position_encoding_type == "range_concat" ) self.position_dropout = nn.Dropout(architecture.dropout) @@ -39,17 +39,17 @@ def __init__(self, architecture: Any): self.range_projection: nn.Module | None = None if ( self.position_encoding_type == "learned" - and self.positional_encoding_scope == "global" + and self.position_encoding_scope == "global" ): self.position_embedding = nn.Embedding( - self.max_context_length, self.dim_model + self.max_context_length, self.model_dim ) elif self.position_encoding_type == "range": - self.range_projection = nn.Linear(self.dim_model + 1, self.dim_model) + self.range_projection = nn.Linear(self.model_dim + 1, self.model_dim) sinusoidal = self._sinusoidal_positions( self.max_context_length, - self.dim_model, + self.model_dim, architecture.position_encoding.theta, ) self.register_buffer("sinusoidal_positions", sinusoidal, persistent=False) @@ -85,7 +85,7 @@ def _add_temporal_position(self, x: Tensor) -> Tensor: if ( self.position_encoding_type == "learned" - and self.positional_encoding_scope == "global" + and self.position_encoding_scope == "global" ): positions = torch.arange(sequence_length, device=x.device) if self.position_embedding is None: @@ -139,7 +139,7 @@ def _forward_with_activations( ) -> tuple[Tensor, dict[int | str, Tensor]]: if x.ndim != 3: raise ValueError( - "TransformerBackbone expects [batch, time, dim_model], got " + "TransformerBackbone expects [batch, time, model_dim], got " f"shape {tuple(x.shape)}." ) if not torch.onnx.is_in_onnx_export() and x.shape[-1] != self.input_dim: @@ -152,9 +152,9 @@ def _forward_with_activations( if not torch.onnx.is_in_onnx_export() and prefix.shape != ( x.shape[0], 1, - self.dim_model, + self.model_dim, ): - raise ValueError("Prefix must have shape [batch, 1, dim_model]") + raise ValueError("Prefix must have shape [batch, 1, model_dim]") if ( not torch.onnx.is_in_onnx_export() and x.shape[1] + 1 > self.max_context_length @@ -166,7 +166,7 @@ def _forward_with_activations( "backbone.positioned", x, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) return self._run_layers( x, attention_mask, layer_indices, capture_final_norm, trace diff --git a/src/sequifier/model/decoders.py b/src/sequifier/model/decoders.py index dded50d2..ca1593d3 100644 --- a/src/sequifier/model/decoders.py +++ b/src/sequifier/model/decoders.py @@ -241,7 +241,7 @@ def __init__( self.input_dim = input_dim self.hidden_dims: list[int] = [] self.hidden_weight_l2 = 0.0 - self.width = architecture.dim_model + self.width = architecture.model_dim sequence_length = len(target_columns) + 1 self.context_projection = nn.Linear(input_dim, self.width) @@ -888,7 +888,7 @@ def build_target_decoding( decoder_n_classes = ( hparams.n_classes if target_n_classes is None else target_n_classes ) - input_dim = model.backbone.architecture.dim_model * model.decoder.support + input_dim = model.backbone.architecture.model_dim * model.decoder.support_length hash_codecs = { target: CategoricalHashCodec( config, hparams.n_classes[target], hparams.target_decoder_ids[target] diff --git a/src/sequifier/model/depth_ingestion.py b/src/sequifier/model/depth_ingestion.py index e9719c9d..dea19d7d 100644 --- a/src/sequifier/model/depth_ingestion.py +++ b/src/sequifier/model/depth_ingestion.py @@ -35,7 +35,7 @@ def __init__( self.categorical_columns = frozenset(categorical_columns) self.mask_key = depth_mask_metadata_key(config.layout) architecture = config.architecture - width = architecture.dim_model + width = architecture.model_dim dims = config.feature_embedding_dims or get_feature_embedding_dims( width, categorical_columns, real_columns, n_classes ) @@ -55,21 +55,21 @@ def __init__( ) self.cls = nn.Parameter(torch.zeros(1, 1, width)) self.position_embedding = ( - nn.Embedding(layout.context_length + 1, width) + nn.Embedding(layout.max_depth_length + 1, width) if architecture.position_encoding.type == "learned" else None ) self.register_buffer( "sinusoidal_positions", TransformerBackbone._sinusoidal_positions( - layout.context_length + 1, + layout.max_depth_length + 1, width, architecture.position_encoding.theta, ), persistent=False, ) self.position_dropout = nn.Dropout(architecture.dropout) - self.stack = TransformerEncoderStack(architecture, layout.context_length + 1) + self.stack = TransformerEncoderStack(architecture, layout.max_depth_length + 1) self.output_projection_layer = ( nn.Linear(width, config.output_dim) if width != config.output_dim diff --git a/src/sequifier/model/encoder_stack.py b/src/sequifier/model/encoder_stack.py index ae444a16..f89f1b7d 100644 --- a/src/sequifier/model/encoder_stack.py +++ b/src/sequifier/model/encoder_stack.py @@ -19,7 +19,7 @@ def __init__( ): super().__init__() self.architecture = architecture - self.dim_model = architecture.dim_model + self.model_dim = architecture.model_dim self.max_context_length = max_context_length if not defer_layers: self._construct_layers() @@ -40,7 +40,7 @@ def _construct_layers(self): normalization_type = architecture.normalization.type norm_class = RMSNorm if normalization_type == "rmsnorm" else nn.LayerNorm norm_eps = 1e-6 if normalization_type == "rmsnorm" else 1e-3 - self.final_norm = norm_class(self.dim_model, eps=norm_eps) + self.final_norm = norm_class(self.model_dim, eps=norm_eps) else: self.final_norm = nn.Identity() @@ -62,7 +62,7 @@ def _run_layers( f"{site_prefix}.layer.{index}.input", x, axes=axes, - width=self.dim_model, + width=self.model_dim, ) x = layer( x, @@ -75,7 +75,7 @@ def _run_layers( f"{site_prefix}.layer.{index}.output", x, axes=axes, - width=self.dim_model, + width=self.model_dim, ) if index in selected_indices: activations[index] = x @@ -85,7 +85,7 @@ def _run_layers( f"{site_prefix}.final_norm", x, axes=axes, - width=self.dim_model, + width=self.model_dim, ) if capture_final_norm: activations["final_norm"] = x diff --git a/src/sequifier/model/execution_schema.py b/src/sequifier/model/execution_schema.py index c7ede7fc..3217c179 100644 --- a/src/sequifier/model/execution_schema.py +++ b/src/sequifier/model/execution_schema.py @@ -46,7 +46,7 @@ def from_interface( for index, column in enumerate(interface.input_columns): layout = layouts.layout_for_column(column) shape = ( - ("batch", context_length, layout.context_length) + ("batch", context_length, layout.max_depth_length) if layout else ("batch", context_length) ) @@ -134,7 +134,7 @@ def from_interface( depth_mask_metadata_key(name), "metadata", "bool", - ("batch", context_length, layout.context_length), + ("batch", context_length, layout.max_depth_length), ) ) outputs = ( diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index 8424eaf4..2017ca46 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -293,7 +293,7 @@ def _build_composable_network( direct_real_dtype_provider=lambda: backbone.layers[ 0 ].ff.get_first_layer_dtype(), - device_max_concat_length=config.global_training.device_max_concat_length, + max_concat_tensors=config.global_training.max_concat_tensors, load_pretrained=initialize, ) from sequifier.model.ingestions import EmbeddingFeatureIngestion @@ -354,8 +354,8 @@ def _build_composable_network( interface.static_categorical_columns, interface.static_real_columns, interface.n_classes, - backbone.dim_model, - interface.static_conditioning.embedding_size, + backbone.model_dim, + interface.static_conditioning.total_embedding_dim, interface.static_conditioning.feature_embedding_dims, ) static_encoder._sequifier_initialization_seed = derived_seed( @@ -367,12 +367,12 @@ def _build_composable_network( image_prefix_encoder = ImagePrefixEncoder( interface.preconditioning_image_encoder, - backbone.dim_model, + backbone.model_dim, len(interface.preconditioning_image.views), initialize, ) prefix_projection = ( - nn.Linear(2 * backbone.dim_model, backbone.dim_model) + nn.Linear(2 * backbone.model_dim, backbone.model_dim) if static_encoder is not None and image_prefix_encoder is not None else None ) @@ -380,8 +380,8 @@ def _build_composable_network( ingestion=built_ingestion.module, ingestion_adapter=adapter, decoder=decoder, - decoder_input_width=interface.decoder.support * backbone.dim_model, - decoding_support=interface.decoder.support, + decoder_input_width=interface.decoder.support_length * backbone.model_dim, + decoding_support=interface.decoder.support_length, prediction_length=interface.decoder.prediction_length, target_columns=tuple(interface.target_columns), target_column_types=dict(interface.target_column_types), diff --git a/src/sequifier/model/images.py b/src/sequifier/model/images.py index 8f90f2fb..16ce9352 100644 --- a/src/sequifier/model/images.py +++ b/src/sequifier/model/images.py @@ -117,7 +117,7 @@ def __init__( views: list[str], encoder_config: Any, output_dim: int, - frame_pool: str, + frame_pooling: str, load_pretrained: bool = True, ): super().__init__() @@ -125,7 +125,7 @@ def __init__( self.encoder = ImageEncoder(encoder_config, output_dim, load_pretrained) self.view_embedding = nn.Parameter(torch.zeros(len(views), output_dim)) self.frame_score = ( - nn.Linear(output_dim, 1) if frame_pool == "attention" else None + nn.Linear(output_dim, 1) if frame_pooling == "attention" else None ) self.frame_sum = _AxisPoolBlock( axes=["frame"], mode="sum", active_axes=["frame"] @@ -189,7 +189,7 @@ def __init__( config.views, config.encoder, config.output_dim, - config.frame_pool, + config.frame_pooling, load_pretrained, ) self.merge = ( diff --git a/src/sequifier/model/ingestion_compiler.py b/src/sequifier/model/ingestion_compiler.py index f1f5780b..d661c19a 100644 --- a/src/sequifier/model/ingestion_compiler.py +++ b/src/sequifier/model/ingestion_compiler.py @@ -63,7 +63,7 @@ class BuiltIngestion: class IngestionBuildContext: hparams: Any direct_real_dtype_provider: Callable[[], torch.dtype] - device_max_concat_length: int + max_concat_tensors: int add_ingestion_position: bool load_pretrained: bool @@ -307,7 +307,7 @@ def resolve_ingestion_plan(hparams: Any) -> IngestionPlan: if objective_name is not None else False ) - transformer_input_width = architecture.dim_model - int( + transformer_input_width = architecture.model_dim - int( architecture.position_encoding.type == "range_concat" ) from sequifier.config.depth_layout import DepthLayoutRegistryModel @@ -443,7 +443,9 @@ def _build_embedding_handler( real_columns=list(branch.real_columns), n_classes=context.hparams.n_classes, context_length=context.hparams.window_view.context_length, - embedding_size=None if feature_embedding_dims is not None else branch.width, + total_embedding_dim=None + if feature_embedding_dims is not None + else branch.width, feature_embedding_dims=feature_embedding_dims, add_ingestion_position=context.add_ingestion_position, dropout=branch.config.dropout, @@ -453,7 +455,7 @@ def _build_embedding_handler( if column in context.hparams.categorical_hashing }, embedding_dim=branch.width, - device_max_concat_length=context.device_max_concat_length, + max_concat_tensors=context.max_concat_tensors, ) @@ -471,7 +473,7 @@ def _build_passthrough_module( dropout=branch.config.dropout, projection_dim=projection_dim, direct_real_dtype_provider=context.direct_real_dtype_provider, - device_max_concat_length=context.device_max_concat_length, + max_concat_tensors=context.max_concat_tensors, ) @@ -503,7 +505,7 @@ def _build_temporal_conv_handler( causal=branch.config.causal, activation=branch.config.activation, dropout=branch.config.dropout, - post_conv_norm=branch.config.post_conv_norm, + post_conv_normalization=branch.config.post_conv_normalization, orientation=branch.config.orientation, context_length=context.hparams.window_view.context_length, ) @@ -554,7 +556,7 @@ def _build_structured_handler( processing_blocks=branch.config.processing_blocks, image_cells=branch.config.image_cells, image_encoder_config=branch.config.image_encoder, - image_frame_pool=branch.config.image_frame_pool, + image_frame_pooling=branch.config.image_frame_pooling, load_pretrained=context.load_pretrained, **_common_branch_kwargs(branch, context), ) @@ -600,7 +602,7 @@ def _validate_depth(hparams, usage, columns, config, layout): ) if config.feature_embedding_dims is None: get_feature_embedding_dims( - config.architecture.dim_model, + config.architecture.model_dim, categorical, real, hparams.n_classes, @@ -669,17 +671,17 @@ def compile_feature_ingestion( *, hparams: Any, direct_real_dtype_provider: Callable[[], torch.dtype], - device_max_concat_length: int, + max_concat_tensors: int, load_pretrained: bool = True, ) -> BuiltIngestion: plan = resolve_ingestion_plan(hparams) context = IngestionBuildContext( hparams=hparams, direct_real_dtype_provider=direct_real_dtype_provider, - device_max_concat_length=device_max_concat_length, + max_concat_tensors=max_concat_tensors, add_ingestion_position=( hparams.model.backbone.architecture.position_encoding.type == "learned" - and hparams.model.backbone.architecture.positional_encoding_scope + and hparams.model.backbone.architecture.position_encoding_scope == "per_feature" ), load_pretrained=load_pretrained, @@ -747,11 +749,11 @@ def build_feature_ingestion( *, hparams: Any, direct_real_dtype_provider: Callable[[], torch.dtype], - device_max_concat_length: int, + max_concat_tensors: int, ) -> BaseFeatureIngestion: """Compatibility wrapper returning only the compiled runtime module.""" return compile_feature_ingestion( hparams=hparams, direct_real_dtype_provider=direct_real_dtype_provider, - device_max_concat_length=device_max_concat_length, + max_concat_tensors=max_concat_tensors, ).module diff --git a/src/sequifier/model/ingestions.py b/src/sequifier/model/ingestions.py index f6a194a8..a908414d 100644 --- a/src/sequifier/model/ingestions.py +++ b/src/sequifier/model/ingestions.py @@ -62,7 +62,7 @@ def _validate_module_dict_key(key: str, usage: str) -> None: @beartype def get_feature_embedding_dims( - embedding_size: int, + total_embedding_dim: int, categorical_columns: list[str], real_columns: list[str], n_classes: Optional[dict[str, int]] = None, @@ -72,9 +72,9 @@ def get_feature_embedding_dims( raise ValueError("No columns found") if len(categorical_columns) == 0 and len(real_columns) > 0: - if embedding_size < len(real_columns): + if total_embedding_dim < len(real_columns): raise ValueError( - f"embedding_size ({embedding_size}) is smaller than the " + f"total_embedding_dim ({total_embedding_dim}) is smaller than the " f"number of real input columns ({len(real_columns)}). " "Cannot allocate at least 1 dimension per column." ) @@ -82,22 +82,22 @@ def get_feature_embedding_dims( feature_embedding_dims = {col: 1 for col in real_columns} column_index = dict(enumerate(real_columns)) - remaining_dims = embedding_size - len(real_columns) + remaining_dims = total_embedding_dim - len(real_columns) for i in range(remaining_dims): j = i % len(real_columns) feature_embedding_dims[column_index[j]] += 1 - if sum(feature_embedding_dims.values()) != embedding_size: + if sum(feature_embedding_dims.values()) != total_embedding_dim: raise ValueError( "Auto-calculated embedding dimensions " f"({sum(feature_embedding_dims.values())}) do not sum to " - f"embedding_size ({embedding_size})." + f"total_embedding_dim ({total_embedding_dim})." ) elif len(real_columns) == 0 and len(categorical_columns) > 0: minimum_size = 2 * len(categorical_columns) - if embedding_size < minimum_size: + if total_embedding_dim < minimum_size: raise ValueError( - f"embedding_size ({embedding_size}) is smaller than the " + f"total_embedding_dim ({total_embedding_dim}) is smaller than the " f"minimum of 2 dimensions per categorical column ({minimum_size})." ) if n_classes is None: @@ -116,7 +116,7 @@ def get_feature_embedding_dims( weights = [math.log(cardinalities[col]) for col in categorical_columns] if not any(weights): weights = [1.0] * len(categorical_columns) - remaining = embedding_size - minimum_size + remaining = total_embedding_dim - minimum_size weight_total = sum(weights) quotas = [remaining * weight / weight_total for weight in weights] extra_dims = [math.floor(quota) for quota in quotas] @@ -239,13 +239,13 @@ def __init__( real_columns: list[str], n_classes: dict[str, int], context_length: int, - embedding_size: Optional[int], + total_embedding_dim: Optional[int], feature_embedding_dims: Optional[dict[str, int]], add_ingestion_position: bool, dropout: float, hashing: Optional[dict[str, Any]] = None, embedding_dim: Optional[int] = None, - device_max_concat_length: int = 12, + max_concat_tensors: int = 12, ): super().__init__() self.categorical_columns = categorical_columns @@ -255,18 +255,18 @@ def __init__( self.add_ingestion_position = add_ingestion_position self.drop = nn.Dropout(dropout) self.hashing = hashing or {} - self.device_max_concat_length = device_max_concat_length + self.max_concat_tensors = max_concat_tensors if feature_embedding_dims is not None: self.feature_embedding_dims = feature_embedding_dims else: - if embedding_size is None: + if total_embedding_dim is None: raise ValueError( "embedding ingestion requires an embedding dimension when " "feature_embedding_dims is not configured" ) self.feature_embedding_dims = get_feature_embedding_dims( - embedding_size, + total_embedding_dim, categorical_columns, real_columns, n_classes, @@ -274,7 +274,7 @@ def __init__( ) self.input_dim = sum(self.feature_embedding_dims.values()) - self.embedding_size = self.input_dim + self.total_embedding_dim = self.input_dim self.embedding_dim = embedding_dim or self.input_dim self.encoder = ModuleDict() @@ -324,14 +324,12 @@ def __init__( @conditional_beartype def _recursive_concat(self, srcs: list[Tensor]) -> Tensor: - if len(srcs) <= self.device_max_concat_length: + if len(srcs) <= self.max_concat_tensors: return torch.cat(srcs, 2) srcs_inner = [] - for start in range(0, len(srcs), self.device_max_concat_length): - src = self._recursive_concat( - srcs[start : start + self.device_max_concat_length] - ) + for start in range(0, len(srcs), self.max_concat_tensors): + src = self._recursive_concat(srcs[start : start + self.max_concat_tensors]) srcs_inner.append(src) return self._recursive_concat(srcs_inner) @@ -386,7 +384,7 @@ def __init__( dropout: float, projection_dim: Optional[int] = None, direct_real_dtype_provider: Optional[Callable[[], torch.dtype]] = None, - device_max_concat_length: int = 12, + max_concat_tensors: int = 12, ): super().__init__() if not real_columns: @@ -398,10 +396,10 @@ def __init__( self.add_ingestion_position = add_ingestion_position self.drop = nn.Dropout(dropout) self.input_dim = len(real_columns) - self.embedding_size = self.input_dim + self.total_embedding_dim = self.input_dim self.projection_dim = projection_dim or self.input_dim self.direct_real_dtype_provider = direct_real_dtype_provider - self.device_max_concat_length = device_max_concat_length + self.max_concat_tensors = max_concat_tensors if self.add_ingestion_position: self.pos_encoder = ModuleDict() @@ -419,14 +417,12 @@ def __init__( @conditional_beartype def _recursive_concat(self, srcs: list[Tensor]) -> Tensor: - if len(srcs) <= self.device_max_concat_length: + if len(srcs) <= self.max_concat_tensors: return torch.cat(srcs, 2) srcs_inner = [] - for start in range(0, len(srcs), self.device_max_concat_length): - src = self._recursive_concat( - srcs[start : start + self.device_max_concat_length] - ) + for start in range(0, len(srcs), self.max_concat_tensors): + src = self._recursive_concat(srcs[start : start + self.max_concat_tensors]) srcs_inner.append(src) return self._recursive_concat(srcs_inner) @@ -482,7 +478,7 @@ def __init__( causal: bool, activation: str, dropout: float, - post_conv_norm: str, + post_conv_normalization: str, orientation: str, context_length: int, ): @@ -515,7 +511,7 @@ def __init__( norm_dim = ( context_length if self.orientation == "within_column" else self.channels ) - self.post_conv_norm = self._norm(post_conv_norm, norm_dim) + self.post_conv_normalization = self._norm(post_conv_normalization, norm_dim) @staticmethod @conditional_beartype @@ -537,19 +533,19 @@ def _norm(name: str, output_dim: int) -> nn.Module: return RMSNorm(output_dim) if name == "none": return nn.Identity() - raise ValueError(f"Unknown temporal_conv post_conv_norm: {name}") + raise ValueError(f"Unknown temporal_conv post_conv_normalization: {name}") @conditional_beartype def _apply_post_conv_norm(self, output: Tensor) -> Tensor: if self.orientation == "within_column": output = output.transpose(1, 2) - output = self.post_conv_norm( - cast_floating_to_module_dtype(output, self.post_conv_norm) + output = self.post_conv_normalization( + cast_floating_to_module_dtype(output, self.post_conv_normalization) ) return output.transpose(1, 2) if self.orientation == "within_item_position": - return self.post_conv_norm( - cast_floating_to_module_dtype(output, self.post_conv_norm) + return self.post_conv_normalization( + cast_floating_to_module_dtype(output, self.post_conv_normalization) ) raise ValueError( f"Unknown temporal_conv normalization orientation: {self.orientation}" @@ -1034,7 +1030,7 @@ def __init__( *, input_dim: int, output_dim: int, - n_heads: int, + num_heads: int, dropout: float, ): super().__init__() @@ -1045,7 +1041,7 @@ def __init__( ) self.attention = nn.MultiheadAttention( embed_dim=output_dim, - num_heads=n_heads, + num_heads=num_heads, dropout=dropout, batch_first=True, ) @@ -1069,7 +1065,7 @@ def __init__( axes: list[str], unshared_axes: list[str], output_dim: int, - n_heads: int, + num_heads: int, dropout: float, active_axes: list[str], axis_sizes: dict[str, int], @@ -1079,7 +1075,7 @@ def __init__( self.axes = axes self.unshared_axes = unshared_axes self.output_dim = output_dim - self.n_heads = n_heads + self.num_heads = num_heads self.active_axes = active_axes self.output_axes = list(active_axes) self.axis_sizes = axis_sizes @@ -1093,7 +1089,7 @@ def __init__( _module_key(indices): _AxisAttentionLayer( input_dim=self.input_dim, output_dim=self.output_dim, - n_heads=self.n_heads, + num_heads=self.num_heads, dropout=dropout, ) for indices in self.unshared_indices @@ -1301,7 +1297,7 @@ def _build_axis_attention( axes=block.axes, unshared_axes=block.unshared_axes, output_dim=block.output_dim, - n_heads=block.n_heads, + num_heads=block.num_heads, dropout=block.dropout, active_axes=list(shape.active_axes), axis_sizes=axis_sizes, @@ -1315,7 +1311,7 @@ def _build_axis_pool( ) -> nn.Module: return _AxisPoolBlock( axes=block.axes, - mode=block.mode, + mode=block.pooling, active_axes=list(shape.active_axes), ) @@ -1352,7 +1348,7 @@ def __init__( processing_blocks: Optional[list[Any]] = None, image_cells: Optional[dict[str, str]] = None, image_encoder_config: Optional[Any] = None, - image_frame_pool: str = "mean", + image_frame_pooling: str = "mean", load_pretrained: bool = True, ): self.layout = layout @@ -1399,7 +1395,7 @@ def __init__( list(dict.fromkeys(self.image_cells.values())), image_encoder_config, self.cell_dim, - image_frame_pool, + image_frame_pooling, load_pretrained, ) if self.add_ingestion_position: @@ -1709,7 +1705,7 @@ def compile_feature_ingestion( *, hparams: Any, direct_real_dtype_provider: Callable[[], torch.dtype], - device_max_concat_length: int, + max_concat_tensors: int, load_pretrained: bool = True, ): from sequifier.model.ingestion_compiler import compile_feature_ingestion as compile @@ -1717,7 +1713,7 @@ def compile_feature_ingestion( return compile( hparams=hparams, direct_real_dtype_provider=direct_real_dtype_provider, - device_max_concat_length=device_max_concat_length, + max_concat_tensors=max_concat_tensors, load_pretrained=load_pretrained, ) @@ -1727,11 +1723,11 @@ def build_feature_ingestion( *, hparams: Any, direct_real_dtype_provider: Callable[[], torch.dtype], - device_max_concat_length: int, + max_concat_tensors: int, ) -> BaseFeatureIngestion: """Compatibility wrapper returning only the compiled runtime module.""" return compile_feature_ingestion( hparams=hparams, direct_real_dtype_provider=direct_real_dtype_provider, - device_max_concat_length=device_max_concat_length, + max_concat_tensors=max_concat_tensors, ).module diff --git a/src/sequifier/model/initialization.py b/src/sequifier/model/initialization.py index 955791b6..c1544bba 100644 --- a/src/sequifier/model/initialization.py +++ b/src/sequifier/model/initialization.py @@ -230,18 +230,13 @@ def _apply_method( elif method == "uniform": for parameter in tensors: nn.init.uniform_(parameter, a=config.low, b=config.high) - elif method in { - "xavier", - "glorot", - "xavier_uniform", - "glorot_uniform", - }: + elif method == "xavier_uniform": if config.fan_mode == "joint": self._joint_xavier_uniform_raw(tensors, gain=config.gain) else: for parameter in tensors: nn.init.xavier_uniform_(parameter, gain=config.gain) - elif method in {"xavier_normal", "glorot_normal"}: + elif method == "xavier_normal": if config.fan_mode == "joint": self._joint_xavier_normal_raw(tensors, gain=config.gain) else: @@ -470,7 +465,7 @@ def _initialize_range_position_projection(self) -> None: raise TypeError("range_position_projection must be an nn.Linear") if projection.out_features + 1 != projection.in_features: raise ValueError( - "range_position_projection must map dim_model + 1 to dim_model" + "range_position_projection must map model_dim + 1 to model_dim" ) if not self._apply_override( diff --git a/src/sequifier/model/layers.py b/src/sequifier/model/layers.py index 74b8e583..7763fd08 100644 --- a/src/sequifier/model/layers.py +++ b/src/sequifier/model/layers.py @@ -74,7 +74,7 @@ def apply_rotary_pos_emb(q, k, cos, sin): class FeedForward(nn.Module): @beartype - def __init__(self, dim_model, dim_feedforward, activation, dropout): + def __init__(self, model_dim, dim_feedforward, activation, dropout): super().__init__() self.activation = activation @@ -82,12 +82,12 @@ def __init__(self, dim_model, dim_feedforward, activation, dropout): # SwiGLU requires 2 gates, so we often adjust dim_feedforward or keep it # but implement the GLU split. Commonly SwiGLU hidden dim is 2/3 of standard. # Here we strictly follow config dim_feedforward. - self.w1 = nn.Linear(dim_model, dim_feedforward) - self.w2 = nn.Linear(dim_model, dim_feedforward) # Gate - self.w3 = nn.Linear(dim_feedforward, dim_model) # Output + self.w1 = nn.Linear(model_dim, dim_feedforward) + self.w2 = nn.Linear(model_dim, dim_feedforward) # Gate + self.w3 = nn.Linear(dim_feedforward, model_dim) # Output else: - self.linear1 = nn.Linear(dim_model, dim_feedforward) - self.linear2 = nn.Linear(dim_feedforward, dim_model) + self.linear1 = nn.Linear(model_dim, dim_feedforward) + self.linear2 = nn.Linear(dim_feedforward, model_dim) self.act = nn.GELU() if activation == "gelu" else nn.ReLU() self.dropout = nn.Dropout(dropout) @@ -146,9 +146,9 @@ class SelfAttention(nn.Module): @beartype def __init__( self, - dim_model, - n_heads, - n_kv_heads, + model_dim, + num_heads, + num_kv_heads, attention_type, dropout, context_length, @@ -157,18 +157,18 @@ def __init__( output_projection=True, ): super().__init__() - self.n_heads = n_heads - self.n_kv_heads = n_kv_heads if n_kv_heads is not None else n_heads - self.dim_model = dim_model - self.head_dim = dim_model // n_heads + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads + self.model_dim = model_dim + self.head_dim = model_dim // num_heads self.attention_type = attention_type self.use_rope = use_rope - self.wq = nn.Linear(dim_model, n_heads * self.head_dim, bias=False) - self.wk = nn.Linear(dim_model, self.n_kv_heads * self.head_dim, bias=False) - self.wv = nn.Linear(dim_model, self.n_kv_heads * self.head_dim, bias=False) + self.wq = nn.Linear(model_dim, num_heads * self.head_dim, bias=False) + self.wk = nn.Linear(model_dim, self.num_kv_heads * self.head_dim, bias=False) + self.wv = nn.Linear(model_dim, self.num_kv_heads * self.head_dim, bias=False) self.wo = ( - nn.Linear(n_heads * self.head_dim, dim_model, bias=False) + nn.Linear(num_heads * self.head_dim, model_dim, bias=False) if output_projection else nn.Identity() ) @@ -200,17 +200,17 @@ def forward( xq = ( self.wq(xq_input) - .view(batch_size, seq_len, self.n_heads, self.head_dim) + .view(batch_size, seq_len, self.num_heads, self.head_dim) .transpose(1, 2) ) xk = ( self.wk(xk_input) - .view(batch_size, seq_len, self.n_kv_heads, self.head_dim) + .view(batch_size, seq_len, self.num_kv_heads, self.head_dim) .transpose(1, 2) ) xv = ( self.wv(xv_input) - .view(batch_size, seq_len, self.n_kv_heads, self.head_dim) + .view(batch_size, seq_len, self.num_kv_heads, self.head_dim) .transpose(1, 2) ) @@ -219,8 +219,8 @@ def forward( xq, xk = apply_rotary_pos_emb(xq, xk, cos, sin) # Handle GQA/MQA by repeating keys/values - if self.n_kv_heads != self.n_heads: - n_rep = self.n_heads // self.n_kv_heads + if self.num_kv_heads != self.num_heads: + n_rep = self.num_heads // self.num_kv_heads xk = xk.repeat_interleave(n_rep, dim=1) xv = xv.repeat_interleave(n_rep, dim=1) @@ -291,7 +291,7 @@ def forward( f"{site_prefix}.update", output, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) return output @@ -300,22 +300,22 @@ class SequifierEncoderLayer(nn.Module): @beartype def __init__(self, architecture, max_context_length=None): super().__init__() - dim_model = architecture.dim_model - self.dim_model = dim_model + model_dim = architecture.model_dim + self.model_dim = model_dim self.norm_first = architecture.normalization.norm_first # Normalization normalization_type = architecture.normalization.type NormClass = RMSNorm if normalization_type == "rmsnorm" else nn.LayerNorm norm_eps = 1e-6 if normalization_type == "rmsnorm" else 1e-3 - self.norm1 = NormClass(dim_model, eps=norm_eps) - self.norm2 = NormClass(dim_model, eps=norm_eps) + self.norm1 = NormClass(model_dim, eps=norm_eps) + self.norm2 = NormClass(model_dim, eps=norm_eps) # Attention self.attn = SelfAttention( - dim_model=dim_model, - n_heads=architecture.attention.n_heads, - n_kv_heads=architecture.attention.n_kv_heads, + model_dim=model_dim, + num_heads=architecture.attention.num_heads, + num_kv_heads=architecture.attention.num_kv_heads, attention_type=architecture.attention.type, dropout=architecture.dropout, context_length=( @@ -330,8 +330,8 @@ def __init__(self, architecture, max_context_length=None): # Feed Forward self.ff = FeedForward( - dim_model, - architecture.feed_forward.dim, + model_dim, + architecture.feed_forward.hidden_dim, architecture.feed_forward.activation, architecture.dropout, ) @@ -365,7 +365,7 @@ def forward( f"{site_prefix}.attention.norm_input", normed_src, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) attention_update = self.dropout( self.attn( @@ -384,7 +384,7 @@ def forward( f"{site_prefix}.attention.output", x, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) normed_x = self.norm2(cast_floating_to_module_dtype(x, self.norm2)) if trace is not None: @@ -392,7 +392,7 @@ def forward( f"{site_prefix}.mlp.norm_input", normed_x, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) mlp_update = self.dropout( self.ff(normed_x, trace=trace, site_prefix=f"{site_prefix}.mlp") @@ -402,7 +402,7 @@ def forward( f"{site_prefix}.mlp.update", mlp_update, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) x = self._residual_add(x, mlp_update) else: @@ -412,7 +412,7 @@ def forward( f"{site_prefix}.attention.norm_input", attention_input, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) x = self._residual_add( src, @@ -431,13 +431,13 @@ def forward( f"{site_prefix}.attention.output", x, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) x = trace.emit( f"{site_prefix}.mlp.norm_input", x, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) mlp_update = self.dropout( self.ff(x, trace=trace, site_prefix=f"{site_prefix}.mlp") @@ -447,7 +447,7 @@ def forward( f"{site_prefix}.mlp.update", mlp_update, axes=("batch", "time", "channel"), - width=self.dim_model, + width=self.model_dim, ) x = self._residual_add(x, mlp_update) x = self.norm2(cast_floating_to_module_dtype(x, self.norm2)) diff --git a/src/sequifier/model/network.py b/src/sequifier/model/network.py index fb92203b..36959264 100644 --- a/src/sequifier/model/network.py +++ b/src/sequifier/model/network.py @@ -263,8 +263,8 @@ def __init__( @property @conditional_beartype - def dim_model(self) -> int: - return int(self.backbone.dim_model) + def model_dim(self) -> int: + return int(self.backbone.model_dim) @property @conditional_beartype @@ -300,7 +300,7 @@ def structural_metadata(self) -> dict[str, object]: "state_dict_prefixes": ("backbone.", "interfaces."), "interfaces": tuple(self.interfaces), "context_length": self.context_length, - "dim_model": self.dim_model, + "model_dim": self.model_dim, } @conditional_beartype @@ -316,7 +316,7 @@ def trace_catalog_for( sites = list( trace_sites( num_layers=len(self.backbone.layers), - model_width=self.dim_model, + model_width=self.model_dim, attention_width=int(first_attention.head_dim), decoder_input_width=route.decoder_input_width, decoder_branches=branch_counts, @@ -446,10 +446,10 @@ def decode_representation( route = self.resolve_interface(interface_name) if ( not torch.onnx.is_in_onnx_export() - and representation.shape[-1] != self.dim_model + and representation.shape[-1] != self.model_dim ): raise ValueError( - f"Representation width must be {self.dim_model}, got " + f"Representation width must be {self.model_dim}, got " f"{representation.shape[-1]}." ) if request.apply_final_norm: diff --git a/src/sequifier/model/static_conditioning.py b/src/sequifier/model/static_conditioning.py index f9d49b32..b87b05c1 100644 --- a/src/sequifier/model/static_conditioning.py +++ b/src/sequifier/model/static_conditioning.py @@ -16,8 +16,8 @@ def __init__( categorical_columns: list[str], real_columns: list[str], n_classes: dict[str, int], - dim_model: int, - embedding_size: int | None = None, + model_dim: int, + total_embedding_dim: int | None = None, feature_embedding_dims: dict[str, int] | None = None, ) -> None: super().__init__() @@ -26,9 +26,9 @@ def __init__( self.real_columns = tuple(real_columns) if feature_embedding_dims is not None: widths = feature_embedding_dims - elif embedding_size is not None: + elif total_embedding_dim is not None: widths = get_feature_embedding_dims( - embedding_size, categorical_columns, real_columns, n_classes + total_embedding_dim, categorical_columns, real_columns, n_classes ) else: widths = { @@ -53,7 +53,7 @@ def __init__( raise ValueError( "Every static conditioning column must be categorical or real" ) - self.projection = nn.Linear(input_width, dim_model) + self.projection = nn.Linear(input_width, model_dim) self.projection._sequifier_layer_group = "ingestion.output_projection" def forward(self, features: dict[str, Tensor]) -> Tensor: diff --git a/src/sequifier/preprocess.py b/src/sequifier/preprocess.py index 4f38df49..e7168c79 100644 --- a/src/sequifier/preprocess.py +++ b/src/sequifier/preprocess.py @@ -196,7 +196,7 @@ def _finalize_cardinality_maps( """Build retained-value maps followed by optional hashing buckets.""" for column, config in cardinality_config.items(): retained = [] - if "min_freq" in config or "top_k" in config: + if "min_count" in config or "max_categories" in config: counts = value_counts.get(column, Counter()) if not counts: raise ValueError( @@ -208,11 +208,11 @@ def _finalize_cardinality_maps( for value, count in counts.items() if value not in SPECIAL_TOKEN_LABELS } - if "min_freq" in config: + if "min_count" in config: retained = [ value for value, count in user_counts.items() - if count >= int(config["min_freq"]) + if count >= int(config["min_count"]) ] else: retained = [ @@ -220,7 +220,7 @@ def _finalize_cardinality_maps( for value, _ in sorted( user_counts.items(), key=lambda item: (-item[1], _category_sort_key(item[0])), - )[: int(config["top_k"])] + )[: int(config["max_categories"])] ] if retained: id_maps[column] = create_id_map(pl.DataFrame({column: retained}), column) @@ -526,10 +526,8 @@ def _normalize_column_types( def _column_types_from_metadata( metadata: dict[str, Any], ) -> Optional[dict[str, str]]: - """Read current and historical metadata field names.""" - return _normalize_column_types( - metadata.get("column_data_types") or metadata.get("column_types") - ) + """Read the canonical storage dtype mapping.""" + return _normalize_column_types(metadata.get("column_data_types")) @beartype @@ -844,7 +842,7 @@ def _audit_folder_sequence_summaries( max_rows: Optional[int], split_column: Optional[str] = None, split_values: Optional[list[Any]] = None, - n_cores: int = 1, + num_workers: int = 1, ) -> tuple[list[str], dict[int, list[int]], Optional[list[int]]]: """Return selected folder files and global sequence coordinate summaries.""" summaries: dict[int, list[int]] = {} @@ -852,11 +850,11 @@ def _audit_folder_sequence_summaries( rows_read = 0 cutoffs = _normalized_split_cutoffs(split_values) cutoff_counts = [0] * (len(cutoffs) + 1) if cutoffs is not None else None - if max_rows is None and n_cores > 1 and len(file_paths) > 1: + if max_rows is None and num_workers > 1 and len(file_paths) > 1: jobs = [ (path, read_format, None, split_column, split_values) for path in file_paths ] - with _create_preprocess_pool(min(n_cores, len(jobs))) as pool: + with _create_preprocess_pool(min(num_workers, len(jobs))) as pool: results = pool.map(_audit_folder_file, jobs) files_and_results = zip(file_paths, results) else: @@ -1093,11 +1091,10 @@ def __init__( allow_sequence_splitting: bool, selected_columns: Optional[list[str]], split_ratios: Optional[list[float]], - window_length: Optional[int], window_stride: int, max_rows: Optional[int], seed: int, - n_cores: Optional[int], + num_workers: Optional[int], batches_per_file: int, process_by_file: bool, prediction_aligned_splits: list[int], @@ -1208,12 +1205,9 @@ def __init__( raise ValueError("Ratio-based split methods require split_ratios") self.split_ratios = split_ratios self.window_stride = window_stride - if window_lengths is None: - if window_length is None: - raise ValueError("Specify window_length or window_lengths") - resolved_window_lengths = [window_length] * self.n_splits - else: - resolved_window_lengths = window_lengths + if not window_lengths: + raise ValueError("window_lengths requires one width per split") + resolved_window_lengths = window_lengths self.window_lengths: list[int] = resolved_window_lengths self.split_layouts = [ StoredWindowLayout( @@ -1254,7 +1248,7 @@ def __init__( self.seed = seed np.random.seed(seed) - self.n_cores = n_cores or multiprocessing.cpu_count() + self.num_workers = num_workers or multiprocessing.cpu_count() self.continue_preprocessing = continue_preprocessing self.storage_layout = StoredWindowLayout( window_length=resolved_window_lengths[0], @@ -1534,7 +1528,7 @@ def __init__( self.data_name_root, data, schema, - self.n_cores, + self.num_workers, self.split_layouts, window_stride, data_columns, @@ -1604,7 +1598,7 @@ def __init__( max_rows, self.split_column if self.metadata_config_path else None, self.split_values if self.metadata_config_path else None, - self.n_cores, + self.num_workers, ) fragmented_sequence_ids = { sequence_id @@ -1664,7 +1658,7 @@ def __init__( if col_types is None: raise ValueError( "Metadata used with folder preprocessing must contain " - "'column_data_types' or the historical 'column_types' field." + "'column_data_types' field." ) col_types = { column: dtype @@ -1830,7 +1824,7 @@ def __init__( selected_columns, max_rows, schema, - self.n_cores, + self.num_workers, self.split_layouts, window_stride, data_columns, @@ -1910,7 +1904,7 @@ def _validate_image_sources(self) -> dict[str, Any]: self.static_data_path, ) _, summaries, _ = _audit_folder_sequence_summaries( - files, self.read_format, self.max_rows, n_cores=1 + files, self.read_format, self.max_rows, num_workers=1 ) if any(summary[0] != 0 for summary in summaries.values()): raise ValueError("Image sequences require itemPosition to start at zero") @@ -2397,10 +2391,10 @@ def consume(path: str, result: _FolderMetadataResult) -> None: } logger.info(f"Data path: {data_path}") - if max_rows is None and self.n_cores > 1 and len(files_to_process) > 1: + if max_rows is None and self.num_workers > 1 and len(files_to_process) > 1: jobs = (job_for(path, None) for path in files_to_process) with _create_preprocess_pool( - min(self.n_cores, len(files_to_process)) + min(self.num_workers, len(files_to_process)) ) as pool: for path, result in zip( files_to_process, pool.imap(_fit_folder_metadata_file, jobs) @@ -2506,7 +2500,7 @@ def _process_fragmented_folder( target_bucket_bytes = 256 * 1024 * 1024 bucket_count = min( 4096, - max(self.n_cores, math.ceil(total_input_bytes / target_bucket_bytes)), + max(self.num_workers, math.ceil(total_input_bytes / target_bucket_bytes)), ) bucket_dir = os.path.join( self.project_root, "data", self.target_dir, "fragment-buckets" @@ -2566,7 +2560,7 @@ def _process_fragmented_folder( split: [] for split in range(self.n_splits) } worker_pool = ( - _create_preprocess_pool(self.n_cores) if self.n_cores > 1 else None + _create_preprocess_pool(self.num_workers) if self.num_workers > 1 else None ) try: for bucket, paths in sorted(bucket_files.items()): @@ -2588,7 +2582,7 @@ def _process_fragmented_folder( bucket_name, data, schema, - self.n_cores, + self.num_workers, layout, window_stride, data_columns, @@ -2666,7 +2660,7 @@ def _process_batches_multiple_files( selected_columns: Optional[list[str]], max_rows: Optional[int], schema: Any, - n_cores: int, + num_workers: int, layout: StoredWindowLayout | list[StoredWindowLayout], window_stride: int, data_columns: list[str], @@ -2701,7 +2695,7 @@ def _process_batches_multiple_files( selected_columns=selected_columns, max_rows=max_rows, schema=schema, - n_cores=n_cores, + num_workers=num_workers, layout=layout, window_stride=window_stride, data_columns=data_columns, @@ -2740,7 +2734,7 @@ def _process_batches_multiple_files( ) else: assert process_by_file is False - worker_count = min(n_cores, len(file_paths)) + worker_count = min(num_workers, len(file_paths)) file_sizes = [os.path.getsize(path) for path in file_paths] file_sets = [] next_file = 0 @@ -2776,7 +2770,7 @@ def _process_batches_multiple_files( "selected_columns": selected_columns, "max_rows": max_rows, "schema": schema, - "n_cores": 1, + "num_workers": 1, "layout": layout, "window_stride": window_stride, "data_columns": data_columns, @@ -2811,7 +2805,7 @@ def _process_batches_multiple_files( + list(kwargs_2.values()) for process_id, file_set in enumerate(file_sets) ] - logger.info(f"_process_batches_multiple_files n_cores: {n_cores}") + logger.info(f"_process_batches_multiple_files num_workers: {num_workers}") logger.info(f"_process_batches_multiple_files {len(job_params) = }") with _create_preprocess_pool(len(job_params)) as pool: @@ -3693,7 +3687,7 @@ def _get_column_statistics( if data_col not in precomputed_id_maps: cardinality = (cardinality_config or {}).get(data_col) if cardinality is not None: - if "min_freq" in cardinality or "top_k" in cardinality: + if "min_count" in cardinality or "max_categories" in cardinality: if categorical_value_counts is None: raise ValueError( "categorical_value_counts is required when fitting " @@ -4043,7 +4037,7 @@ def _process_batches_multiple_files_inner( selected_columns: Optional[list[str]], max_rows: Optional[int], schema: Any, - n_cores: int, + num_workers: int, layout: StoredWindowLayout | list[StoredWindowLayout], window_stride: int, data_columns: list[str], @@ -4085,7 +4079,7 @@ def _process_batches_multiple_files_inner( raise ValueError("No files found to process.") pad_width = len(str(n_files - 1)) n_rows_running_count = 0 - worker_pool = _create_preprocess_pool(n_cores) if n_cores > 1 else None + worker_pool = _create_preprocess_pool(num_workers) if num_workers > 1 else None for file_index, path in enumerate(file_paths): max_rows_inner = None if max_rows is None else max_rows - n_rows_running_count if max_rows_inner is None or max_rows_inner > 0: @@ -4160,7 +4154,7 @@ def _process_batches_multiple_files_inner( data_name_root_inner, data, schema, - n_cores, + num_workers, layout, window_stride, data_columns, @@ -4218,7 +4212,7 @@ def _process_batches_single_file( data_name_root: str, data: pl.DataFrame, schema: Any, - n_cores: Optional[int], + num_workers: Optional[int], layout: StoredWindowLayout | list[StoredWindowLayout], window_stride: int, data_columns: list[str], @@ -4247,26 +4241,26 @@ def _process_batches_single_file( raise ValueError( "split_column values must be non-decreasing within each sequence" ) - n_cores = n_cores or multiprocessing.cpu_count() + num_workers = num_workers or multiprocessing.cpu_count() sequence_count = len(_sequence_run_bounds(data)[0]) maximum_batches = data.height if allow_sequence_splitting else sequence_count - n_cores = min(n_cores, maximum_batches) + num_workers = min(num_workers, maximum_batches) if ( not normalize_on_all_data and allow_sequence_splitting and split_method == "within_sequence" - and n_cores > sequence_count + and num_workers > sequence_count ): raise ValueError( "Unsafe preprocessing configuration: normalize_on_all_data=False, " "allow_sequence_splitting=True, and split_method='within_sequence' " "would split sequences across worker batches " - f"(effective n_cores={n_cores}, sequences={sequence_count}). This " + f"(effective num_workers={num_workers}, sequences={sequence_count}). This " "would make split-0 metadata fitting disagree with the materialized " - "training split. Set allow_sequence_splitting=False, reduce n_cores " + "training split. Set allow_sequence_splitting=False, reduce num_workers " f"to at most {sequence_count}, or set normalize_on_all_data=True." ) - batch_limits = get_batch_limits(data, n_cores, allow_sequence_splitting) + batch_limits = get_batch_limits(data, num_workers, allow_sequence_splitting) valid_batch_limits = [(s, e) for s, e in batch_limits if (e - s) > 0] batches = [ ( diff --git a/src/sequifier/rl/attachment.py b/src/sequifier/rl/attachment.py index 7cf5d527..b39308d4 100644 --- a/src/sequifier/rl/attachment.py +++ b/src/sequifier/rl/attachment.py @@ -12,7 +12,9 @@ def resolve_binding(execution, interface_name, binding): ) if ( -binding.decision_position - > execution.global_training.context_length - interface.decoder.support + 1 + > execution.global_training.context_length + - interface.decoder.support_length + + 1 ): raise ValueError("decision_position lies outside the source decoder output") values = binding.model_dump() diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index 998aa053..dbf18bbf 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -68,11 +68,11 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): raise ValueError("DPO requires a frozen reference actor") session = PhaseSession(context) state = session.state - if not 0 <= session.transaction <= phase.updates: + if not 0 <= session.transaction <= phase.iterations: raise ValueError("DPO checkpoint exceeds authored update budget") batches = BatchOrder( len(data.training), - phase.batch_size * phase.accumulation_steps, + phase.batch_size * phase.accumulation_batches, context.run.seed + context.index, state=state["batch_order"] if state else None, ) @@ -81,7 +81,7 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): or session.steps["actor"] != session.transaction ): raise ValueError("DPO sampler position disagrees with optimizer transaction") - for update in range(session.transaction + 1, phase.updates + 1): + for update in range(session.transaction + 1, phase.iterations + 1): group = batches.next() session.zero_grad() loss_total = 0.0 @@ -107,7 +107,7 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): (sign * coefficient * chunk.sum()).backward() session.step() session.transaction = update - final = update == phase.updates + final = update == phase.iterations if session.should_evaluate(final): held_out = evaluate_dpo(actor, reference, data.validation, phase) session.record_evaluation( diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 6233d0c1..9d305396 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -32,10 +32,12 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): if ( not 0 <= groups <= phase.total_groups or session.transaction - != (groups + phase.groups_per_update - 1) // phase.groups_per_update + != (groups + phase.groups_per_iteration - 1) // phase.groups_per_iteration ): raise ValueError("GRPO checkpoint is not a complete update/group boundary") - if groups != min(session.transaction * phase.groups_per_update, phase.total_groups): + if groups != min( + session.transaction * phase.groups_per_iteration, phase.total_groups + ): raise ValueError("GRPO checkpoint group counter is not a complete iteration") if ( not groups * phase.group_size @@ -78,7 +80,7 @@ def evaluate(): ) while groups < phase.total_groups: - count = min(phase.groups_per_update, phase.total_groups - groups) + count = min(phase.groups_per_iteration, phase.total_groups - groups) buffer, executed = collect_groups( context, adapter, diff --git a/src/sequifier/rl/grpo_update.py b/src/sequifier/rl/grpo_update.py index 9a699fdb..81e7be09 100644 --- a/src/sequifier/rl/grpo_update.py +++ b/src/sequifier/rl/grpo_update.py @@ -14,7 +14,7 @@ def grpo_objective( ) reference_delta = reference_log_probability.float() - log_probability.float() penalty = torch.expm1(reference_delta) - reference_delta - loss = policy.loss + phase.kl_coefficient * penalty + loss = policy.loss + phase.reference_kl_coefficient * penalty old_kl = policy.approximate_kl if any(not torch.isfinite(t).all() for t in (policy.ratio, penalty, loss, old_kl)): raise ValueError( @@ -54,7 +54,7 @@ def update_grpo(session, buffer): "clip_fraction", ) scored, stopped = 0, False - size = phase.batch_size * phase.accumulation_steps + size = phase.batch_size * phase.accumulation_batches epoch = -1 for epoch in range(phase.optimization_epochs): for indices in shuffled_groups(len(trajectories), size, session.sampler): diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index c9035139..487e5aaf 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -26,8 +26,8 @@ def infer_policy(config: ResolvedPolicyInferenceConfig) -> None: configure_determinism(config.seed, config.enforce_determinism) policy = load_policy( - config._artifacts.get(config.model_path) - or normalize_path(config.model_path, config.project_root), + config._artifacts.get(config.model_paths[0]) + or normalize_path(config.model_paths[0], config.project_root), device=config.device, interface_name=config.model_interface, ) @@ -57,7 +57,7 @@ def infer_policy(config: ResolvedPolicyInferenceConfig) -> None: config.project_root, config.write_format, "predictions", - Path(config.model_path).stem, + Path(config.model_paths[0]).stem, 0, ) else: @@ -107,7 +107,7 @@ def infer_rollout(policy, data, config): ) trajectory.metadata.update( { - "model_path": config.model_path, + "model_path": config.model_paths[0], "model_interface": config.model_interface, "seed_row_index": rollout.seed_row_index, "enforce_determinism": config.enforce_determinism, diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 245b06c2..c50dfff6 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -79,11 +79,11 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): modules={"value": value, "critic": torch.nn.ModuleList([q1, q2])}, frozen={"target_q1": targets[0], "target_q2": targets[1]}, ) - if not 0 <= session.transaction <= phase.updates: + if not 0 <= session.transaction <= phase.iterations: raise ValueError("IQL checkpoint exceeds phase update budget") if any(steps != session.transaction for steps in session.steps.values()): raise ValueError("IQL optimizer clocks disagree with committed updates") - for update in range(session.transaction + 1, phase.updates + 1): + for update in range(session.transaction + 1, phase.iterations + 1): indices = torch.randint( len(data[0]), (phase.batch_size,), generator=session.sampler ) @@ -99,7 +99,7 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): advantage = qbar - old_v weight = torch.exp( (phase.inverse_temperature * advantage).clamp( - max=math.log(phase.max_weight) + max=math.log(phase.max_advantage_weight) ) ) successor_value = torch.zeros_like(old_v) @@ -130,7 +130,7 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): _polyak(targets[0], q1, phase.target_update_rate) _polyak(targets[1], q2, phase.target_update_rate) session.transaction = update - final = update == phase.updates + final = update == phase.iterations if session.should_evaluate(final): held_out = _evaluate( actor, @@ -151,7 +151,9 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): **{f"validation_{key}": val for key, val in held_out.items()}, advantage_mean=float(advantage.mean()), weight_mean=float(weight.mean()), - weight_cap_fraction=float((weight >= phase.max_weight).float().mean()), + weight_cap_fraction=float( + (weight >= phase.max_advantage_weight).float().mean() + ), ) session.checkpoint(final) return session.finish( diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 39f2afcb..d660055c 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -41,7 +41,7 @@ def _plan_contract(config): for role in phase["optimization"].values(): if role["scheduler"]["name"] == "OneCycleLR": # total_steps is an assertion, not a separately configurable budget. - role["scheduler"].pop("total_steps", None) + role["scheduler"]["arguments"].pop("total_steps", None) return values diff --git a/src/sequifier/rl/models.py b/src/sequifier/rl/models.py index 9b61e913..f11c3d52 100644 --- a/src/sequifier/rl/models.py +++ b/src/sequifier/rl/models.py @@ -68,7 +68,7 @@ def __init__( interface_name ).decoder.parameters(): parameter.requires_grad_(False) - width = self.network.dim_model + width = self.network.model_dim self.action_embedding = ( ActionEncoder( action_count, diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index 7de83d60..5eee4de6 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -29,7 +29,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): continuations = list(state.get("continuations", [])) if ( env_steps - != min(session.transaction * phase.rollout_length, phase.total_env_steps) + != min(session.transaction * phase.rollout_env_steps, phase.total_env_steps) or not 0 <= env_steps <= phase.total_env_steps ): raise ValueError("PPO checkpoint is not a complete collection/update boundary") @@ -65,7 +65,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): ) session.restore_random() while env_steps < phase.total_env_steps: - horizon = min(phase.rollout_length, phase.total_env_steps - env_steps) + horizon = min(phase.rollout_env_steps, phase.total_env_steps - env_steps) buffer, episodes = collector.collect(horizon, session.transaction) if phase.archive_directory: buffer.archive( diff --git a/src/sequifier/rl/ppo_update.py b/src/sequifier/rl/ppo_update.py index cf1c0279..7d9a3d28 100644 --- a/src/sequifier/rl/ppo_update.py +++ b/src/sequifier/rl/ppo_update.py @@ -23,9 +23,9 @@ def ppo_objectives( ) policy_loss, ratio = policy.loss, policy.ratio value_error = (prediction.float() - target).square() - if phase.value_clip is not None: + if phase.value_clip_delta is not None: clipped_value = old_value + (prediction.float() - old_value).clamp( - -phase.value_clip, phase.value_clip + -phase.value_clip_delta, phase.value_clip_delta ) value_error = torch.maximum(value_error, (clipped_value - target).square()) value_loss = 0.5 * value_error @@ -74,7 +74,7 @@ def update_ppo(session, value, buffer): ) actor.eval() # Keep dropout off while retaining autograd. value.eval() - validate_collection_scores(actor, value, buffer, phase.minibatch_size) + validate_collection_scores(actor, value, buffer, phase.batch_size) totals = MetricTotals( "policy_loss", "value_loss", "entropy", "approximate_kl", "clip_fraction" ) @@ -82,12 +82,12 @@ def update_ppo(session, value, buffer): count = 0 norms = {"actor": [], "value": []} stopped = False - group_size = phase.minibatch_size * phase.accumulation_steps + group_size = phase.batch_size * phase.accumulation_batches epochs = 0 for epoch in range(phase.optimization_epochs): for group in shuffled_groups(len(buffer), group_size, session.sampler): - minibatches = list(batches(group, phase.minibatch_size)) - if phase.target_kl is not None: + minibatches = list(batches(group, phase.batch_size)) + if phase.old_policy_target_kl is not None: kl_sum = 0.0 with torch.no_grad(): for indices in minibatches: @@ -99,7 +99,7 @@ def update_ppo(session, value, buffer): if not torch.isfinite(kl).all(): raise ValueError("non-finite PPO KL") kl_sum += float(kl.sum()) - if kl_sum / len(group) > phase.target_kl: + if kl_sum / len(group) > phase.old_policy_target_kl: stopped = True break session.zero_grad() @@ -142,9 +142,9 @@ def update_ppo(session, value, buffer): targets = torch.tensor(buffer.targets, dtype=torch.float32) with torch.no_grad(): predictions = [] - for start in range(0, len(buffer), phase.minibatch_size): + for start in range(0, len(buffer), phase.batch_size): history, *_ = buffer.batch( - range(start, min(start + phase.minibatch_size, len(buffer))), + range(start, min(start + phase.batch_size, len(buffer))), actor.device, ) predictions.append(value(history).cpu()) diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 8fc3c510..cd2a0e35 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -161,7 +161,7 @@ def preflight_optimization(config, actor, data): """Resolve constructors and authored scheduler budgets before creating outputs.""" budgets = [] for phase, part in zip(config.phases, data): - for path in phase.freeze: + for path in phase.freeze_modules: actor.network.get_submodule(path) size = len(part.training) if part is not None else None budget = phase.optimizer_step_budget(size) diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 6b74e2b7..c69c4e60 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -223,7 +223,7 @@ def __init__(self, context, *, modules=None, frozen=None): } for name, parameter in actor.network.named_parameters(): parameter.requires_grad_(actor._phase_trainability[name]) - for path in context.phase.freeze: + for path in context.phase.freeze_modules: actor.network.get_submodule(path).requires_grad_(False) self.modules = {"actor": context.actor.network, **(modules or {})} self.networks = {**self.modules, **(frozen or {})} @@ -276,7 +276,7 @@ def __init__(self, context, *, modules=None, frozen=None): selection = context.phase.evaluation.monitor monitor = selection.metric self.selection = Selection( - context.phase, monitor, selection.mode, self.state.get("selection") + context.phase, monitor, selection.direction, self.state.get("selection") ) if self.state: require_equal( @@ -321,7 +321,7 @@ def step(self): role: float( torch.nn.utils.clip_grad_norm_( module.parameters(), - self.configs[role].gradient_clip or float("inf"), + self.configs[role].gradient_clip_norm or float("inf"), error_if_nonfinite=True, ) ) @@ -341,7 +341,10 @@ def step(self): return norms def should_evaluate(self, final): - return final or self.transaction % self.context.phase.evaluation_interval == 0 + return ( + final + or self.transaction % self.context.phase.evaluation_interval_iterations == 0 + ) def observe(self, metric): self.selection.observe(self.context.actor, metric, self.transaction) @@ -361,7 +364,10 @@ def optimization_state(self, *, include_steps=False): } def should_checkpoint(self, final): - return final or self.transaction % self.context.phase.checkpoint_interval == 0 + return ( + final + or self.transaction % self.context.phase.checkpoint_interval_iterations == 0 + ) def checkpoint(self, final, **extra): if not self.should_checkpoint(final): diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 0d831e5d..062e4989 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -39,7 +39,7 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin training = data.training batches = BatchOrder( len(training), - phase.batch_size * phase.accumulation_steps, + phase.batch_size * phase.accumulation_batches, context.run.seed + context.index, sample=training.sampler, state=state["batch_order"] if state else None, diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index f67da761..1fc1adec 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -73,8 +73,10 @@ def start_phase(self, phase_index: int) -> None: if self.config.training_plan[phase_index].optimizer_state == "reset": self.optimization = self.optimization_factory(phase_index) self.optimization.gradient_policy = GradientPolicy( - self.config.training_plan[phase_index].accumulation_steps, - self.config.training_plan[phase_index].optimization["model"].gradient_clip, + self.config.training_plan[phase_index].accumulation_batches, + self.config.training_plan[phase_index] + .optimization["model"] + .gradient_clip_norm, ) self.optimization.optimizer_step = optimizer_step self.optimization_phase_index = phase_index @@ -251,7 +253,7 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: str(execution.device), selected_parameters, parameter_catalog=catalog, - accumulation_steps=phase.accumulation_steps, + accumulation_batches=phase.accumulation_batches, layer_type_dtypes=config.global_training.layer_type_dtypes, ) diff --git a/src/sequifier/sequifier.py b/src/sequifier/sequifier.py index b5445c76..6d59f48f 100644 --- a/src/sequifier/sequifier.py +++ b/src/sequifier/sequifier.py @@ -1,7 +1,6 @@ -from argparse import ArgumentParser -from typing import Any +from argparse import ArgumentParser, ArgumentTypeError +from typing import Any, NoReturn -import numpy as np import yaml from sequifier.docs import docs @@ -14,6 +13,53 @@ from sequifier.visualize_training import visualize_training +class ConfigArgumentParser(ArgumentParser): + """Diagnose removed flags without accepting them as aliases.""" + + def __init__(self, *args: Any, **kwargs: Any): + kwargs.setdefault("allow_abbrev", False) + super().__init__(*args, **kwargs) + + def error(self, message: str) -> NoReturn: + replacements = { + "--model-type": "--task forecast|policy|embedding", + "--model-path": "--model-paths", + "--randomize": "--seed random", + "--data-path": "--preprocessing-data-path for preprocessing", + "-dp": "--preprocessing-data-path (preprocess) or --data-path (infer)", + "-mp": "--model-paths", + "-ic": "--input-columns", + "-mc": "--metadata-config-path", + "-sc": "--selected-columns", + "-mn": "--model-name", + "-sm": "--skip-metadata", + "-s": "--seed", + "-r": "--seed random", + } + prefix = "unrecognized arguments: " + if message.startswith(prefix): + for token in message[len(prefix) :].split(): + flag = token.partition("=")[0] + if flag in replacements: + message += f"\n{flag} was removed; use {replacements[flag]}" + super().error(message) + + +def parse_seed(value: str) -> int | str: + if value == "random": + return value + try: + return int(value) + except ValueError as error: + raise ArgumentTypeError("seed must be an integer or random") from error + + +def parse_boolean(value: str) -> bool: + if value not in {"true", "false"}: + raise ArgumentTypeError("boolean overrides must be true or false") + return value == "true" + + @beartype def build_args_config(args: Any) -> dict[str, Any]: """Build config overrides from parsed CLI args.""" @@ -21,15 +67,8 @@ def build_args_config(args: Any) -> dict[str, Any]: args_config = { k: v for k, v in vars(args).items() - if v is not None and k not in ["randomize", "command", "config_path"] + if v is not None and k not in ["command", "config_path"] } - if args.command not in ["make", "visualize-training"]: - if args.randomize: - seed = np.random.choice(np.arange(int(1e9))) - args_config["seed"] = seed - elif args.command in ["train", "infer"] and args.seed: - args_config["seed"] = args.seed - if "selected_columns" in args_config: if ( len(args_config["selected_columns"]) == 1 @@ -50,7 +89,7 @@ def build_args_config(args: Any) -> dict[str, Any]: @beartype def setup_parser() -> ArgumentParser: """Build the sequifier CLI parser.""" - parser = ArgumentParser() + parser = ConfigArgumentParser() subparsers = parser.add_subparsers(dest="command", help="Available commands") parser_make = subparsers.add_parser("make", help="Set up sequifier project") @@ -90,51 +129,42 @@ def setup_parser() -> ArgumentParser: ) for subparser in [parser_preprocess, parser_train, parser_infer]: - subparser.add_argument("-r", "--randomize", action="store_true") + subparser.add_argument( + "--seed", type=parse_seed, help="Integer run seed, or random" + ) parser_preprocess.add_argument( - "-dp", - "--data-path", - dest="preprocessing_data_path", + "--preprocessing-data-path", type=str, ) - parser_infer.add_argument("-dp", "--data-path", type=str) + parser_infer.add_argument("--data-path", type=str) parser_infer.add_argument( - "-ic", "--input-columns", type=str, nargs="+", help="Space-separated list of input columns (e.g. --input-columns col1 col2)", ) - parser_infer.add_argument("-mc", "--metadata-config-path", type=str) + parser_infer.add_argument("--metadata-config-path", type=str) for subparser in [parser_train, parser_infer, parser_hyperparameter_search]: - subparser.add_argument("-sm", "--skip-metadata", action="store_true") + subparser.add_argument("--skip-metadata", action="store_true") parser_preprocess.add_argument( - "-sc", "--selected-columns", type=str, nargs="+", help="Space-separated list of selected columns", ) - parser_train.add_argument("-mn", "--model-name", type=str) - parser_train.add_argument("-s", "--seed", type=int) + parser_train.add_argument("--model-name", type=str) parser_train.add_argument("--global-training", type=yaml.safe_load) parser_train.add_argument("--training-plan", type=yaml.safe_load) - parser_infer.add_argument("-mp", "--model-path", type=str) - parser_infer.add_argument("-s", "--seed", type=int) - parser_infer.add_argument( - "--model-type", choices=("generative", "embedding", "policy") - ) - parser_infer.add_argument( - "--enforce-determinism", action="store_true", default=None - ) + parser_infer.add_argument("--model-paths", type=str, nargs="+") + parser_infer.add_argument("--enforce-determinism", type=parse_boolean, default=None) parser_infer.add_argument( "--action-selection", choices=("deterministic", "stochastic") ) - parser_infer.add_argument("--task", choices=("policy", "forecast")) + parser_infer.add_argument("--task", choices=("forecast", "policy", "embedding")) parser_infer.add_argument("--dataset", type=str) parser_infer.add_argument("--part", type=str) parser_infer.add_argument("--model-interface", type=str) diff --git a/src/sequifier/templates/infer-policy.yaml b/src/sequifier/templates/infer-policy.yaml index f14b468a..6c6663ab 100644 --- a/src/sequifier/templates/infer-policy.yaml +++ b/src/sequifier/templates/infer-policy.yaml @@ -1,10 +1,11 @@ -project_root: . -model_path: models/policy/model.pt +project_root: "." data_path: PLEASE_FILL -model_type: generative read_format: csv write_format: csv device: cpu desired_return: 10.0 enforce_determinism: true action_selection: deterministic +task: policy +model_paths: +- models/policy/model.pt diff --git a/src/sequifier/templates/train-dpo.yaml b/src/sequifier/templates/train-dpo.yaml index 3ff24d48..7e942620 100644 --- a/src/sequifier/templates/train-dpo.yaml +++ b/src/sequifier/templates/train-dpo.yaml @@ -1,58 +1,82 @@ -project_root: . +project_root: "." model_name: dpo-control device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 1 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 64 + position_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [cue, __policy_previous_action] - target_columns: [action] + input_columns: + - cue + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + cue: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 4 - normalization: {source: identity} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [left, right]} + normalization: + source: identity dataset_training: decisions: parts: default: type: sequence - paths: [data/control-decisions.csv] + paths: + - data/control-decisions.csv columns: episode_id: episodeId reward: reward terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: {cue: next_cue} + successors: + cue: next_cue group: contextId model_interface: actor rankings: parts: - default: {type: preferences, manifest: data/control-preferences.json} + default: + type: preferences + manifest_path: data/control-preferences.json training_plan: phases: - - name: preferences - algorithm: dpo - preferences: {source: rankings.default} - updates: 100 - beta: 0.1 + - name: preferences + algorithm: dpo + preferences: + source: rankings.default + beta: 0.1 + iterations: 100 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right diff --git a/src/sequifier/templates/train-grpo.yaml b/src/sequifier/templates/train-grpo.yaml index 7126cc5a..9b459f2f 100644 --- a/src/sequifier/templates/train-grpo.yaml +++ b/src/sequifier/templates/train-grpo.yaml @@ -1,54 +1,81 @@ -project_root: . +project_root: "." model_name: grpo-control device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 1 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 64 + position_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [cue, __policy_previous_action] - target_columns: [action] + input_columns: + - cue + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + cue: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 4 - normalization: {source: identity} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [left, right]} + normalization: + source: identity training_plan: phases: - - name: group_improvement - algorithm: grpo - total_groups: 32 - groups_per_update: 2 - group_size: 4 - max_steps: 8 + - name: group_improvement + algorithm: grpo + total_groups: 32 + groups_per_iteration: 2 + group_size: 4 + max_steps: 8 + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 + evaluation: environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 - evaluation: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 - monitor: {metric: score_mean, mode: max} + seeds: + - 17 + - 29 + - 43 + - 71 + max_episode_steps: 8 + timeout_seconds: 30 + monitor: + metric: score_mean + direction: max +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right diff --git a/src/sequifier/templates/train-iql.yaml b/src/sequifier/templates/train-iql.yaml index bace396a..3eeadd7a 100644 --- a/src/sequifier/templates/train-iql.yaml +++ b/src/sequifier/templates/train-iql.yaml @@ -1,54 +1,75 @@ -project_root: . +project_root: "." model_name: offline-iql device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 2 max_context_length: 16 - attention: {n_heads: 4} - feed_forward: {dim: 128} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 128 + position_encoding_scope: global interfaces: actor: - input_columns: [OBSERVATION_COLUMN, __policy_previous_action] - target_columns: [action] + input_columns: + - OBSERVATION_COLUMN + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + OBSERVATION_COLUMN: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 16 validation_fraction: 0.2 - resume: {policy: if_exists} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [0.0, 1.0]} + resume: + policy: if_exists dataset_training: demonstrations: parts: recorded: type: sequence - paths: [PLEASE_FILL.csv] + paths: + - PLEASE_FILL.csv columns: reward: reward terminated: terminated truncated: truncated bootstrap_allowed: bootstrap_allowed - successors: {OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN} + successors: + OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN model_interface: actor training_plan: phases: - - name: offline_improvement - algorithm: iql - sources: - - {source: demonstrations.recorded} - updates: 10000 - batch_size: 64 + - name: offline_improvement + algorithm: iql + sources: + - source: demonstrations.recorded + batch_size: 64 + iterations: 10000 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - 0.0 + - 1.0 diff --git a/src/sequifier/templates/train-policy-from-supervised.yaml b/src/sequifier/templates/train-policy-from-supervised.yaml index 49d28caa..f72f805e 100644 --- a/src/sequifier/templates/train-policy-from-supervised.yaml +++ b/src/sequifier/templates/train-policy-from-supervised.yaml @@ -1,21 +1,35 @@ -project_root: . +project_root: "." model_name: posttrained device: cpu initialize_from: models/supervised/model.pt global_training: - resume: {policy: if_exists, mode: reset_environments} -policy_binding: - actions: - mode: {type: categorical, values: ['off', 'on']} - quantity: {type: real, scale: 0.5} + resume: + policy: if_exists + mode: reset_environments dataset_training: demonstrations: parts: - default: {type: sequence, paths: [data/demonstrations.parquet], format: table} + default: + type: sequence + paths: + - data/demonstrations.parquet + format: table training_plan: phases: - - name: imitation - algorithm: policy_imitation - sources: - - {source: demonstrations.default} - epochs: 5 + - name: imitation + algorithm: policy_imitation + sources: + - source: demonstrations.default + epochs: 5 +policy_bindings: + default: + actions: + mode: + type: categorical + values: + - 'off' + - 'on' + quantity: + type: real + scale: + fixed: 0.5 diff --git a/src/sequifier/templates/train-policy.yaml b/src/sequifier/templates/train-policy.yaml index a5445a54..ceccac2b 100644 --- a/src/sequifier/templates/train-policy.yaml +++ b/src/sequifier/templates/train-policy.yaml @@ -1,50 +1,76 @@ -project_root: . +project_root: "." model_name: policy device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 2 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + activation: swiglu + hidden_dim: 128 + position_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [OBSERVATION_COLUMN, __policy_previous_action, __policy_return] - target_columns: [action] + input_columns: + - OBSERVATION_COLUMN + - __policy_previous_action + - __policy_return + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + OBSERVATION_COLUMN: 1 + __policy_previous_action: 6 + __policy_return: 1 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 4 validation_fraction: 0.2 episode_cache_size: 8 - resume: {policy: never} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: true - actions: - action: {type: categorical, values: [0.0, 1.0]} + resume: + policy: never dataset_training: demonstrations: parts: - recorded: {type: sequence, paths: [PLEASE_FILL.csv]} + recorded: + type: sequence + paths: + - PLEASE_FILL.csv model_interface: actor training_plan: phases: - - algorithm: policy_imitation - epochs: 80 - sources: - - {source: demonstrations.recorded} - name: policy_imitation - batch_size: 64 - optimization: - actor: {learning_rate: 0.005} + - algorithm: policy_imitation + epochs: 80 + sources: + - source: demonstrations.recorded + name: policy_imitation + batch_size: 64 + optimization: + actor: + learning_rate: 0.005 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: true + actions: + action: + type: categorical + values: + - 0.0 + - 1.0 diff --git a/src/sequifier/templates/train-ppo.yaml b/src/sequifier/templates/train-ppo.yaml index d5ab5760..950df086 100644 --- a/src/sequifier/templates/train-ppo.yaml +++ b/src/sequifier/templates/train-ppo.yaml @@ -1,60 +1,87 @@ -project_root: . +project_root: "." model_name: online-ppo device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 1 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 64} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + hidden_dim: 64 + position_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [cue, __policy_previous_action] - target_columns: [action] + input_columns: + - cue + - __policy_previous_action + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {cue: 2, __policy_previous_action: 6} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + cue: 2 + __policy_previous_action: 6 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 4 - normalization: {source: identity} - resume: {policy: if_exists, mode: exact} -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: false - actions: - action: {type: categorical, values: [left, right]} + normalization: + source: identity + resume: + policy: if_exists + mode: exact training_plan: phases: - - name: online_control - algorithm: ppo + - name: online_control + algorithm: ppo + environment: + factory: sequifier.rl.control:CategoricalControlAdapter + config: + horizon: 8 + implementation_id: sequifier.categorical-control.v1 + total_env_steps: 8192 + rollout_env_steps: 256 + optimization_epochs: 4 + accumulation_batches: 1 + gamma: 0.99 + gae_lambda: 0.95 + clip_epsilon: 0.2 + evaluation: environment: factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} + config: + horizon: 8 implementation_id: sequifier.categorical-control.v1 - total_env_steps: 8192 - rollout_length: 256 - optimization_epochs: 4 - minibatch_size: 64 - accumulation_steps: 1 - gamma: 0.99 - gae_lambda: 0.95 - clip_epsilon: 0.2 - evaluation_interval: 2 - checkpoint_interval: 2 - evaluation: - environment: - factory: sequifier.rl.control:CategoricalControlAdapter - config: {horizon: 8} - implementation_id: sequifier.categorical-control.v1 - seeds: [17, 29, 43, 71] - max_episode_steps: 8 - timeout_seconds: 30 + seeds: + - 17 + - 29 + - 43 + - 71 + max_episode_steps: 8 + timeout_seconds: 30 + batch_size: 64 + evaluation_interval_iterations: 2 + checkpoint_interval_iterations: 2 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: false + actions: + action: + type: categorical + values: + - left + - right diff --git a/src/sequifier/templates/train-supervised-to-policy.yaml b/src/sequifier/templates/train-supervised-to-policy.yaml index ad4fea31..c8318128 100644 --- a/src/sequifier/templates/train-supervised-to-policy.yaml +++ b/src/sequifier/templates/train-supervised-to-policy.yaml @@ -1,74 +1,106 @@ -project_root: . +project_root: "." model_name: supervised-to-policy device: cpu seed: 1010 model: backbone: architecture: - dim_model: 32 + model_dim: 32 num_layers: 2 max_context_length: 4 - attention: {n_heads: 4} - feed_forward: {dim: 128, activation: swiglu} - positional_encoding_scope: global + attention: + num_heads: 4 + feed_forward: + activation: swiglu + hidden_dim: 128 + position_encoding_scope: global dropout: 0.0 interfaces: actor: - input_columns: [OBSERVATION_COLUMN, __policy_previous_action, __policy_return] - target_columns: [action] + input_columns: + - OBSERVATION_COLUMN + - __policy_previous_action + - __policy_return + target_columns: + - action ingestion: type: embedding output_dim: 8 - feature_embedding_dims: {OBSERVATION_COLUMN: 1, __policy_previous_action: 6, __policy_return: 1} - decoder: {type: linear, prediction_length: 1, support: 1} - categorical_decoder_special_tokens: {action: []} + feature_embedding_dims: + OBSERVATION_COLUMN: 1 + __policy_previous_action: 6 + __policy_return: 1 + decoder: + type: linear + prediction_length: 1 + support_length: 1 + categorical_decoder_special_tokens: + action: [] global_training: context_length: 4 validation_fraction: 0.2 episode_cache_size: 8 - resume: {policy: never} + resume: + policy: never training_objective: causal torch_compile: none -policy_binding: - previous_action_inputs: {__policy_previous_action: action} - initial_inputs: {__policy_previous_action: start} - return_conditioned: true - actions: - action: {type: categorical, values: [0.0, 1.0]} dataset_training: demonstrations: parts: - recorded: {type: sequence, paths: [PLEASE_FILL.csv]} + recorded: + type: sequence + paths: + - PLEASE_FILL.csv model_interface: actor observations: model_interface: actor parts: - train: {metadata_config_path: configs/metadata_configs/observations.json} - criterion: {action: CrossEntropyLoss} + train: + metadata_config_path: configs/metadata_configs/observations.json + criterion: + action: CrossEntropyLoss training_plan: phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential - sources: - - {source: observations.train} - batch_size: 64 - accumulation_steps: 1 - optimization: - model: - optimizer: {name: Adam} - learning_rate: 0.001 - scheduler: - name: StepLR - arguments: {step_size: 1, gamma: 0.99} - scheduler_step_on: epoch - gradient_clip: 1.0 - - algorithm: policy_imitation - epochs: 80 - sources: - - {source: demonstrations.recorded} - name: policy_imitation - batch_size: 64 - optimization: - actor: {learning_rate: 0.005} + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - source: observations.train + batch_size: 64 + accumulation_batches: 1 + optimization: + model: + optimizer: + name: Adam + arguments: {} + learning_rate: 0.001 + scheduler: + name: StepLR + arguments: + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip_norm: 1.0 + - algorithm: policy_imitation + epochs: 80 + sources: + - source: demonstrations.recorded + name: policy_imitation + batch_size: 64 + optimization: + actor: + learning_rate: 0.005 +policy_bindings: + actor: + previous_action_inputs: + __policy_previous_action: action + initial_inputs: + __policy_previous_action: start + return_conditioned: true + actions: + action: + type: categorical + values: + - 0.0 + - 1.0 diff --git a/src/sequifier/templates/train-supervised.yaml b/src/sequifier/templates/train-supervised.yaml index 5fdc61a0..af8591d7 100644 --- a/src/sequifier/templates/train-supervised.yaml +++ b/src/sequifier/templates/train-supervised.yaml @@ -1,41 +1,63 @@ -project_root: . +project_root: "." model_name: supervised device: cpu -global_training: {training_objective: causal, context_length: 48, inference_batch_size: 10} +global_training: + training_objective: causal + context_length: 48 + inference_batch_size: 10 model: backbone: architecture: - dim_model: 128 + model_dim: 128 num_layers: 3 - attention: {n_heads: 16} - feed_forward: {dim: 128} + attention: + num_heads: 16 + feed_forward: + hidden_dim: 128 dropout: 0.2 - interface: - input_columns: [EXAMPLE_INPUT_COLUMN_NAME] - target_columns: [EXAMPLE_TARGET_COLUMN_NAME] - ingestion: {type: embedding, output_dim: 128} - decoder: {type: linear, prediction_length: 1} -dataset: - part: {metadata_config_path: PLEASE FILL} - criterion: {EXAMPLE_TARGET_COLUMN_NAME: MSELoss} + interfaces: + default: + input_columns: + - EXAMPLE_INPUT_COLUMN_NAME + target_columns: + - EXAMPLE_TARGET_COLUMN_NAME + ingestion: + type: embedding + output_dim: 128 + decoder: + type: linear + prediction_length: 1 training_plan: phases: - - name: supervised - algorithm: supervised - epochs: 10 - mode: sequential + - name: supervised + algorithm: supervised + epochs: 10 + mode: sequential + sources: + - source: default.default + batch_size: 10 + accumulation_batches: 1 + optimization: + model: + learning_rate: 0.0001 + optimizer: + name: AdamW + arguments: {} + scheduler: + name: StepLR + arguments: + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + evaluation: sources: - - {source: default.default} - batch_size: 10 - accumulation_steps: 1 - optimization: - model: - learning_rate: 0.0001 - optimizer: {name: AdamW} - scheduler: - name: StepLR - arguments: {step_size: 1, gamma: 0.99} - scheduler_step_on: epoch -evaluation: true + - source: default.default export_generative_model: true export_embedding_model: false +dataset_training: + default: + criterion: + EXAMPLE_TARGET_COLUMN_NAME: MSELoss + parts: + default: + metadata_config_path: PLEASE FILL diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index 515689e7..02d75504 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -111,7 +111,7 @@ def strip_provenance(value: Any) -> Any: for role in phase["optimization"].values(): if role["scheduler"]["name"] == "OneCycleLR": # Recomputed from the phase/data budget before optimizer construction. - role["scheduler"].pop("total_steps", None) + role["scheduler"]["arguments"].pop("total_steps", None) return { "model": execution, "initialization_provenance": { diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 5a1b685f..3e1968be 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -88,7 +88,7 @@ def _identity( global_batch_step=run.state.global_batch_step + 1, optimizer_step=run.optimization.optimizer_step, accumulation_index=accumulation_index, - accumulation_steps=run.optimization.gradient_policy.accumulation_steps, + accumulation_batches=run.optimization.gradient_policy.accumulation_batches, rank=run.distributed.rank, world_size=run.distributed.world_size, ) @@ -170,11 +170,13 @@ def _update_monitor( if monitor is None or monitor.source not in result.sources: return float("nan"), False value = result.sources[monitor.source].total_loss - if monitor.mode == "max" and run.state.best_validation_loss == float("inf"): + if monitor.direction == "max" and run.state.best_validation_loss == float( + "inf" + ): run.state.best_validation_loss = float("-inf") improved = ( value < run.state.best_validation_loss - if monitor.mode == "min" + if monitor.direction == "min" else value > run.state.best_validation_loss ) if improved: @@ -232,8 +234,8 @@ def run(self, run: Any) -> RunResult: config = run.config state = run.state time_checkpoint_intervals_enabled = bool( - config.global_training.save_latest_interval_minutes is not None - or config.global_training.save_interval_minutes is not None + config.global_training.latest_checkpoint_interval_minutes is not None + or config.global_training.snapshot_interval_minutes is not None ) defer_ddp_gradient_sync = bool( isinstance( @@ -241,7 +243,7 @@ def run(self, run: Any) -> RunResult: torch.nn.parallel.DistributedDataParallel, ) and len(run.datasets) == 1 - and run.optimization.gradient_policy.accumulation_steps > 1 + and run.optimization.gradient_policy.accumulation_batches > 1 and not time_checkpoint_intervals_enabled ) accumulation_count = 0 @@ -332,7 +334,7 @@ def flush_metrics( if ( state.global_batch_step == 0 and config.training_plan[0].evaluation_sources - and config.global_training.calculate_validation_loss_on_initialization + and config.global_training.evaluate_before_training ): first_sources = [ build_source_runtime(source, run.datasets) @@ -450,17 +452,17 @@ def flush_metrics( ) if run.integrations.enabled: run.integrations.emit(prepared_event) - save_interval_batches = ( - config.global_training.save_interval_batches + snapshot_interval_batches = ( + config.global_training.snapshot_interval_batches ) batch_snapshot_due = bool( - save_interval_batches is not None + snapshot_interval_batches is not None and identity.global_batch_step - last_snapshot_step - >= save_interval_batches + >= snapshot_interval_batches ) gradient_sync_due = bool( accumulation_count + 1 - >= run.optimization.gradient_policy.accumulation_steps + >= run.optimization.gradient_policy.accumulation_batches or current_batch >= current_batches_total or batch_snapshot_due ) @@ -548,7 +550,10 @@ def flush_metrics( window["seconds"] += time.perf_counter() - started if window["part"] != runtime_batch.part: window["part"] = None - if window["batches"] >= config.global_training.log_interval: + if ( + window["batches"] + >= config.global_training.log_interval_batches + ): flush_metrics( dataset.name, current_epoch, @@ -557,24 +562,25 @@ def flush_metrics( ) if ( accumulation_count - >= run.optimization.gradient_policy.accumulation_steps + >= run.optimization.gradient_policy.accumulation_batches ): flush() now = time.monotonic() snapshot_due = bool( - config.global_training.save_interval_batches is not None + config.global_training.snapshot_interval_batches is not None and state.global_batch_step - last_snapshot_step - >= config.global_training.save_interval_batches + >= config.global_training.snapshot_interval_batches ) or bool( - config.global_training.save_interval_minutes is not None + config.global_training.snapshot_interval_minutes is not None and now - last_snapshot - >= config.global_training.save_interval_minutes * 60 + >= config.global_training.snapshot_interval_minutes * 60 ) latest_due = bool( - config.global_training.save_latest_interval_minutes + config.global_training.latest_checkpoint_interval_minutes is not None and now - last_latest - >= config.global_training.save_latest_interval_minutes * 60 + >= config.global_training.latest_checkpoint_interval_minutes + * 60 ) if ( run.distributed.world_size > 1 @@ -602,7 +608,7 @@ def flush_metrics( last_latest = time.monotonic() if snapshot_due: if ( - config.global_training.save_interval_val_loss + config.global_training.evaluate_on_snapshot and phase.evaluation_sources ): self._evaluate( @@ -654,7 +660,7 @@ def flush_metrics( ) if phase.evaluation_sources and ( - phase_epoch % phase.evaluation_interval == 0 + phase_epoch % phase.evaluation_interval_epochs == 0 or phase_epoch == phase.epochs ) else EvaluationResult() @@ -674,14 +680,14 @@ def flush_metrics( boundary = capture_runtime_boundary(run) boundary_identity = last_identity if ( - phase_epoch % phase.checkpoint_interval == 0 + phase_epoch % phase.checkpoint_interval_epochs == 0 or phase_epoch == phase.epochs ): run.checkpoints.save( CheckpointRequest(f"epoch-{current_epoch}", last_identity), run, ) - patience = config.global_training.early_stopping_epochs + patience = config.global_training.early_stopping_patience_epochs if ( patience is not None and phase.evaluation_monitor is not None diff --git a/src/sequifier/training/optimization.py b/src/sequifier/training/optimization.py index d2ffcd5b..5b1de80c 100644 --- a/src/sequifier/training/optimization.py +++ b/src/sequifier/training/optimization.py @@ -52,7 +52,7 @@ class SchedulerPolicy: @dataclass(frozen=True) class GradientPolicy: - accumulation_steps: int + accumulation_batches: int clip_norm: float | None @@ -97,7 +97,7 @@ def create( parameters: Iterable[nn.Parameter] | list[dict[str, Any]], *, parameter_catalog: ParameterCatalog | None = None, - accumulation_steps: int = 1, + accumulation_batches: int = 1, layer_type_dtypes: dict[str, str] | None = None, ) -> "OptimizationRuntime": optimizer = create_optimizer( @@ -129,8 +129,8 @@ def create( scaler=GradScaler(device=device.split(":")[0], enabled=use_scaler), scheduler_policy=SchedulerPolicy(training.scheduler_step_on), gradient_policy=GradientPolicy( - accumulation_steps=accumulation_steps, - clip_norm=training.gradient_clip, + accumulation_batches=accumulation_batches, + clip_norm=training.gradient_clip_norm, ), ) @@ -239,7 +239,7 @@ def complete_step( overflow = has_gradients and not skip and not applied if applied: self.optimizer_step += 1 - if self.scheduler_policy.step_on == "batch": + if self.scheduler_policy.step_on == "optimizer_step": if directive is None or not directive.skip_scheduler_step: self.step_scheduler() elif directive is not None and directive.skip_scheduler_step: @@ -254,7 +254,7 @@ def complete_step( global_batch_step=identity.global_batch_step, optimizer_step=self.optimizer_step, accumulation_index=identity.accumulation_index, - accumulation_steps=identity.accumulation_steps, + accumulation_batches=identity.accumulation_batches, rank=identity.rank, world_size=identity.world_size, ), diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 0f0f6b6b..6f3a264c 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -424,17 +424,19 @@ def iter_batches( @beartype def _policy_parameter_ids(module: nn.Module, policy: Any, usage: str) -> set[int]: - if policy.freeze is None and policy.freezing_except is None: + if policy.freeze_groups is None and policy.freeze_except_groups is None: return set() groups = semantic_parameter_groups(module) configured = set( - policy.freeze if policy.freeze is not None else policy.freezing_except or [] + policy.freeze_groups + if policy.freeze_groups is not None + else policy.freeze_except_groups or [] ) matched = {group for group in configured if groups.get(group)} unmatched = configured - matched - if unmatched and policy.freezing_except is not None: + if unmatched and policy.freeze_except_groups is not None: raise ValueError( - f"{usage} freezing_except groups matched no parameters: " + f"{usage} freeze_except_groups groups matched no parameters: " f"{', '.join(sorted(unmatched))}" ) if unmatched: @@ -445,7 +447,7 @@ def _policy_parameter_ids(module: nn.Module, policy: Any, usage: str) -> set[int selected = { id(parameter) for group in configured for parameter in groups.get(group, ()) } - if policy.freeze is not None: + if policy.freeze_groups is not None: return selected return {id(parameter) for parameter in module.parameters()} - selected @@ -474,7 +476,8 @@ def visit(node, selection, inherited, path, *, is_root=False): if is_root: local = {identity: identity in root_frozen for identity in own} elif selection is not None and ( - selection.freeze is not None or selection.freezing_except is not None + selection.freeze_groups is not None + or selection.freeze_except_groups is not None ): frozen = _policy_parameter_ids(node, selection, path) local = {identity: identity in frozen for identity in own} @@ -687,7 +690,9 @@ def _choose(self, active: list[int]) -> int: if self.phase.selection == "weighted_random": return self.rng.choices( active, - weights=[self.phase.sources[index].weight for index in active], + weights=[ + self.phase.sources[index].selection_weight for index in active + ], k=1, )[0] for _ in range(len(self.sources)): diff --git a/src/sequifier/training/scheduler_budgets.py b/src/sequifier/training/scheduler_budgets.py index e7f01b4a..23260de2 100644 --- a/src/sequifier/training/scheduler_budgets.py +++ b/src/sequifier/training/scheduler_budgets.py @@ -66,7 +66,7 @@ def _phase_update_budget(config, datasets, phase_index): accumulated = 0 active_dataset = batch.dataset accumulated += 1 - if accumulated == phase.accumulation_steps: + if accumulated == phase.accumulation_batches: updates += 1 accumulated = 0 updates += int(accumulated > 0) @@ -78,7 +78,10 @@ def validate_supervised_scheduler_budgets(config, datasets, active_phase_index): phase_budgets = {} for index, phase in enumerate(config.training_plan): role = phase.optimization["model"] - if role.scheduler.name != "OneCycleLR" or role.scheduler_step_on != "batch": + if ( + role.scheduler.name != "OneCycleLR" + or role.scheduler_step_on != "optimizer_step" + ): continue start = index while start > 0 and config.training_plan[start].optimizer_state == "continue": diff --git a/src/sequifier/trajectory/sources.py b/src/sequifier/trajectory/sources.py index 997c3fd3..91dbfb82 100644 --- a/src/sequifier/trajectory/sources.py +++ b/src/sequifier/trajectory/sources.py @@ -30,10 +30,10 @@ def _checksum(path, checksums): def _source_files(source, checksums): manifest = None - if source.manifest is None: + if source.manifest_path is None: paths = [Path(path) for path in source.paths] else: - manifest_path = Path(source.manifest) + manifest_path = Path(source.manifest_path) content = manifest_path.read_bytes() checksum = hashlib.sha256(content).hexdigest() if manifest_path in checksums and checksums[manifest_path] != checksum: @@ -94,7 +94,7 @@ def resolve_source( dependencies.update( Path(path) for path in ( - source.manifest, + source.manifest_path, source.metadata_config_path, source.static_data_path, ) From 6037b4caeac00bc9ea2c2285b5989d1f0987da55 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 12 Nov 2026 11:32:19 +0100 Subject: [PATCH 30/71] unify schema --- README.md | 2 +- docs/source/guides/ownership.rst | 2 + docs/source/index.rst | 1 + .../configs/hyperparameter-search.md | 10 +- documentation/configs/infer.md | 37 +- documentation/configs/naming.md | 162 ++- documentation/configs/ownership.md | 146 +++ documentation/configs/preprocess.md | 15 +- documentation/configs/train.md | 474 ++++--- .../demos/self-contained-example.ipynb | 100 +- .../configs/import-policy-sequences.yaml | 3 +- .../example_inputs/configs/infer-policy.yaml | 9 +- .../example_inputs/configs/infer-ppo-toy.yaml | 9 +- .../configs/rollout-ppo-toy.yaml | 9 +- .../example_inputs/configs/train-bc-toy.yaml | 48 +- .../configs/train-iql-from-table.yaml | 54 +- .../example_inputs/configs/train-iql-toy.yaml | 54 +- .../configs/train-multiphase-control.yaml | 98 +- .../configs/train-policy-from-table.yaml | 48 +- .../configs/train-return-conditioned.yaml | 58 +- documentation/learning-architecture.md | 8 +- documentation/policy-learning.md | 10 +- documentation/training/multi-gpu-training.md | 20 +- .../training/runtime-architecture.md | 2 +- .../unreleased-canonical-config-names.md | 1 + .../unreleased-configuration-ownership.md | 25 +- release-notes/unreleased-phase-ownership.md | 5 +- src/sequifier/artifacts/model_config.py | 13 +- src/sequifier/artifacts/model_export.py | 4 +- src/sequifier/artifacts/run_checkpoint.py | 10 +- .../canonical_hyperparameter_search_config.py | 24 +- src/sequifier/config/command_ownership.py | 69 + src/sequifier/config/components.py | 2 +- src/sequifier/config/composition.py | 10 +- src/sequifier/config/freezing_config.py | 84 -- src/sequifier/config/infer_config.py | 17 +- src/sequifier/config/inference_common.py | 42 +- .../config/input_contract_resolution.py | 56 + src/sequifier/config/learning_config.py | 40 + src/sequifier/config/metadata.py | 2 +- src/sequifier/config/naming.py | 115 +- src/sequifier/config/normalization_config.py | 29 + src/sequifier/config/optimizer_config.py | 33 +- src/sequifier/config/ownership.py | 125 ++ src/sequifier/config/parameter_selection.py | 189 +++ src/sequifier/config/phase_controls.py | 109 ++ src/sequifier/config/policy_plan.py | 70 +- src/sequifier/config/preprocess_config.py | 36 +- src/sequifier/config/rollout_config.py | 2 +- .../config/sequence_import_config.py | 7 +- src/sequifier/config/train_config.py | 1136 +++++++++-------- src/sequifier/config/training_resolution.py | 155 ++- src/sequifier/docs.py | 1 + src/sequifier/evaluation/service.py | 2 +- src/sequifier/export/preflight.py | 4 +- src/sequifier/export/service.py | 14 +- src/sequifier/hyperparameter_search.py | 6 +- src/sequifier/integration/callbacks.py | 2 +- src/sequifier/io/config.py | 13 - src/sequifier/io/folder_dataset.py | 3 +- src/sequifier/io/preference_dataset.py | 4 +- .../io/sequifier_dataset_from_file.py | 4 +- src/sequifier/io/yaml.py | 8 - src/sequifier/make.py | 74 +- src/sequifier/model/execution.py | 18 +- src/sequifier/model/factory.py | 18 +- src/sequifier/model/parameter_catalog.py | 44 +- src/sequifier/objectives.py | 8 +- src/sequifier/optimizers/plan.py | 17 +- src/sequifier/rl/attachment.py | 6 +- src/sequifier/rl/dpo.py | 9 + src/sequifier/rl/grpo.py | 4 +- src/sequifier/rl/iql.py | 20 +- src/sequifier/rl/learner.py | 7 +- src/sequifier/rl/offline_data.py | 21 +- src/sequifier/rl/policy.py | 2 +- src/sequifier/rl/ppo.py | 12 +- src/sequifier/rl/preparation.py | 63 +- src/sequifier/rl/prepared_inputs.py | 3 + src/sequifier/rl/run_checkpoint.py | 28 +- src/sequifier/rl/run_config.py | 10 +- src/sequifier/rl/run_state.py | 122 +- src/sequifier/rl/training.py | 11 +- src/sequifier/rl_api.py | 13 +- src/sequifier/runtime/builder.py | 52 +- src/sequifier/runtime/stopping.py | 31 + src/sequifier/sequifier.py | 31 +- src/sequifier/templates/infer-policy.yaml | 9 +- src/sequifier/templates/train-dpo.yaml | 54 +- src/sequifier/templates/train-grpo.yaml | 49 +- src/sequifier/templates/train-iql.yaml | 52 +- .../train-policy-from-supervised.yaml | 49 +- src/sequifier/templates/train-policy.yaml | 54 +- src/sequifier/templates/train-ppo.yaml | 57 +- .../templates/train-supervised-to-policy.yaml | 72 +- src/sequifier/templates/train-supervised.yaml | 57 +- src/sequifier/train.py | 20 +- src/sequifier/training/checkpoint_service.py | 26 +- src/sequifier/training/coordinator.py | 91 +- src/sequifier/training/engine.py | 149 ++- src/sequifier/training/metrics.py | 9 +- src/sequifier/training/metrics_service.py | 12 +- src/sequifier/training/runtime.py | 256 ++-- src/sequifier/training/scheduler_budgets.py | 15 +- src/sequifier/training/state.py | 4 + src/sequifier/training_api.py | 13 + tools/convert_folder.sh | 27 - tools/convert_v0_config_to_v1.py | 224 ---- tools/extract_validation_losses_to_table.py | 1 - 109 files changed, 3639 insertions(+), 2135 deletions(-) create mode 100644 docs/source/guides/ownership.rst create mode 100644 documentation/configs/ownership.md create mode 100644 src/sequifier/config/command_ownership.py delete mode 100644 src/sequifier/config/freezing_config.py create mode 100644 src/sequifier/config/input_contract_resolution.py create mode 100644 src/sequifier/config/learning_config.py create mode 100644 src/sequifier/config/normalization_config.py create mode 100644 src/sequifier/config/ownership.py create mode 100644 src/sequifier/config/parameter_selection.py create mode 100644 src/sequifier/config/phase_controls.py delete mode 100644 src/sequifier/io/config.py create mode 100644 src/sequifier/runtime/stopping.py delete mode 100644 tools/convert_folder.sh delete mode 100644 tools/convert_v0_config_to_v1.py diff --git a/README.md b/README.md index c286b649..d4b1a018 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ Once you have your data in the input format described above, you can train a tra 7. **Configure inference.** Point `model_paths` in `infer.yaml` at the default ONNX export. Keep the scaffold's explicit contract, or replace it with `training_config_path` and `dataset`; see the [ONNX/PT trade-offs](./documentation/configs/infer.md#onnx-or-pt). - **GPU ONNX inference:** `device: cuda` requires `onnxruntime-gpu` built for + **GPU ONNX inference:** `run.execution.device: cuda` requires `onnxruntime-gpu` built for your CUDA and cuDNN versions. The CPU-only `onnxruntime` package does not provide `CUDAExecutionProvider`. diff --git a/docs/source/guides/ownership.rst b/docs/source/guides/ownership.rst new file mode 100644 index 00000000..d1db5c5e --- /dev/null +++ b/docs/source/guides/ownership.rst @@ -0,0 +1,2 @@ +.. include:: ../../../documentation/configs/ownership.md + :parser: myst_parser.sphinx_ diff --git a/docs/source/index.rst b/docs/source/index.rst index 19611fc8..abef44b4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ Welcome to Sequifier's documentation! Inference Hyperparameter search Configuration naming + Configuration ownership Visualizing training Distributed and multi-node training Training runtime architecture diff --git a/documentation/configs/hyperparameter-search.md b/documentation/configs/hyperparameter-search.md index 304b0e17..50fde83c 100644 --- a/documentation/configs/hyperparameter-search.md +++ b/documentation/configs/hyperparameter-search.md @@ -22,12 +22,12 @@ trial_config_directory: configs/hp-search method: bayesian trials: 40 parameters: - global_training: - context_length: - "$choices": - - 64 - - 128 model: + execution: + context_length: + "$choices": + - 64 + - 128 backbone: architecture: num_layers: diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index 2562bfea..6cf53fb9 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -39,10 +39,10 @@ sequifier infer --config-path configs/infer.yaml ## Start here: ONNX -> **GPU ONNX inference requires `onnxruntime-gpu`.** If you set `device: cuda`, +> **GPU ONNX inference requires `onnxruntime-gpu`.** If you set `run.execution.device: cuda`, > install a build compatible with your CUDA and cuDNN versions. The CPU-only > `onnxruntime` package does not provide `CUDAExecutionProvider`; setting -> `device: cuda` alone does not enable GPU inference. See the +> `run.execution.device: cuda` alone does not enable GPU inference. See the > [ONNX Runtime CUDA compatibility guide](https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html). ONNX is the default training export and the deployment-oriented inference path. @@ -50,14 +50,16 @@ Exports embed their execution contract. Select a training route when providing model configuration through a training config: ```yaml -project_root: "." training_config_path: configs/train.yaml dataset: events data_path: data/events-test.parquet -device: cuda task: forecast model_paths: - models/event-model-best-5.onnx +run: + project_root: . + execution: + device: cuda ``` The route supplies columns, types, objective, window sizes, and preprocessing @@ -78,16 +80,18 @@ Benchmark the target workload rather than assuming either runtime is faster. Choose PT when portability matters less than a compact, self-contained config: ```yaml -project_root: "." data_path: data/events-test.parquet -device: cuda task: forecast model_paths: - models/event-model-best-5.pt +run: + project_root: . + execution: + device: cuda ``` For multi-interface PT, add `model_interface`. `task` is required for both formats: `forecast`, `policy`, or `embedding`. -`project_root` is required (normally `.`), and `inference_batch_size` defaults +`run.project_root` defaults to `.`, and `run.execution.batch_size` defaults to `1`. ## Effective configuration and validation @@ -118,8 +122,8 @@ fragments or define the same field twice. | --- | --- | --- | | `model_paths` | Required | Non-empty list of compatible model paths; policy requires exactly one. | | `task` | Required | `forecast`, `policy`, or `embedding`. | -| `device` | `cpu` | ONNX: `cpu` or `cuda`; PT also supports `mps`. ONNX CUDA requires an installed runtime exposing `CUDAExecutionProvider`. | -| `project_root` | Required | Base for project paths; normally `.`. | +| `run.execution.device` | `cpu` | ONNX: `cpu` or `cuda`; PT also supports `mps`. ONNX CUDA requires an installed runtime exposing `CUDAExecutionProvider`. | +| `run.project_root` | Required | Base for project paths; normally `.`. | | `data_path` | Metadata test/last split | Input file or folder. Usually required with artifact-only inference because artifacts do not store split paths. | | `metadata_config_path` | `null` | Explicit preprocessing metadata. | | `image_paths` | `null` | View-to-folder mapping for every selected temporal view. Required when the model consumes temporal images. | @@ -129,7 +133,7 @@ fragments or define the same field twice. | `model_interface` | Implicit when unique | Select a route from a training config or portable artifact. | | `read_format` | `parquet` | `csv`, `parquet`, or folder-based `pt`. | | `write_format` | `csv` | `csv` or `parquet`. | -| `inference_batch_size` | `1` | Sequences processed per batch. | +| `run.execution.batch_size` | `1` | Sequences processed per batch. | ### Model contract @@ -168,8 +172,8 @@ stored folder window if the remaining layout and semantics match. | `decode_categories` | `true` | Decode categorical predictions to their original values. | | `sample_from_distribution_columns` | `null` | Sample these categorical targets instead of using argmax. | | `infer_with_dropout` | `false` | Enable dropout at inference; ONNX also requires dropout-preserving export. | -| `enforce_determinism` | `false` | Request deterministic PyTorch algorithms. | -| `seed` | `1010` | Random seed. | +| `run.reproducibility.enforce_determinism` | `false` | Request deterministic PyTorch algorithms. | +| `run.reproducibility.seed` | `1010` | Random seed. | | `autoregressive_generation` | `false` | Feed predictions back for multi-step generation. | | `generation_steps` | `null` | Required positive step count when autoregressive is enabled. | @@ -224,15 +228,18 @@ inference does not configure it again. Inference supplies the matching named layout tensors and masks through the preprocessed PT input. ```yaml -project_root: "." read_format: pt data_path: data/raw-items-split2 -device: cpu infer_with_dropout: false -seed: 1010 task: forecast model_paths: - models/item-model-best-5.onnx +run: + project_root: . + execution: + device: cpu + reproducibility: + seed: 1010 ``` Use the corresponding embedding artifact with `task: embedding`, or change diff --git a/documentation/configs/naming.md b/documentation/configs/naming.md index b04160e2..8d97b782 100644 --- a/documentation/configs/naming.md +++ b/documentation/configs/naming.md @@ -1,10 +1,7 @@ # Configuration naming reference Sequifier accepts one canonical spelling and location for each authored setting. -Removed names produce replacement guidance; they are not compatibility aliases. -These are breaking configuration changes. Update authored files and any embedded -configuration used to reconstruct a model or resume a run before reusing them. -Settings not listed here retain their names, scopes, and defaults. +See [configuration ownership](ownership.md) for the supported locations. ## Named structures and fixed shapes @@ -27,22 +24,22 @@ Generated metadata retains its selected-split `window_length`. ## Dimensions, counts, and selection -| Previous spelling | Canonical spelling | -| --- | --- | -| `dim_model` | `model_dim` | -| `feed_forward.dim` | `feed_forward.hidden_dim` | -| `n_heads` / `n_kv_heads` | `num_heads` / `num_kv_heads` | -| Static `embedding_size` | `total_embedding_dim` | -| `positional_encoding_scope` | `position_encoding_scope` | -| `post_conv_norm` | `post_conv_normalization` | -| `frame_pool` / `image_frame_pool` | `frame_pooling` / `image_frame_pooling` | -| Axis-pooling `mode` | `pooling` | -| Decoder `support` | `support_length` | -| Inference `decoder_support` | `decoder_support_length` | -| Depth-layout `context_length` | `max_depth_length` | -| `device_max_concat_length` | `max_concat_tensors` | -| Preprocessing `n_cores` | `num_workers` | -| Source `weight` | `selection_weight` | +| Setting | +| --- | +| `model_dim` | +| `feed_forward.hidden_dim` | +| `num_heads` / `num_kv_heads` | +| `total_embedding_dim` | +| `position_encoding_scope` | +| `post_conv_normalization` | +| `frame_pooling` / `image_frame_pooling` | +| `pooling` | +| `support_length` | +| `decoder_support_length` | +| `max_depth_length` | +| `max_concat_tensors` | +| `run.execution.data_loading.num_workers` | +| `selection_weight` | `model_dim`, `hidden_dim`, `output_dim`, `cell_dim`, `hidden_dims`, and `feature_embedding_dims` describe different representation widths. @@ -63,25 +60,25 @@ DPO perform one optimizer step per role; PPO and GRPO may perform several. An epoch is one pass through the applicable data. Environment steps count transitions; generation steps count temporal autoregressive output steps. -| Previous spelling | Canonical spelling | -| --- | --- | -| `accumulation_steps` | `accumulation_batches` | -| PPO `minibatch_size` | `batch_size` | -| IQL/DPO `updates` | `iterations` | -| GRPO `groups_per_update` | `groups_per_iteration` | -| PPO `rollout_length` | `rollout_env_steps` | -| Supervised `evaluation_interval` | `evaluation_interval_epochs` | -| Policy `evaluation_interval` | `evaluation_interval_iterations` | -| Supervised `checkpoint_interval` | `checkpoint_interval_epochs` | -| Policy `checkpoint_interval` | `checkpoint_interval_iterations` | -| `log_interval` | `log_interval_batches` | -| `early_stopping_epochs` | `early_stopping_patience_epochs` | -| Supervised `scheduler_step_on: batch` | `scheduler_step_on: optimizer_step` | -| `gradient_clip` | `gradient_clip_norm` | -| PPO `value_clip` | `value_clip_delta` | -| PPO `target_kl` | `old_policy_target_kl` | -| GRPO `kl_coefficient` | `reference_kl_coefficient` | -| IQL `max_weight` | `max_advantage_weight` | +| Setting | +| --- | +| `accumulation_batches` | +| `batch_size` | +| `iterations` | +| `groups_per_iteration` | +| `rollout_env_steps` | +| `evaluation.cadence.epochs` | +| `evaluation.cadence.iterations` | +| `checkpoints.cadence.epochs` | +| `checkpoints.cadence.iterations` | +| `log_interval_batches` | +| `stopping.patience` (evaluation observations) | +| `scheduler_step_on: optimizer_step` | +| `gradient_clip_norm` | +| `value_clip_delta` | +| `old_policy_target_kl` | +| `reference_kl_coefficient` | +| `max_advantage_weight` | Keep `epochs`, `optimization_epochs`, `total_env_steps`, `max_env_steps`, `total_groups`, `group_size`, `max_steps`, `max_episode_steps`, and @@ -96,24 +93,15 @@ Scheduler `total_steps` remains an assertion against the resolved scheduler budg ## Freezing, checkpoints, and direction -The dataset `freeze` container remains. Inside its component tree, `freeze` -becomes `freeze_groups` and `freezing_except` becomes `freeze_except_groups`. -Policy phase `freeze` becomes `freeze_modules`: its entries are actual module -paths, not semantic group selectors. Empty `freeze_groups` unfreezes a scope; -empty `freeze_except_groups` freezes it. - -| Previous spelling | Canonical spelling | -| --- | --- | -| `save_interval_batches` | `snapshot_interval_batches` | -| `save_interval_minutes` | `snapshot_interval_minutes` | -| `save_latest_interval_minutes` | `latest_checkpoint_interval_minutes` | -| `save_interval_val_loss` | `evaluate_on_snapshot` | -| `calculate_validation_loss_on_initialization` | `evaluate_before_training` | -| Monitor `mode` | `direction: min` or `direction: max` | - -Snapshots are retained checkpoints triggered by the global batch/time intervals. -Refreshing `latest` is distinct. `resume`, `initialize_from`, `optimizer_state`, -and `handoff` keep their separate lifecycle meanings. +Freezing now belongs to `phase.trainability.roles.` for all algorithms; +source-dependent supervised masks use `phase.trainability.by_dataset`. +`freeze`/`freeze_except` contain catalog selector objects, shared with optimizer +routing. Explicit module selectors and semantic scopes share the same selection language. + +Recovery is `run.recovery`; retained snapshots are `phase.checkpoints`; +evaluation triggers are `phase.evaluation.before_phase` and `.on_snapshot`. +Handoff is `phase.model_selection.handoff`; source scheduling retains `selection`. +See [ownership](ownership.md) for the complete structure and stopping units. ## Inference, sources, and seeds @@ -123,20 +111,20 @@ chooses the task implicitly. Policy requires exactly one entry in `model_paths`. `autoregressive` becomes `autoregressive_generation`. Inference's `preprocessing_data_path` metadata shortcut is removed; use `metadata_config_path`. -| Previous spelling | Canonical spelling | -| --- | --- | -| Inference `model_path` | `model_paths` | -| `image_path` | `image_paths` | -| `preconditioning_image_path` | `preconditioning_image_paths` | -| Preconditioning image `path` | `paths` | -| Global interface `metadata_config_path` | `interface_metadata_paths` | -| Sequence/preference `manifest` | `manifest_path` | +| Setting | +| --- | +| `model_paths` | +| `image_paths` | +| `preconditioning_image_paths` | +| `paths` | +| `model.interfaces..input_contract.metadata_config_path` | +| `manifest_path` | -`interface_metadata_paths` maps interface names to files. Per-part and inference +Interface input contracts select metadata files. Per-part and inference `metadata_config_path` still select one file. Source-local `project_root` is removed; the run supplies it during resolution. -Run `seed` accepts an integer or `random`. Random seeds resolve once to an integer +Run `run.reproducibility.seed` accepts an integer or `random`. Random seeds resolve once to an integer recorded in the effective configuration. `sampler_seed` controls an independent search stream. Initialization and hash seeds stay scoped integer declarations. `enforce_determinism`, `action_selection`, and `seed` remain independent controls. @@ -147,17 +135,17 @@ and model-owned versus fixed policy `scale`: these are meaningful distinctions. ## Search expressions -| Previous spelling | Canonical spelling | -| --- | --- | -| `global_seed` | `sampler_seed` | -| `method: sample` | `method: random` | -| `model_config_write_path` | `trial_config_directory` | -| `evaluation_script` | `evaluation_script_path` | -| `evaluation_inference_config` | `evaluation_inference_config_path` | -| `choices`, implicit categorical list | `$choices` | -| `fixed` | `$fixed` | -| `variants` | `$variants` | -| Bare numeric range mapping | `$range: {low, high, step?, log?, type?}` | +| Setting | +| --- | +| `sampler_seed` | +| `method: random` | +| `trial_config_directory` | +| `evaluation_script_path` | +| `evaluation_inference_config_path` | +| `$choices` | +| `$fixed` | +| `$variants` | +| `$range: {low, high, step?, log?, type?}` | Primitive scalars are fixed replacements; ordinary mappings recursively patch. `$fixed` replaces a complete mapping or list. `$choices` samples any complete @@ -170,15 +158,15 @@ Training parameter paths are exactly the canonical training paths. ## Cardinality, initialization, and dtype values -| Previous spelling | Canonical spelling | -| --- | --- | -| `num_buckets`, `buckets` | `num_buckets` | -| `min_freq`, `min_count` | `min_count` | -| `top_k`, `k`, `max_categories` | `max_categories` | -| `column_types`, `column_data_types` | `column_data_types` | -| `xavier`, `glorot`, `glorot_uniform`, `xavier_uniform` | `xavier_uniform` | -| `glorot_normal`, `xavier_normal` | `xavier_normal` | -| Distribution `standard_deviation` | `std` | +| Setting | +| --- | +| `num_buckets` | +| `min_count` | +| `max_categories` | +| `column_data_types` | +| `xavier_uniform` | +| `xavier_normal` | +| `std` | Numeric storage dtypes use canonical `Float32`, `Int64`, etc.; lowercase aliases are rejected. Compute dtypes remain PyTorch-style `float32`, `bfloat16`, etc. diff --git a/documentation/configs/ownership.md b/documentation/configs/ownership.md new file mode 100644 index 00000000..f35e7023 --- /dev/null +++ b/documentation/configs/ownership.md @@ -0,0 +1,146 @@ +# Configuration ownership + +Every authored setting has one owner. Configuration files use the ownership +structure below directly. + +| Owner | Purpose | +| --- | --- | +| `run` | Identity, execution resources, reproducibility, recovery, logging, outputs and export. | +| `model.execution` | Durable objective/window/attention facts required by the artifact. | +| `model.interfaces..input_contract` | Metadata and frozen normalization assertions. | +| `dataset_training.` | Named interface reference, source parts and split policy. | +| `policy_bindings.` | Actions, task units, return interpretation and action behavior. | +| `training_plan.phases[]` | Learning, trainability, budgets, optimization, evaluation, stopping and checkpoint snapshots. | +| `training_plan` | Ordered phase transitions. | + +Preprocessing and inference share `run.project_root`, `run.execution` and +`run.reproducibility`. Their command-specific source/output and operation fields +remain on the command. Inference batching is `run.execution.batch_size`; +preprocessing workers are `run.execution.data_loading.num_workers`. Unsupported +execution controls fail validation. Component initialization and hash seeds stay +on their components; evaluation seeds define a benchmark and stay in evaluation. + +## Run settings + +```yaml +run: + project_root: . + model_name: example + execution: + device: cpu + distribution: {strategy: local} + precision: {float32_matmul_precision: highest} + compilation: {torch_compile: none} + data_loading: {num_workers: 0, load_full_data_to_ram: true} + reproducibility: {seed: 1010, enforce_determinism: false} + recovery: + resume: {policy: never} + latest_checkpoint_interval_minutes: 10 + logging: {log_interval_batches: 10} + outputs: {model_path: models/example/model.pt} + export: {generative_model: true, onnx: true, pt: false, batch_size: 1} +``` + +Distribution is one coherent choice: `local`, `ddp` or `fsdp`. FSDP additionally +requires an explicit `cpu_offload`. Current policy execution is local, without +custom precision, compilation, concatenation limits, batch logging, or worker/lazy-loader/RAM-budget settings. +Policy-containing plans export the portable PT actor; graph, embedding, dropout and +batched exports are rejected. Runs default to PT export. Supervised runs can request graph exports explicitly. Defaults do not +request those capabilities. Random run seeds resolve once; derived phase streams +and resolved phase declarations are recorded in native recovery/provenance. + +## Dataset and model contracts + +Prepared parts own `read_format`; sequence parts own `format`. Raw datasets own +`split.validation_fraction`. Combining raw sources into one automatic, leakage-safe +partition requires equal fractions; use explicit `evaluation.sources` when the +partitions differ. Prepared split metadata remains authoritative. Fitting uses +training partitions only. + +`model.execution.context_length` is the selected temporal view; +`model.backbone.architecture.max_context_length` is capacity. Frozen input contracts +and artifact assertions must agree. Phases cannot redefine immutable window, +attention or codec facts. `learning.bert_spec` and `learning.next_occurrence_config` +configure objective behavior; reconstruction-required next-occurrence facts also +belong to the durable model contract and must agree across phases. + +## Phase settings + +```yaml +training_plan: + phases: + - name: fit + algorithm: supervised + epochs: 10 + mode: sequential + sources: [{source: events}] + learning: + datasets: + events: + criterion: {event: CrossEntropyLoss} + loss_weights: {event: 1.0} + trainability: + roles: + model: + freeze: + - {component: backbone, semantic_groups: [attention.qkv]} + evaluation: + sources: [{source: events}] + monitor: {source: events, metric: loss, direction: min} + cadence: {epochs: 1} + batch_size: 64 + before_phase: false + on_snapshot: false + model_selection: {handoff: best} + stopping: {patience: 3, min_delta: 0.001, on_trigger: end_phase} + checkpoints: + cadence: {epochs: 2} + snapshot_interval_minutes: 30 +``` + +Declare phase learning losses for all training and evaluation datasets. Source +scheduling still uses supervised `selection: round_robin | weighted_random`; +`model_selection` is distinct. Curriculum is `curriculum: {column: name}` on a +supervised phase. Sampling stride is `learning.window_stride`. + +All algorithms share these owners. Supervised cadence uses `{epochs: N}`; +policy cadence uses `{iterations: N}`. A policy iteration is a complete algorithm +transaction, possibly containing multiple optimizer steps. Final progress always +evaluates when enabled and checkpoints. Initial evaluation and snapshot evaluation +can select best weights but do not consume stopping patience. Patience counts +scheduled finite evaluation observations; `on_trigger` explicitly ends the phase +or the entire plan. Stopping state is persisted so resume cannot count twice. + +Recovery refreshes rolling resumable state at safe boundaries. Phase snapshots +are retained checkpoints. Best-weight handoff is distinct from latest recovery; +it does not restore older optimizer/environment state. Snapshot time intervals +request the next safe committed boundary. Policy batch snapshots are rejected; +use transaction cadence or time instead. + +## Parameter selection + +Freezing and optimizer routing use the same catalog selectors: `component`, +`semantic_groups` (glob patterns), `parameter_kind`, `ndim`, named `interface`, +`branch_path`, or a `module` subtree. Conditions in a selector are conjunctive; +selector lists form a union. `exclude_modules` and `exclude_semantic_groups` +express explicit subtractions, for nested component policies. +Unknown or empty selections fail preflight. + +Each role accepts either `freeze` or `freeze_except`. Empty `freeze` masks nothing; +empty `freeze_except` masks everything and fails if an update has no trainable +parameters. Structural constraints remain: fixed Gaussian scales and frozen image +encoders cannot be enabled by a phase mask. Gradient freezing leaves model mode +semantics intact. + +Supervised `trainability.by_dataset..roles.model` uses exactly the same +policy shape. A supplied role policy replaces the phase policy for that dataset; +absence inherits it. Only training datasets can have update overrides. Policy +algorithms reject source-dependent masks, and support their own declared role set. +Frozen gradients are removed before AMP handling and stepping, so momentum and +weight decay do not update them. Masks are recalculated at phase boundaries. + +## Recovery + +Run checkpoints contain the current resolved settings, phase state and input +identities. Recovery requires these to agree with the requested run. Portable +model artifacts contain only the facts needed to reconstruct model execution. diff --git a/documentation/configs/preprocess.md b/documentation/configs/preprocess.md index 66632d24..2546ee0a 100644 --- a/documentation/configs/preprocess.md +++ b/documentation/configs/preprocess.md @@ -34,7 +34,7 @@ The configuration is defined in a YAML file (e.g., `preprocess.yaml`). Below are | Field | Type | Mandatory | Default | Description | | :--- | :--- | :--- | :--- | :--- | -| `project_root` | `str` | **Yes** | - | The root directory of your Sequifier project. Usually `.` | +| `run.project_root` | `str` | **Yes** | - | The root directory of your Sequifier project. Usually `.` | | `additional_config_paths` | `list[str]` or `null` | No | `null` | Direct complementary YAML fragments. Relative paths resolve against `project_root`; recursive composition and duplicate fields are rejected. | | `preprocessing_data_path` | `str` | **Yes** | - | Path to the raw input file or folder. | | `static_data_path` | `str` | No | `null` | Separate CSV or Parquet table with exactly one row per `sequenceId`. Preprocessing requires its ID set to match the sequence data in both directions and writes a processed static feature table. | @@ -250,7 +250,7 @@ left-padded. The stored windows never read beyond the split end. An aligned split requires `target_offset == max_target_offset` and `allow_sequence_splitting: false`. Training and inference must use matching `target_offset` and `prediction_length`. Training can sample additional model -views with `global_training.window_stride`; aligned inference requires +views with `phase learning.window_stride`; aligned inference requires `window_stride: null`. Empty splits produce no windows. Prediction-aligned inference supports generative and embedding output without @@ -268,8 +268,8 @@ PT output. | Field | Type | Mandatory | Default | Description | | :--- | :--- | :--- | :--- | :--- | -| `seed` | `int` | No | `1010` | Random seed for reproducibility. | -| `num_workers` | `int` | No | Max Cores | Maximum worker processes for eligible preprocessing stages. Depth preprocessing parallelizes a single Parquet file only when it has multiple row groups and `max_rows` is unset. | +| `run.reproducibility.seed` | `int` | No | `1010` | Random seed for reproducibility. | +| `run.execution.data_loading.num_workers` | `int` | No | Max Cores | Maximum worker processes for eligible preprocessing stages. Depth preprocessing parallelizes a single Parquet file only when it has multiple row groups and `max_rows` is unset. | | `batches_per_file` | `int` | No | `1024` | Buffer flush threshold per split. For flat input, counts sequence-window groups in PT and Parquet output (including merged output); a byte limit can flush earlier. For depth input, counts windows in PT output. | | `process_by_file` | `bool` | No | `true` | Memory optimization. If `true`, processes one input file at a time. | @@ -386,7 +386,7 @@ cardinality_config: ``` `num_buckets` is a positive integer. The optional `hashing.seed` defaults to -`0` and is separate from the top-level preprocessing `seed`. Hashing is stable +`0` and is separate from the preprocessing `run.reproducibility.seed`. Hashing is stable for strings, booleans, and integers. Multiple values can land in the same bucket, so their original identities cannot be recovered from preprocessed data; decoded bucket values appear as `[hash_bucket:k]`. Null values retain the @@ -441,7 +441,6 @@ Depth behavior is explicit; a column named `subItemPosition` alone remains an ordinary flat feature. ```yaml -project_root: "." preprocessing_data_path: data/raw-items read_format: parquet write_format: pt @@ -470,6 +469,8 @@ window_lengths: - 129 - 129 - 129 +run: + project_root: . ``` Every file must contain the depth position column, which is read automatically @@ -494,7 +495,7 @@ statistics count occupied child slots. Both populations are selected before split extraction. Materialization encodes each item once and uses bounded windows and output batches; `batches_per_file` bounds the number of windows accumulated per split. For a single Parquet file with multiple row groups and -no `max_rows`, this adapter uses `num_workers` to parallelize auditing, metadata +no `max_rows`, this adapter uses `run.execution.data_loading.num_workers` to parallelize auditing, metadata fitting, and window materialization. Other depth inputs use the sequential merge path. diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 9a2b87ee..106bb2b8 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -5,8 +5,7 @@ IQL and online PPO through the same commands, see [policy learning](../policy-le `sequifier train` trains one shared transformer backbone through one or more named model interfaces. An interface is an ingestion module, its generated -adapter, and a decoder. Phases own batching and optimization; datasets own data and losses; `model` -owns architecture. +adapter, and a decoder. Phases own batching, learning, trainability and lifecycle controls; datasets own sources; `model` owns architecture and execution contracts. See the [ownership reference](ownership.md) for the complete field map and migration command. ```console sequifier train --config-path configs/train.yaml @@ -17,15 +16,6 @@ sequifier train --config-path configs/train.yaml Use the same named structure for one or many interfaces, datasets, parts, and phases: ```yaml -project_root: "." -model_name: event-model -device: cuda -seed: 1010 -global_training: - read_format: parquet - training_objective: causal - context_length: 128 - inference_batch_size: 256 model: backbone: architecture: @@ -49,11 +39,14 @@ model: dropout: 0.1 shared_layer_groups: [] interfaces: - default: + event_prediction: input_columns: - event target_columns: - event + categorical_decoder_special_tokens: + event: + - other ingestion: type: embedding output_dim: 128 @@ -61,32 +54,160 @@ model: type: linear prediction_length: 1 support_length: 1 + execution: + context_length: 128 + target_offset: 1 + training_objective: causal +dataset_training: + events: + model_interface: event_prediction + parts: + original: + metadata_config_path: configs/metadata_configs/events.json + read_format: parquet + increment: + metadata_config_path: configs/metadata_configs/events-increment.json + read_format: parquet + split: + validation_fraction: 0.2 training_plan: phases: - - epochs: 5 + - name: incremental_finetuning + epochs: 2 mode: sequential - name: supervised + sources: + - source: events.increment + batch_size: 64 + accumulation_batches: 4 + optimization: + model: + learning_rate: 0.0001 + optimizer: + name: AdamW + arguments: + weight_decay: 0.01 + scheduler: + name: StepLR + arguments: + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip_norm: 1.0 + optimizer_state: reset + evaluation: + sources: + - source: events + cadence: + epochs: 1 + batch_size: 64 + before_phase: true + on_snapshot: true algorithm: supervised + model_selection: + handoff: last + checkpoints: + cadence: + epochs: 1 + learning: + window_stride: 1 + datasets: + events: + criterion: + event: CrossEntropyLoss + loss_weights: + event: 1.0 + trainability: + by_dataset: + events: + roles: + model: &id001 + freeze: + - component: backbone + exclude_modules: [] + semantic_groups: + - attention.qkv + - name: complete_retraining + epochs: 5 + mode: interleaved + selection: round_robin + sources: + - source: events + batches_per_selection: 4 batch_size: 64 + accumulation_batches: 4 optimization: model: learning_rate: 0.0001 + optimizer: + name: AdamW + arguments: + weight_decay: 0.01 + scheduler: + name: StepLR + arguments: + step_size: 1 + gamma: 0.99 + scheduler_step_on: epoch + gradient_clip_norm: 1.0 + optimizer_state: reset evaluation: sources: - - source: default.default -dataset_training: - default: - criterion: - event: CrossEntropyLoss - parts: - default: - metadata_config_path: configs/metadata_configs/events.json + - source: events + cadence: + epochs: 1 + batch_size: 64 + before_phase: false + on_snapshot: true + algorithm: supervised + model_selection: + handoff: last + checkpoints: + cadence: + epochs: 1 + learning: + window_stride: 1 + datasets: + events: + criterion: + event: CrossEntropyLoss + loss_weights: + event: 1.0 + trainability: + by_dataset: + events: + roles: + model: *id001 +run: + project_root: . + model_name: event-model + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cuda + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: outer + recovery: {} + logging: {} + outputs: {} + export: + generative_model: true + embedding_model: false + onnx: true + pt: false + with_dropout: false + batch_size: 256 ``` All configurations use `model.interfaces`, `dataset_training`, dataset `parts`, `policy_bindings` where applicable, and `training_plan.phases`. A unique interface -and source may be inferred when omitted. Singular aliases and direct phase fields -are rejected with canonical replacement paths. See the [naming reference](naming.md). +and source may be inferred when omitted. See the [naming reference](naming.md). To condition every model window on features from a separate per-sequence table, preprocess with `static_data_path` and configure the interface: @@ -127,22 +248,11 @@ the existing cardinality-based default and real features use one channel. The concatenated features are projected to the backbone `model_dim`. The model creates one prefix token at runtime and removes its output before decoding. Set backbone `max_context_length` to -at least `global_training.context_length + 1`. +at least `model.execution.context_length + 1`. ## Canonical configuration ```yaml -project_root: "." -model_name: event-model -device: cuda -seed: 1010 -global_training: - read_format: parquet - training_objective: causal - context_length: 128 - target_offset: 1 - window_stride: 1 - inference_batch_size: 256 model: backbone: architecture: @@ -181,22 +291,22 @@ model: type: linear prediction_length: 1 support_length: 1 + execution: + context_length: 128 + target_offset: 1 + training_objective: causal dataset_training: events: model_interface: event_prediction parts: original: metadata_config_path: configs/metadata_configs/events.json + read_format: parquet increment: metadata_config_path: configs/metadata_configs/events-increment.json - criterion: - event: CrossEntropyLoss - loss_weights: - event: 1.0 - freeze: - backbone: - freeze_groups: - - attention.qkv + read_format: parquet + split: + validation_fraction: 0.2 training_plan: phases: - name: incremental_finetuning @@ -224,6 +334,35 @@ training_plan: evaluation: sources: - source: events + cadence: + epochs: 1 + batch_size: 64 + before_phase: true + on_snapshot: true + algorithm: supervised + model_selection: + handoff: last + checkpoints: + cadence: + epochs: 1 + learning: + window_stride: 1 + datasets: + events: + criterion: + event: CrossEntropyLoss + loss_weights: + event: 1.0 + trainability: + by_dataset: + events: + roles: + model: &id001 + freeze: + - component: backbone + exclude_modules: [] + semantic_groups: + - attention.qkv - name: complete_retraining epochs: 5 mode: interleaved @@ -251,16 +390,58 @@ training_plan: evaluation: sources: - source: events -export_generative_model: true -export_embedding_model: false -export_onnx: true -export_pt: false -export_with_dropout: false + cadence: + epochs: 1 + batch_size: 64 + before_phase: false + on_snapshot: true + algorithm: supervised + model_selection: + handoff: last + checkpoints: + cadence: + epochs: 1 + learning: + window_stride: 1 + datasets: + events: + criterion: + event: CrossEntropyLoss + loss_weights: + event: 1.0 + trainability: + by_dataset: + events: + roles: + model: *id001 +run: + project_root: . + model_name: event-model + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cuda + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: outer + recovery: {} + logging: {} + outputs: {} + export: + generative_model: true + embedding_model: false + onnx: true + pt: false + with_dropout: false + batch_size: 256 ``` -The historical flat training schema is not accepted. In particular, -`training_spec`, top-level dataset paths/columns, `model.ingestion`, -`model.decoder`, and architecture-owned freezing are not canonical fields. For a causal objective, `target_offset` sets the target position relative to each input position. With `target_offset: 1`, every decoded position predicts @@ -272,16 +453,11 @@ computes loss over all valid decoded positions, regardless of ## Ownership and resolution -- `global_training` owns objective, window, precision, distribution, - compilation, recovery checkpoint, and data-loader behavior. Phases own - budgets, batching, optimization, evaluation and epoch/transaction checkpoint cadence. -- `model` contains exactly one backbone and one or more named interfaces. - Different interface names create distinct ingestion and decoder weights; - repeated references to one name share those weights. -- `dataset_training` owns parts, criterion/weights, class-share logging, - freezing, and the interface reference. -- Preprocessing metadata owns split paths, data types, class counts, ID maps, - special-token IDs, normalization facts, and stored-window layout. +- `run` owns execution resources, reproducibility, recovery, logging, outputs and export. +- `model.execution` owns immutable window/objective/attention facts; named interfaces own architecture and input contracts. +- `dataset_training` owns sources, interface references and split policy. +- Phases own losses, trainability, optimization, scheduling, evaluation, model selection, stopping and snapshots. +- Preprocessing metadata owns frozen storage, codec and normalization facts; explicit assertions must agree. For categorical hard-label targets, set a target's criterion to `AccurateCrossEntropyLoss` to preserve small loss values and target-logit @@ -289,10 +465,14 @@ gradients at high confidence. `CrossEntropyLoss` retains the existing PyTorch behavior. For example: ```yaml -dataset_training: - events: - criterion: - event: AccurateCrossEntropyLoss +training_plan: + phases: + - name: fit + learning: + datasets: + events: + criterion: + event: AccurateCrossEntropyLoss ``` The choice is per target and is checked when resuming a run. Class @@ -402,18 +582,51 @@ ordinary sample shuffle is preserved. ```yaml training_plan: - curriculum_training: true - curriculum_column: difficulty phases: - epochs: 5 name: train mode: sequential + algorithm: supervised + model_selection: + handoff: last + checkpoints: + cadence: + epochs: 1 + learning: + datasets: + default: {} + curriculum: + column: difficulty dataset_training: default: parts: default: metadata_config_path: configs/metadata_configs/events.json file_order: name + read_format: parquet + split: + validation_fraction: 0.2 +run: + project_root: . + model_name: training + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: {} + logging: {} + outputs: {} + export: + batch_size: 1 ``` `loss_weights` scales each target's contribution to the training and reported @@ -488,44 +701,21 @@ valid with `CyclicLR`. ## Canonical training settings -Batching and optimization are configured only on phases. Global aliases for -`batch_size`, `accumulation_batches`, `learning_rate`, `optimizer`, `scheduler`, -`scheduler_step_on`, and `gradient_clip_norm` are rejected. CLI overrides use -`--global-training` and `--training-plan` YAML mappings with the same schema. +Batching, role-keyed optimization and trainability belong to phases. Evaluation +uses `evaluation.cadence: {epochs: N}` for supervised phases and +`{iterations: N}` for policy phases. Retained checkpoint cadence is +`checkpoints.cadence` with the same explicit units. Final progress always +evaluates when enabled and checkpoints. -Every phase owns evaluation and checkpoint cadence. Supervised phases use -`evaluation_interval_epochs` and `checkpoint_interval_epochs`; policy phases use -`evaluation_interval_iterations` and `checkpoint_interval_iterations`. A policy -iteration is one complete algorithm transaction and can contain several optimizer -steps. Final epochs/iterations always evaluate when enabled and save a checkpoint. -Supervised `evaluation` is an explicit declaration or `null`; boolean shorthand -and top-level evaluation are rejected. Multiple sources require an explicit monitor -for best handoff, validation saving, or early stopping. +Evaluation owns `before_phase`, `on_snapshot` and `batch_size`. Model selection +uses `model_selection.handoff: best | last`; stopping uses +`stopping: {patience: N, min_delta: 0, on_trigger: end_phase | end_plan}`. +Patience counts scheduled finite observations. Initial/snapshot evaluations can +select best weights without consuming patience. Best handoff, stopping and plateau +scheduling require an evaluation monitor. See [ownership](ownership.md#phase-settings). -```yaml -training_plan: - phases: - - name: fit - algorithm: supervised - epochs: 10 - mode: sequential - sources: - - source: observations.train - evaluation: - sources: - - source: observations.validation - monitor: - source: observations.validation - metric: loss - direction: min - evaluation_interval_epochs: 1 - checkpoint_interval_epochs: 2 -``` - -Epoch-based early stopping requires `evaluation_interval_epochs: 1`. -Time- and batch-based recovery checkpoints remain global execution settings. -Frozen normalization belongs to `global_training.normalization` or the incoming -artifact; phases cannot redeclare it. +Frozen normalization is an interface input contract or inherited artifact fact. +Recovery policy is `run.recovery`; phase snapshots belong to `checkpoints`. ## Model prediction parameters and policy actions @@ -606,18 +796,18 @@ training_plan: `optimizer_state: reset | continue` describes the transition **into** this phase; its default is `reset`. Reset creates new optimizer, scheduler and scaler state. Continue preserves these states from the immediately preceding phase and requires -that phase's `handoff: last`, identical algorithm, role configurations, parameter +that phase's `model_selection.handoff: last`, identical algorithm, role configurations, parameter ownership and freeze selections. The first phase cannot continue. Supervised continuation requires the same dataset ownership; policy continuation requires the same interface. Mixed plans reject continuation because their artifact handoffs do not preserve optimizer state. Policy OneCycleLR continuation is also rejected. Native checkpoints restore the active lifecycle in either mode. Supervised -`handoff: best` requires an evaluation monitor and selects the preceding phase’s +`model_selection.handoff: best` requires an evaluation monitor and selects the preceding phase’s best weights before starting the next phase; its monitor state is phase-local. Supervised `scheduler_step_on: epoch` advances once after a completed phase epoch; ReduceLROnPlateau receives the configured evaluation monitor at that boundary -and requires the phase's `evaluation.monitor` and `evaluation_interval_epochs: 1`. `optimizer_step` advances once per applied optimizer +and requires the phase's `evaluation.monitor` and `evaluation.cadence: {epochs: 1}`. `optimizer_step` advances once per applied optimizer update, after accumulation, rather than once per microbatch. Policy roles only accept `scheduler_step_on: optimizer_step`, their default. Policy plateau and compound schedulers are unsupported. Constructor arguments are validated before @@ -650,8 +840,8 @@ Early stopping, skipped updates or KL limits can consume less than the maximum. The authored schema is separate from resolved training phases. Checkpoints and plan fingerprints contain resolved roles, batching and scheduler budgets; -reproduction does not depend on global optimizer values. `global_training` -contains execution/distribution, resume, data/window and model contracts only. +reproduction does not depend on global optimizer values. Run execution policy and +durable model execution facts have separate owners; see [ownership](ownership.md). ## Training plans @@ -680,20 +870,17 @@ evaluation: direction: min ``` -Checkpoint interval fields retain distinct behavior. Phase epoch/iteration intervals, -global `snapshot_interval_batches`, and global `snapshot_interval_minutes` create persistent -epoch/batch snapshots and refresh the rolling `latest` checkpoint; -`latest_checkpoint_interval_minutes` refreshes only `latest`. -`evaluate_on_snapshot` controls whether timed or batch snapshots also run -validation and record its monitored loss. Every checkpoint is written after -flushing any partial gradient-accumulation window. +Phase `checkpoints.cadence` and snapshot triggers create retained checkpoints. +`run.recovery.latest_checkpoint_interval_minutes` refreshes rolling recovery. +`evaluation.on_snapshot` requests validation when a snapshot is due. Checkpoints +are written after flushing partial accumulation, or at a committed policy transaction. ## Composable YAML files An entry file may declare complementary fragments with `additional_config_paths`. Relative paths resolve against the entry file's `project_root`. Fragments can contribute disjoint children under containers -such as `global_training`, `model.interfaces`, and +such as `run`, `model.interfaces`, and `dataset_training`; duplicate fields are rejected. CLI overrides are applied after composition and before metadata resolution. @@ -853,7 +1040,7 @@ that one prefix position. ## Depth encoders and nested composites -Use `global_training.read_format: pt` for preprocessed depth inputs. Layout +Use `read_format: pt` on every prepared part for preprocessed depth inputs. Layout capacities come from dataset metadata. The following named model fragment combines one shallow branch with a depth encoder; its metadata comes from the preprocessing example in [preprocess.md](preprocess.md#named-depth-layouts). @@ -1045,47 +1232,36 @@ namespace. Omitted seeds preserve the existing RNG stream and initialization traversal. Initialization seeds do not control runtime dropout. Dropout continues using the run RNG and checkpointed per-rank state. -Dataset freezing can follow the same tree: +Trainability belongs to the phase for every algorithm. Both supervised and policy +roles use catalog selectors; supervised source-dependent updates use +`trainability.by_dataset..roles.model` with the same shape. ```yaml -dataset_training: - default: - criterion: - nextAction: CrossEntropyLoss - freeze: - backbone: - freeze_groups: - - attention.qkv - ingestion: - branches: - subitems: - freeze_except_groups: - - free_parameter - - ingestion.output_projection - item: - freeze_groups: [] - parts: - default: - metadata_config_path: configs/metadata_configs/raw-items.json +training_plan: + phases: + - name: adapt + trainability: + roles: + model: + freeze: + - {component: backbone, semantic_groups: [attention.qkv]} + - {module: interfaces.default.ingestion.branches.subitems, + exclude_semantic_groups: [free_parameter, ingestion.output_projection]} ``` -A local child selector replaces the inherited decision, including selective -unfreezing. `freeze_groups: []` unfreezes the scope; `freeze_except_groups: []` freezes it. -A node containing only `branches` delegates without changing inherited decisions. -Merge parameters retain their parent's policy; the adapter has its own dataset -selector. Unknown branches, policies below leaves, and contradictory alias -selections fail. Freezing leaves dropout active and permits gradients through -frozen modules. At optimizer boundaries, frozen gradients are removed **before** -AMP unscaling/overflow detection, so a frozen-only overflow cannot suppress a -healthy active update. Momentum and weight decay cannot update parameters whose -gradients are absent. +Role overrides replace the phase policy for that dataset; absence inherits it. +`freeze: []` masks nothing and `freeze_except: []` masks everything. Selectors +are strict and shared with optimizer routing. Fixed model capabilities cannot be +unfrozen by a phase mask. Gradients still traverse masked modules; their gradients +are removed before AMP checks and stepping, preventing momentum/weight decay +updates. See [parameter selection](ownership.md#parameter-selection). ## Policy and mixed plans All algorithms use `training_plan.phases`. Declare policy sequence/preference parts in `dataset_training`, and attach actions with `policy_bindings` keyed by interface. Run-wide settings remain in -`global_training`; no alternate run discriminator is needed. +`run` and `model.execution`; no alternate run discriminator is needed. See the complete [supervised template](../../src/sequifier/templates/train-supervised.yaml), [policy template](../../src/sequifier/templates/train-policy.yaml), and diff --git a/documentation/demos/self-contained-example.ipynb b/documentation/demos/self-contained-example.ipynb index 13a32a60..30f6d2a7 100644 --- a/documentation/demos/self-contained-example.ipynb +++ b/documentation/demos/self-contained-example.ipynb @@ -10,10 +10,11 @@ "import os\n", "import shutil\n", "import subprocess\n", - "import yaml\n", + "\n", "import numpy as np\n", "import pandas as pd\n", - "import torch" + "import torch\n", + "import yaml" ] }, { @@ -94,17 +95,16 @@ "outputs": [], "source": [ "preprocess_config = {\n", - " \"project_root\": PROJECT_ROOT,\n", " \"preprocessing_data_path\": file_path,\n", " \"read_format\": \"parquet\",\n", " \"write_format\": \"parquet\",\n", " \"selected_columns\": [\"targetToken\"],\n", - " \"split_ratios\": [0.8, 0.1, 0.1], # Train, Val, Test\n", - " \"window_lengths\": [11, 11, 11], # 10-token model context plus one future target\n", + " \"split_ratios\": [0.8, 0.1, 0.1],\n", + " \"window_lengths\": [11, 11, 11],\n", " \"max_target_offset\": 1,\n", - " \"window_stride\": 1, # Dense sampling\n", + " \"window_stride\": 1,\n", " \"max_rows\": None,\n", - " \"seed\": 42,\n", + " \"run\": {\"project_root\": PROJECT_ROOT, \"reproducibility\": {\"seed\": 42}},\n", "}\n", "\n", "pp_conf_path = os.path.join(CONFIG_DIR, \"preprocess.yaml\")\n", @@ -119,17 +119,6 @@ "outputs": [], "source": [ "train_config = {\n", - " \"project_root\": PROJECT_ROOT,\n", - " \"model_name\": \"demo-cycle-model\",\n", - " \"device\": \"cuda\" if torch.cuda.is_available() else \"cpu\",\n", - " \"global_training\": {\n", - " \"read_format\": \"parquet\",\n", - " \"training_objective\": \"causal\",\n", - " \"context_length\": 10,\n", - " \"inference_batch_size\": 10,\n", - " \"log_interval_batches\": 10000,\n", - " \"torch_compile\": \"none\", # Keep the demo independent of a host C++ toolchain.\n", - " },\n", " \"model\": {\n", " \"backbone\": {\n", " \"architecture\": {\n", @@ -139,7 +128,7 @@ " \"attention\": {\"num_heads\": 2},\n", " \"feed_forward\": {\"hidden_dim\": 32},\n", " \"dropout\": 0.0,\n", - " },\n", + " }\n", " },\n", " \"interfaces\": {\n", " \"default\": {\n", @@ -151,18 +140,20 @@ " \"feature_embedding_dims\": {\"targetToken\": 16},\n", " },\n", " \"decoder\": {\"type\": \"linear\", \"prediction_length\": 1},\n", - " },\n", + " }\n", " },\n", + " \"execution\": {\"context_length\": 10, \"training_objective\": \"causal\"},\n", " },\n", " \"dataset_training\": {\n", " \"default\": {\n", " \"parts\": {\n", " \"default\": {\n", - " \"metadata_config_path\": \"configs/metadata_configs/synthetic-sequences.json\"\n", - " },\n", + " \"metadata_config_path\": \"configs/metadata_configs/synthetic-sequences.json\",\n", + " \"read_format\": \"parquet\",\n", + " }\n", " },\n", - " \"criterion\": {\"targetToken\": \"CrossEntropyLoss\"},\n", - " },\n", + " \"split\": {\"validation_fraction\": 0.2},\n", + " }\n", " },\n", " \"training_plan\": {\n", " \"phases\": [\n", @@ -173,7 +164,6 @@ " \"mode\": \"sequential\",\n", " \"sources\": [{\"source\": \"default.default\"}],\n", " \"batch_size\": 32,\n", - " \"checkpoint_interval_epochs\": 5,\n", " \"optimization\": {\n", " \"model\": {\n", " \"learning_rate\": 0.001,\n", @@ -182,16 +172,47 @@ " \"name\": \"StepLR\",\n", " \"arguments\": {\"step_size\": 1, \"gamma\": 0.99},\n", " },\n", - " },\n", + " }\n", + " },\n", + " \"evaluation\": {\n", + " \"sources\": [{\"source\": \"default.default\"}],\n", + " \"cadence\": {\"epochs\": 1},\n", + " \"batch_size\": 32,\n", + " \"before_phase\": True,\n", + " \"on_snapshot\": True,\n", + " },\n", + " \"model_selection\": {\"handoff\": \"last\"},\n", + " \"checkpoints\": {\"cadence\": {\"epochs\": 5}},\n", + " \"learning\": {\n", + " \"datasets\": {\n", + " \"default\": {\"criterion\": {\"targetToken\": \"CrossEntropyLoss\"}}\n", + " }\n", " },\n", - " \"evaluation\": {\"sources\": [{\"source\": \"default.default\"}]},\n", " }\n", - " ],\n", + " ]\n", + " },\n", + " \"run\": {\n", + " \"project_root\": PROJECT_ROOT,\n", + " \"model_name\": \"demo-cycle-model\",\n", + " \"reproducibility\": {\"seed\": 1010, \"enforce_determinism\": False},\n", + " \"execution\": {\n", + " \"device\": \"cuda\" if torch.cuda.is_available() else \"cpu\",\n", + " \"distribution\": {\"strategy\": \"local\", \"world_size\": 1, \"backend\": \"nccl\"},\n", + " \"precision\": {},\n", + " \"data_loading\": {},\n", + " \"compilation\": {\"torch_compile\": \"none\"},\n", + " },\n", + " \"recovery\": {},\n", + " \"logging\": {\"log_interval_batches\": 10000},\n", + " \"outputs\": {},\n", + " \"export\": {\n", + " \"generative_model\": True,\n", + " \"embedding_model\": False,\n", + " \"onnx\": True,\n", + " \"pt\": False,\n", + " \"batch_size\": 10,\n", + " },\n", " },\n", - " \"export_generative_model\": True,\n", - " \"export_embedding_model\": False,\n", - " \"export_onnx\": True,\n", - " \"export_pt\": False,\n", "}\n", "\n", "tr_conf_path = os.path.join(CONFIG_DIR, \"train.yaml\")\n", @@ -206,21 +227,20 @@ "outputs": [], "source": [ "infer_config = {\n", - " \"project_root\": PROJECT_ROOT,\n", " \"task\": \"forecast\",\n", - " # Training exports best and last artifacts with the final epoch in the filename.\n", " \"model_paths\": [\"models/demo-cycle-model-best-5.onnx\"],\n", " \"training_config_path\": tr_conf_path,\n", " \"dataset\": \"default\",\n", - " # Use the 'test' split created during preprocessing\n", " \"data_path\": \"data/synthetic-sequences-split2.parquet\",\n", " \"write_format\": \"parquet\",\n", " \"output_probabilities\": False,\n", - " \"decode_categories\": True, # Map integer classes back to original values (e.g., \"1\", \"2\")\n", - " \"device\": \"cpu\",\n", - " \"inference_batch_size\": 10,\n", + " \"decode_categories\": True,\n", " \"autoregressive_generation\": True,\n", - " \"generation_steps\": 5, # Predict 5 steps into the future\n", + " \"generation_steps\": 5,\n", + " \"run\": {\n", + " \"project_root\": PROJECT_ROOT,\n", + " \"execution\": {\"device\": \"cpu\", \"batch_size\": 10},\n", + " },\n", "}\n", "\n", "inf_conf_path = os.path.join(CONFIG_DIR, \"infer.yaml\")\n", @@ -308,7 +328,7 @@ " sequence_id_change | correct_increment | (last_was_five & is_one)\n", ").mean()\n", "\n", - "print(f\"{mean_accuracy*100:.2f}% of generated tokens follow the cycle rule\")" + "print(f\"{mean_accuracy * 100:.2f}% of generated tokens follow the cycle rule\")" ] }, { diff --git a/documentation/example_inputs/configs/import-policy-sequences.yaml b/documentation/example_inputs/configs/import-policy-sequences.yaml index 15b0babf..f9b14741 100644 --- a/documentation/example_inputs/configs/import-policy-sequences.yaml +++ b/documentation/example_inputs/configs/import-policy-sequences.yaml @@ -1,5 +1,4 @@ operation: import_sequences -project_root: "." source: paths: - data/demonstrations.csv @@ -17,3 +16,5 @@ policy_binding: values: - left - right +run: + project_root: . diff --git a/documentation/example_inputs/configs/infer-policy.yaml b/documentation/example_inputs/configs/infer-policy.yaml index 92fcd94e..fb5c102f 100644 --- a/documentation/example_inputs/configs/infer-policy.yaml +++ b/documentation/example_inputs/configs/infer-policy.yaml @@ -1,11 +1,14 @@ -project_root: "." data_path: data/policy-observations.csv read_format: csv write_format: csv -device: cpu desired_return: 10.0 -enforce_determinism: true action_selection: deterministic task: policy model_paths: - models/policy/model.pt +run: + project_root: . + execution: + device: cpu + reproducibility: + enforce_determinism: true diff --git a/documentation/example_inputs/configs/infer-ppo-toy.yaml b/documentation/example_inputs/configs/infer-ppo-toy.yaml index 6d331b65..42683d30 100644 --- a/documentation/example_inputs/configs/infer-ppo-toy.yaml +++ b/documentation/example_inputs/configs/infer-ppo-toy.yaml @@ -1,11 +1,14 @@ -project_root: "." data_path: documentation/example_inputs/ppo-observations.csv read_format: csv write_format: csv -device: cpu output_path: outputs/ppo-actions.csv -enforce_determinism: true action_selection: deterministic task: policy model_paths: - models/online-ppo/model.pt +run: + project_root: . + execution: + device: cpu + reproducibility: + enforce_determinism: true diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index 8959d470..3216fb08 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -1,8 +1,6 @@ -project_root: "." data_path: documentation/example_inputs/ppo-observations.csv read_format: csv write_format: csv -device: cpu rollout: trajectory_path: outputs/ppo-rollout.json seed_row_index: 0 @@ -13,8 +11,13 @@ rollout: config: horizon: 8 implementation_id: sequifier.categorical-control.v1 -enforce_determinism: true action_selection: deterministic task: policy model_paths: - models/online-ppo/model.pt +run: + project_root: . + execution: + device: cpu + reproducibility: + enforce_determinism: true diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml index 5385b85f..7e2595a3 100644 --- a/documentation/example_inputs/configs/train-bc-toy.yaml +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: toy-bc -device: cpu -seed: 1010 model: backbone: architecture: @@ -32,9 +28,8 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 16 - validation_fraction: 0.2 + execution: + context_length: 16 dataset_training: demonstrations: parts: @@ -43,6 +38,8 @@ dataset_training: paths: - data/offline-transitions.csv model_interface: actor + split: + validation_fraction: 0.2 training_plan: phases: - name: behavior_cloning @@ -51,6 +48,17 @@ training_plan: - source: demonstrations.recorded epochs: 30 batch_size: 64 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 64 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -64,3 +72,29 @@ policy_bindings: values: - left - right +run: + project_root: . + model_name: toy-bc + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: {} + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index fa19f4c4..56a5f498 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: offline-iql -device: cpu -seed: 1010 model: backbone: architecture: @@ -34,11 +30,8 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 16 - validation_fraction: 0.2 - resume: - policy: if_exists + execution: + context_length: 16 dataset_training: demonstrations: parts: @@ -58,6 +51,8 @@ dataset_training: position: next_position velocity: next_velocity model_interface: actor + split: + validation_fraction: 0.2 training_plan: phases: - name: offline_improvement @@ -71,8 +66,17 @@ training_plan: max_advantage_weight: 100.0 target_update_rate: 0.005 iterations: 10000 - evaluation_interval_iterations: 100 - checkpoint_interval_iterations: 100 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 100 + evaluation: + cadence: + iterations: 100 + batch_size: 64 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -86,3 +90,31 @@ policy_bindings: values: - left - right +run: + project_root: . + model_name: offline-iql + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: + resume: + policy: if_exists + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml index e2cdd27f..485b3127 100644 --- a/documentation/example_inputs/configs/train-iql-toy.yaml +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: toy-iql -device: cpu -seed: 1010 initialize_from: models/toy-bc/actor.pt model: backbone: @@ -33,11 +29,8 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 16 - validation_fraction: 0.2 - resume: - policy: if_exists + execution: + context_length: 16 dataset_training: demonstrations: parts: @@ -53,6 +46,8 @@ dataset_training: successors: position: next_position model_interface: actor + split: + validation_fraction: 0.2 training_plan: phases: - name: offline_improvement @@ -61,8 +56,17 @@ training_plan: - source: demonstrations.recorded batch_size: 64 iterations: 10000 - evaluation_interval_iterations: 100 - checkpoint_interval_iterations: 100 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 100 + evaluation: + cadence: + iterations: 100 + batch_size: 64 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -76,3 +80,31 @@ policy_bindings: values: - left - right +run: + project_root: . + model_name: toy-iql + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: + resume: + policy: if_exists + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index e675ecec..03030a62 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: multiphase-control -device: cpu -seed: 1010 model: backbone: architecture: @@ -33,10 +29,11 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 4 - normalization: - source: identity + input_contract: + normalization: + source: identity + execution: + context_length: 4 dataset_training: decisions: parts: @@ -54,11 +51,15 @@ dataset_training: cue: next_cue group: contextId model_interface: actor + split: + validation_fraction: 0.2 rankings: parts: default: type: preferences manifest_path: data/control-preferences.json + split: + validation_fraction: 0.2 training_plan: phases: - name: imitation @@ -66,17 +67,50 @@ training_plan: sources: - source: decisions.default epochs: 4 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 32 + before_phase: false + on_snapshot: false - name: offline_improvement algorithm: iql sources: - source: decisions.default iterations: 100 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 100 + evaluation: + cadence: + iterations: 100 + batch_size: 64 + before_phase: false + on_snapshot: false - name: preferences algorithm: dpo preferences: source: rankings.default beta: 0.1 iterations: 100 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 1 + before_phase: false + on_snapshot: false - name: group_improvement algorithm: grpo total_groups: 32 @@ -104,6 +138,16 @@ training_plan: monitor: metric: score_mean direction: max + cadence: + iterations: 1 + batch_size: 1 + before_phase: false + on_snapshot: false + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 - name: ppo_improvement algorithm: ppo environment: @@ -131,9 +175,17 @@ training_plan: - 71 max_episode_steps: 8 timeout_seconds: 30 + cadence: + iterations: 2 + batch_size: 1 + before_phase: false + on_snapshot: false batch_size: 64 - evaluation_interval_iterations: 2 - checkpoint_interval_iterations: 2 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 2 policy_bindings: actor: previous_action_inputs: @@ -147,3 +199,29 @@ policy_bindings: values: - left - right +run: + project_root: . + model_name: multiphase-control + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: {} + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml index 6d76be98..7e637a97 100644 --- a/documentation/example_inputs/configs/train-policy-from-table.yaml +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: table-policy -device: cpu -seed: 1010 model: backbone: architecture: @@ -34,9 +30,8 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 16 - validation_fraction: 0.2 + execution: + context_length: 16 dataset_training: demonstrations: parts: @@ -45,6 +40,8 @@ dataset_training: paths: - data/demonstrations.parquet model_interface: actor + split: + validation_fraction: 0.2 training_plan: phases: - name: behavior_cloning @@ -56,6 +53,17 @@ training_plan: optimization: actor: learning_rate: 0.005 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 64 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -69,3 +77,29 @@ policy_bindings: values: - left - right +run: + project_root: . + model_name: table-policy + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: {} + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index c0a350b6..de7b565a 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: policy -device: cpu -seed: 1010 model: backbone: architecture: @@ -36,13 +32,8 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -report_path: outputs/policy/training-report.json -global_training: - context_length: 4 - validation_fraction: 0.2 - episode_cache_size: 8 - resume: - policy: never + execution: + context_length: 4 dataset_training: demonstrations: parts: @@ -51,6 +42,8 @@ dataset_training: paths: - data/demonstrations.parquet model_interface: actor + split: + validation_fraction: 0.2 training_plan: phases: - name: offline_policy @@ -62,6 +55,17 @@ training_plan: optimization: actor: learning_rate: 0.005 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 64 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -78,4 +82,34 @@ policy_bindings: values: - 0.0 - 1.0 -model_path: models/policy/actor.pt +run: + project_root: . + model_name: policy + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: + episode_cache_size: 8 + compilation: + torch_compile: none + recovery: + resume: + policy: never + logging: {} + outputs: + model_path: models/policy/actor.pt + report_path: outputs/policy/training-report.json + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/documentation/learning-architecture.md b/documentation/learning-architecture.md index b2a8664e..8d8a19a5 100644 --- a/documentation/learning-architecture.md +++ b/documentation/learning-architecture.md @@ -8,14 +8,14 @@ and [inference reference](configs/infer.md) for the supported configuration surf | Owner | Contract | | --- | --- | | Model execution | Named interfaces, forward attention, context geometry, codecs, normalization, input schemas, decoder order and prediction scales. These survive changes of learning objective. | -| Authored run | Model or initialization artifact, datasets, policy bindings, ordered phases, output paths and global execution settings. | -| Phase | Algorithm budget, batching, trainable roles, optimizer/scheduler settings, evaluation, freezing and handoff selection. | +| Authored run | Model or initialization artifact, datasets, policy bindings, ordered phases, output paths and run execution settings. | +| Phase | Algorithm budget, batching, trainable roles, optimizer/scheduler settings, evaluation, trainability, stopping and model-selection handoff. | | Sources | Prepared histories, raw sequences and preference manifests with explicit timing, episode boundaries and optional transition capabilities. | | Native runner | Algorithm-specific sampling, objectives, critics, environment state and in-phase recovery. | | Coordinator | Dependency validation, incoming/selected artifacts, phase boundaries, receipts and reports. | `ModelExecution` exposes execution facts without assigning them dataset ownership. -Compatibility adapters retain existing resolved training objects and artifact formats. +Resolution constructs the execution objects used by model loading and native runners. Policy bindings select existing targets; lagged actions and return conditioning are explicit inputs. Categorical support, real scales/bounds/atoms and conditional activity have the same meaning during training, collection, replay and inference. @@ -36,4 +36,4 @@ Phase handoffs publish selected weights before committing progress. Checkpoints retain optimizer/sampler/RNG state and validate representation, source, adapter and reference identities. Mixed plans use native checkpoints inside an atomic run envelope. Existing checkpoint formats, public configuration aliases, standalone -scaffold templates and portable model formats remain compatibility boundaries. +scaffold templates and portable model formats describe the supported contracts. diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 90346719..b2a7b97e 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -12,9 +12,11 @@ actions and mixed supervised/policy plans. ## Authoring and optimization `policy_bindings` maps interface names to action declarations, including runs -with one interface. `global_training` owns context length, frozen normalization, validation -fraction, episode cache size, determinism and resume. Phases own batching, -`optimization.`, evaluation/checkpoint intervals and `handoff: last | best`. +with one interface. `model.execution` owns context geometry; interface input +contracts own frozen normalization; datasets own raw split policy. `run` owns +resources, reproducibility and recovery. Phases own learning, trainability, +optimization, evaluation, stopping, snapshots and `model_selection.handoff`. +See the [ownership reference](configs/ownership.md). | Algorithm | Required budget/data | Trainable roles | | --- | --- | --- | @@ -31,7 +33,7 @@ on optimizer steps; plateau/compound schedulers are unsupported. `OneCycleLR.tot an explicitly supplied value must match that budget. Unknown roles fail validation. Imitation, DPO and GRPO default to `batch_size: 32`, `accumulation_batches: 1`, -`evaluation_interval_iterations: 1` and `checkpoint_interval_iterations: 1`. Accumulation uses actual +`evaluation.cadence: {iterations: 1}` and `checkpoints.cadence: {iterations: 1}`. Accumulation uses actual example counts, including partial batches. Network stochastic layers are disabled while gradients remain enabled. Best handoff keeps the earliest finite best on ties; last handoff keeps the latest transaction. Final updates always evaluate. diff --git a/documentation/training/multi-gpu-training.md b/documentation/training/multi-gpu-training.md index c2fb0ae4..e3aee8ee 100644 --- a/documentation/training/multi-gpu-training.md +++ b/documentation/training/multi-gpu-training.md @@ -29,18 +29,18 @@ write_format: pt Once your data is preprocessed into `.pt` shards, or beta `.parquet` shards, you need to tell the Sequifier training engine to expect a distributed environment. -In your `train.yaml`, configure the canonical `global_training` block: +In your `train.yaml`, configure `run.execution.distribution` and each prepared part’s `read_format`: ```yaml -global_training: - read_format: pt - distributed: true - data_parallelism: fsdp - fsdp_cpu_offload: false - layer_type_dtypes: null - torch_compile: inner - world_size: 32 - backend: nccl +run: + execution: + device: cuda + distribution: {strategy: fsdp, cpu_offload: false, world_size: 32, backend: nccl} + compilation: {torch_compile: inner} +dataset_training: + events: + parts: + train: {read_format: pt, metadata_config_path: metadata_configs/events.json} ``` When shards do not divide evenly across ranks, Sequifier automatically pads shorter ranks with repeated samples for step alignment. Those repeats are masked out of loss calculation, so each real sample contributes once. diff --git a/documentation/training/runtime-architecture.md b/documentation/training/runtime-architecture.md index e36308f0..7f56b201 100644 --- a/documentation/training/runtime-architecture.md +++ b/documentation/training/runtime-architecture.md @@ -62,7 +62,7 @@ New portable model artifacts use format version 2; version 1 flat models remain readable. Tensor payload versions and stored-window versions remain separate. Run checkpoints record initialization-policy and seed-derivation versions in addition to resolved branch policies and the existing optimizer/scaler/RNG state. -Inactive legacy layout/payload fields are normalized in resume comparisons. +Resume comparisons use the current layout and payload fields. ## Export preflight and publication diff --git a/release-notes/unreleased-canonical-config-names.md b/release-notes/unreleased-canonical-config-names.md index c64aa562..e117e4e8 100644 --- a/release-notes/unreleased-canonical-config-names.md +++ b/release-notes/unreleased-canonical-config-names.md @@ -1,5 +1,6 @@ # Canonical public configuration names + - Use named training interfaces, datasets, parts, bindings, and phases for every cardinality. Remove singleton aliases and boolean evaluation shorthand. - Standardize dimensions, counts, batch/optimizer/iteration clocks, selectors, diff --git a/release-notes/unreleased-configuration-ownership.md b/release-notes/unreleased-configuration-ownership.md index ac25aa8e..94feff63 100644 --- a/release-notes/unreleased-configuration-ownership.md +++ b/release-notes/unreleased-configuration-ownership.md @@ -1,20 +1,11 @@ # Configuration ownership -- Give every algorithm phase its own evaluation declaration, monitor and cadence. - Supervised phases also own epoch checkpoint cadence through `checkpoint_interval`. -- Separate model-owned Gaussian scales (`initial`, `trainable`, `min`, `max`) from - policy action behavior. Real actions select `scale: model` or `scale: {fixed: ...}`; - action bounds, point masses and conditional activity remain task declarations. -- Preserve model scale parameters when a policy uses fixed exploration. Explicit - model declarations may add new scale parameters to an initialized artifact. -- Keep frozen normalization global or inherited from an artifact; remove phase copies. -- Derive OneCycleLR budgets for supervised and policy training. Explicit `total_steps` - values are assertions checked against the derived budget. -- Remove global optimization aliases, policy inference/evaluation aliases and rollout - seed aliases. CLI overrides use canonical `--global-training`/`--training-plan` - mappings. Scalar action mappings use `actions`/`previous_actions`; structured - records use `action_record`/`previous_action_record`. +- Replace authored `global_training` and root resources with explicit `run`, durable `model.execution` and named interface input contracts. +- Keep dataset declarations focused on source identity and split policy. Move losses, metrics, curriculum and trainability to phases. +- Share typed catalog parameter selectors between optimizer routing and every algorithm's role trainability; supervised phases retain source-dependent masks. +- Put evaluation cadence, initial/snapshot triggers and batching on each phase. Keep retained snapshots phase-owned and rolling recovery run-owned. +- Add observation-based stopping with explicit `end_phase` or `end_plan` scope across native and mixed runners, with checkpointed patience and stop state. +- Update CLI mappings, search paths, preprocessing/inference resource ownership, scaffold templates, examples, notebooks and documentation. +- Bump native recovery formats because phase ownership and stopping state changed; portable execution artifacts remain usable for initialization/inference. -No configuration migration or compatibility layer is provided. Existing configuration -files and tests are intentionally left unchanged by this refactor. No tests were -written or run for this change. +No tests were written or run for this implementation, at the user's request. Validation is limited to static source inspection, syntax, lint and type analysis. diff --git a/release-notes/unreleased-phase-ownership.md b/release-notes/unreleased-phase-ownership.md index 3173df00..90a2335a 100644 --- a/release-notes/unreleased-phase-ownership.md +++ b/release-notes/unreleased-phase-ownership.md @@ -18,8 +18,9 @@ Forecast version-1 bindings and policy version-3 environments remain distinct. Document forecast `seed_window_index` versus policy `seed_row_index` units. - Update shipped examples, scaffolding, Python examples and consolidated docs. - CPU smoke tests cover native supervised and all policy algorithms, mixed - handoff/recovery, inference, rollout and configuration rejection. + CPU smoke coverage is required before release for native supervised and policy algorithms, mixed + handoff/recovery, inference, rollout and configuration rejection. No tests were written or run for the ownership implementation. + All algorithms own evaluation on their phase. Offline policy evaluation uses `evaluation.sources` and `evaluation.monitor: {metric, mode}` without aliases. diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 223c1ee2..5bca16e5 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -5,11 +5,11 @@ from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.train_config import ( BackboneComponentConfig, - GlobalTrainingSpecModel, ModelInterfaceSpecModel, ModelSpecModel, ResolvedModelInterface, ResolvedSequifierConfig, + TrainingRuntimeSettings, ) from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view from sequifier.special_tokens import SPECIAL_TOKEN_IDS @@ -105,12 +105,7 @@ def resolved_config_from_model_config( context_length = int(values["context_length"]) target_offset = int(values.get("target_offset", 1)) objective = str(values["training_objective"]) - global_spec = GlobalTrainingSpecModel( - read_format=( - "pt" - if any(i.get("depth_layouts") for i in interface_values.values()) - else "parquet" - ), + global_spec = TrainingRuntimeSettings( training_objective=objective, forward_attention=values.get("forward_attention"), context_length=context_length, @@ -213,11 +208,11 @@ def resolved_config_from_model_config( return config, interface_name -def execution_config(model, interfaces, global_training, *, device="cpu", **options): +def execution_config(model, interfaces, runtime_settings, *, device="cpu", **options): """Adapt model execution to the existing public resolved training contract.""" from sequifier.model.execution import ModelExecution - return ModelExecution(model, interfaces, global_training).training_config( + return ModelExecution(model, interfaces, runtime_settings).training_config( device=device, **options ) diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index fdfaf280..d3039062 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -26,7 +26,7 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: }, ) values["execution_schema"] = ExecutionSchema.from_interface( - interface, training_config.global_training.context_length + interface, training_config.runtime_settings.context_length ).to_dict() values["depth_layouts"] = interface.depth_layouts.model_dump(mode="json") interfaces[interface.name] = values @@ -34,7 +34,7 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: interfaces[interface.name]["policy"] = interface.policy.model_dump( mode="json" ) - spec = training_config.global_training + spec = training_config.runtime_settings return { "training_objective": spec.training_objective, "forward_attention": spec.forward_attention diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 48e64e5d..a8e99e69 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -12,7 +12,7 @@ from sequifier.artifacts.model_artifact import ModelArtifact from sequifier.io.atomic import atomic_save -RUN_CHECKPOINT_FORMAT_VERSION = 2 +RUN_CHECKPOINT_FORMAT_VERSION = 3 @dataclass(frozen=True) @@ -78,6 +78,10 @@ def state_dict(self) -> dict[str, Any]: def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": if payload.get("artifact_type") != "sequifier_run_checkpoint": raise ValueError("Unsupported checkpoint: expected new Sequifier format.") + if payload.get("format_version") != RUN_CHECKPOINT_FORMAT_VERSION: + raise ValueError( + f"unsupported native checkpoint format {payload.get('format_version')}; expected {RUN_CHECKPOINT_FORMAT_VERSION}. Initialize from its portable model artifact to start a new run." + ) from sequifier.config.train_config import ResolvedSequifierConfig if ( @@ -146,7 +150,7 @@ def write_run_checkpoint( def checkpoint_path(training_config: Any) -> Path: - resume = training_config.global_training.resume + resume = training_config.runtime_settings.resume configured_path = resume.checkpoint_path if resume is not None else None if configured_path: path = Path(configured_path) @@ -164,7 +168,7 @@ def checkpoint_path(training_config: Any) -> Path: def select_run_checkpoint(training_config: Any) -> Path | None: - resume = training_config.global_training.resume + resume = training_config.runtime_settings.resume if resume is None or resume.policy == "never": return None path = checkpoint_path(training_config) diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index 979f6c34..2b680962 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -285,7 +285,7 @@ def _component_discriminator(path: ConfigPath) -> str | None: return "name" if path and path[-1] in {"weight", "bias"} and "initialization" in path: return "method" - if path == ("global_training", "bert_spec", "span_masking"): + if path == ("runtime_settings", "bert_spec", "span_masking"): return "type" if len(path) < 4 or path[:2] != ("model", "interfaces"): return None @@ -555,14 +555,6 @@ def _compile_value(expression: Any, base: Any, path: ConfigPath) -> _CompiledVal f"{_path_name(path)}.$range requires low, high and optional step/log/type" ) return _distribution_space(path, spec, base) - if not isinstance(base, dict): - for old, new in ( - ("choices", "$choices"), - ("fixed", "$fixed"), - ("variants", "$variants"), - ): - if old in expression: - raise ValueError(f"{_path_name(path)}.{old} was removed; use {new}") if _is_distribution(expression) and not isinstance(base, dict): raise ValueError( f"{_path_name(path)}: wrap numeric distributions in $range" @@ -667,7 +659,9 @@ class CanonicalHyperparameterSearchConfig(ConfigModel): @model_validator(mode="after") @beartype def validate_search_controls(self): - reserved_parameters = {"model_name", "project_root"} & set(self.parameters) + reserved_parameters = {"model_name", "project_root"} & set( + self.parameters.get("run", {}) + ) if reserved_parameters: raise ValueError( "Search parameters cannot set run-controlled " @@ -798,9 +792,11 @@ def _materialized_values(self, trial: Any | None, run_index: int) -> dict[str, A "range_concat", }: architecture["position_encoding_scope"] = "global" - values["project_root"] = self.project_root + values["run"]["project_root"] = self.project_root trial_count = self.trials if self.trials is not None else self.grid_size() - values["model_name"] = f"{self.name}-run-{run_index:0{len(str(trial_count))}d}" + values["run"]["model_name"] = ( + f"{self.name}-run-{run_index:0{len(str(trial_count))}d}" + ) return values @beartype @@ -869,13 +865,13 @@ def compile_canonical_hyperparameter_search_config( search_values = copy.deepcopy(config_values) search_values["base_config_path"] = base_config_path - search_values.setdefault("project_root", base_config.project_root) + search_values.setdefault("project_root", base_config.run.project_root) try: base_values = base_config.model_dump(mode="python") search_config = CanonicalHyperparameterSearchConfig.model_validate( search_values ) - base_values["project_root"] = search_config.project_root + base_values["run"]["project_root"] = search_config.project_root search_config._compiled_config = _compile_value( search_config.parameters, base_values, diff --git a/src/sequifier/config/command_ownership.py b/src/sequifier/config/command_ownership.py new file mode 100644 index 00000000..7c1e5987 --- /dev/null +++ b/src/sequifier/config/command_ownership.py @@ -0,0 +1,69 @@ +"""Shared run settings and CLI overrides for command configurations.""" + +from copy import deepcopy + +from pydantic import Field + +from sequifier.config.ownership import ( + ExecutionSettings, + ReproducibilitySettings, + Settings, +) + + +class CommandExecutionSettings(ExecutionSettings): + batch_size: int = Field(default=1, gt=0) + + +class CommandRunSettings(Settings): + project_root: str = "." + execution: CommandExecutionSettings = Field( + default_factory=CommandExecutionSettings + ) + reproducibility: ReproducibilitySettings = Field( + default_factory=ReproducibilitySettings + ) + + def validate_for(self, command): + execution = self.execution + defaults = CommandExecutionSettings() + for field in ("distribution", "precision", "compilation", "max_concat_tensors"): + if getattr(execution, field) != getattr(defaults, field): + raise ValueError(f"{command} does not support run.execution.{field}") + loading = execution.data_loading + if command == "infer" and loading != defaults.data_loading: + raise ValueError("inference does not support data_loading controls") + if command == "preprocess": + if ( + execution.device != "cpu" + or execution.batch_size != 1 + or self.reproducibility.enforce_determinism + ): + raise ValueError( + "preprocessing supports run execution workers and a seed; device/batching/determinism controls are unsupported" + ) + other = loading.model_dump(exclude={"num_workers"}) + if other != defaults.data_loading.model_dump(exclude={"num_workers"}): + raise ValueError("preprocessing only supports data_loading.num_workers") + return self + + +def apply_run_overrides(values, overrides): + """CLI scalar conveniences write the same canonical paths as YAML.""" + values, overrides = deepcopy(values), dict(overrides) + run = values.setdefault("run", {}) + for key in ("project_root", "model_name"): + if key in overrides and overrides[key] is not None: + run[key] = overrides.pop(key) + if "seed" in overrides and overrides["seed"] is not None: + run.setdefault("reproducibility", {})["seed"] = overrides.pop("seed") + if "device" in overrides and overrides["device"] is not None: + run.setdefault("execution", {})["device"] = overrides.pop("device") + if ( + "enforce_determinism" in overrides + and overrides["enforce_determinism"] is not None + ): + run.setdefault("reproducibility", {})["enforce_determinism"] = overrides.pop( + "enforce_determinism" + ) + return values, {key: value for key, value in overrides.items() if value is not None} diff --git a/src/sequifier/config/components.py b/src/sequifier/config/components.py index 7c2bbaf1..778d1643 100644 --- a/src/sequifier/config/components.py +++ b/src/sequifier/config/components.py @@ -1082,7 +1082,7 @@ class ResumeConfig(ConfigModel): def validate_checkpoint_path(self): if self.policy == "required" and not self.checkpoint_path: raise ValueError( - "global_training.resume.checkpoint_path is required when " + "run.recovery.resume.checkpoint_path is required when " "policy is 'required'" ) return self diff --git a/src/sequifier/config/composition.py b/src/sequifier/config/composition.py index f472f18c..3821734f 100644 --- a/src/sequifier/config/composition.py +++ b/src/sequifier/config/composition.py @@ -21,7 +21,7 @@ DEFAULT_ATOMIC_PATHS: frozenset[ConfigPath] = frozenset( { ("model", "backbone"), - ("global_training", "bert_spec", "span_masking"), + ("training_plan", "phases", "learning", "bert_spec", "span_masking"), } ) @@ -95,10 +95,14 @@ def load_composed_yaml_config( if not additional_paths: return entry_values - project_root = entry_values.get("project_root") + project_root = ( + entry_values.get("project_root") + if "base_config_path" in entry_values + else entry_values.get("run", {}).get("project_root", ".") + ) if not isinstance(project_root, str) or not project_root.strip(): raise ValueError( - f"Config '{entry_path}' must define a non-empty string project_root " + f"Config '{entry_path}' must define a non-empty run.project_root " f"when {ADDITIONAL_CONFIG_PATHS_KEY} is configured." ) diff --git a/src/sequifier/config/freezing_config.py b/src/sequifier/config/freezing_config.py deleted file mode 100644 index ffae7899..00000000 --- a/src/sequifier/config/freezing_config.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Typed component fields for semantic layer freezing.""" - -from typing import Optional - -from pydantic import ( - ConfigDict, - Field, - field_validator, - model_serializer, - model_validator, -) - -from sequifier.config.layer_groups import LayerGroup -from sequifier.config.naming import ConfigModel -from sequifier.typechecking import beartype - - -class LayerFreezingConfigFields(ConfigModel): - """Mutually exclusive semantic layer-freezing selections.""" - - model_config = ConfigDict(extra="forbid") - - freeze_groups: Optional[list[LayerGroup]] = None - freeze_except_groups: Optional[list[LayerGroup]] = None - - @field_validator("freeze_groups", "freeze_except_groups") - @classmethod - @beartype - def validate_unique_groups( - cls, value: Optional[list[LayerGroup]] - ) -> Optional[list[LayerGroup]]: - if value is not None and len(value) != len(set(value)): - raise ValueError("layer freezing groups cannot contain duplicates") - return value - - @model_validator(mode="after") - @beartype - def validate_mutually_exclusive(self): - if self.freeze_groups is not None and self.freeze_except_groups is not None: - raise ValueError( - "freeze_groups and freeze_except_groups are mutually exclusive; " - "only one may be non-null" - ) - return self - - @model_serializer(mode="wrap") - @beartype - def serialize_freezing_fields(self, serializer): - """Omit inactive fields from canonical dataset freezing policies.""" - - values = serializer(self) - if self.freeze_groups is None: - values.pop("freeze_groups", None) - if self.freeze_except_groups is None: - values.pop("freeze_except_groups", None) - return values - - @property - @beartype - def has_freezing_policy(self) -> bool: - return self.freeze_groups is not None or self.freeze_except_groups is not None - - -class IngestionFreezingConfig(LayerFreezingConfigFields): - """Recursive policies following the declared composite branch tree.""" - - branches: dict[str, "IngestionFreezingConfig"] = Field(default_factory=dict) - - @property - def has_freezing_policy(self) -> bool: - return super().has_freezing_policy or any( - child.has_freezing_policy for child in self.branches.values() - ) - - @model_serializer(mode="wrap") - def serialize_ingestion_policy(self, serializer): - values = serializer(self) - if self.freeze_groups is None: - values.pop("freeze_groups", None) - if self.freeze_except_groups is None: - values.pop("freeze_except_groups", None) - if not self.branches: - values.pop("branches", None) - return values diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index b97d9033..61fe93fa 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -135,7 +135,7 @@ def _execution_source( training: Any, dataset: Any ) -> tuple[dict[str, Any], DatasetMetadata]: interface = dataset.interface - global_spec = training.global_training + global_spec = training.runtime_settings layout = interface.storage_layout return ( { @@ -397,7 +397,11 @@ def load_inferer_config( cli_values = { key: value for key, value in args_config.items() if key != "skip_metadata" } - authored_values = merge_config_fragments((config_values, cli_values)) + from sequifier.config.command_ownership import apply_run_overrides + + config_values, cli_values = apply_run_overrides(config_values, cli_values) + canonical_values = merge_config_fragments((config_values, cli_values)) + authored_values = canonical_values artifacts = {} policy_config = policy_inference_config(authored_values, artifacts) if policy_config is not None: @@ -405,13 +409,14 @@ def load_inferer_config( raise ValueError("skip_metadata is not supported for policy inference") return policy_config forecast_values, _ = extract_inline_metadata(authored_values) - authored_values["seed"] = InferenceConfig.model_validate(forecast_values).seed + request = InferenceConfig.model_validate(forecast_values) + authored_values["run"] = request.run.model_dump(mode="python", exclude_unset=True) authored_values, extracted_metadata_values = extract_inline_metadata( authored_values ) inline_metadata_values = extracted_metadata_values if skip_metadata else None - project_root = authored_values.get("project_root", ".") + project_root = request.project_root selected_metadata = None selected_metadata_path = None artifact_metadata_source = None @@ -616,7 +621,7 @@ def _effective_metadata_config_path(config: "InferenceConfig") -> Optional[str]: @beartype def resolve_inference_config( - config: "InferenceConfig", + config: Any, metadata: DatasetMetadata, *, allow_wider_artifact_folder: bool = False, @@ -1229,7 +1234,7 @@ def validate_autoregressive(cls, v: bool, info: Any): def validate_data_path(cls, v: Optional[str], info: Any) -> Optional[str]: if v is None: return v - v2 = normalize_path(v, info.data.get("project_root")) + v2 = normalize_path(v, info.data["run"].project_root) if not os.path.exists(v2): raise ValueError(f"{v2} does not exist") return v diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index 2d885e53..dae255e5 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -4,33 +4,59 @@ from dataclasses import dataclass from typing import Literal -from pydantic import ConfigDict, Field, field_validator +from pydantic import ConfigDict, Field, field_validator, model_validator -from sequifier.config.naming import ConfigModel, RunSeed +from sequifier.config.command_ownership import CommandRunSettings +from sequifier.config.naming import ConfigModel class InferenceExecutionConfig(ConfigModel): model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) - project_root: str = "." + run: CommandRunSettings = Field(default_factory=CommandRunSettings) model_interface: str | None = None read_format: str = "parquet" write_format: str = "csv" - device: str = "cpu" - seed: RunSeed = 1010 - enforce_determinism: bool = False - inference_batch_size: int = Field(default=1, gt=0) + + @property + def project_root(self): + return self.run.project_root + + @property + def device(self): + return self.run.execution.device + + @property + def seed(self): + return self.run.reproducibility.seed + + @property + def enforce_determinism(self): + return self.run.reproducibility.enforce_determinism + + @property + def inference_batch_size(self): + return self.run.execution.batch_size + + @model_validator(mode="after") + def validate_resources(self): + self.run.validate_for("infer") + return self class InferenceRouteConfig(ConfigModel): """Validate artifact and interface selection before route-specific options.""" model_config = ConfigDict(extra="ignore") - project_root: str = "." + run: CommandRunSettings = Field(default_factory=CommandRunSettings) model_paths: list[str] = Field(min_length=1) model_interface: str | None = None task: Literal["forecast", "policy", "embedding"] + @property + def project_root(self): + return self.run.project_root + @field_validator("model_paths") @classmethod def validate_paths(cls, value): diff --git a/src/sequifier/config/input_contract_resolution.py b/src/sequifier/config/input_contract_resolution.py new file mode 100644 index 00000000..436d28f0 --- /dev/null +++ b/src/sequifier/config/input_contract_resolution.py @@ -0,0 +1,56 @@ +"""Assert authored interface facts against frozen artifact/source metadata.""" + +from pathlib import Path + + +def validate_input_contracts(config, resolved): + if config.model is None: + return + from sequifier.config.metadata import load_dataset_metadata + from sequifier.config.train_config import _assert_compatible, _part_signature + + for name, spec in config.model.interfaces.items(): + declaration = spec.input_contract + datasets = [ + value + for value in resolved.dataset_training.values() + if value.model_interface == name + ] + if not datasets: + continue + if declaration.metadata_config_path: + expected = load_dataset_metadata( + str(Path(config.run.project_root) / declaration.metadata_config_path) + ) + for dataset in datasets: + for part in dataset.parts.values(): + _assert_compatible( + _part_signature(expected, config.model.interfaces[name]), + _part_signature(part.metadata, config.model.interfaces[name]), + f"model.interfaces.{name}.input_contract", + ) + normalization = declaration.normalization + if normalization is None: + continue + resolved.normalization_by_interface[name] = normalization + for dataset in datasets: + interface = dataset.interface + expected = normalization.statistics + if normalization.source == "identity": + expected = { + column: {"mean": 0.0, "std": 1.0} + for column in interface.input_columns + interface.static_columns + if column in interface.real_columns + } + for column, values in expected.items(): + if column not in interface.real_columns: + raise ValueError( + f"{name}: normalization references unknown/non-real input {column!r}" + ) + actual = interface.selected_columns_statistics.get(column) + if not interface.normalize_real_columns: + actual = {"mean": 0.0, "std": 1.0} + if actual != values: + raise ValueError( + f"model.interfaces.{name}.input_contract.normalization.{column} differs from frozen metadata/artifact" + ) diff --git a/src/sequifier/config/learning_config.py b/src/sequifier/config/learning_config.py new file mode 100644 index 00000000..ac3ca123 --- /dev/null +++ b/src/sequifier/config/learning_config.py @@ -0,0 +1,40 @@ +"""Phase-owned supervised losses and sampling views.""" + +import math +from typing import Optional + +import torch +from pydantic import ConfigDict, Field, model_validator + +from sequifier.config.components import BERTSpecModel, NextOccurrenceConfigModel +from sequifier.config.naming import ConfigModel + + +class DatasetLearningConfig(ConfigModel): + model_config = ConfigDict(extra="forbid") + criterion: dict[str, str] = Field(default_factory=dict) + class_weights: Optional[dict[str, list[float]]] = None + loss_weights: Optional[dict[str, float]] = None + class_share_log_columns: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def losses(self): + for name in self.criterion.values(): + if name != "AccurateCrossEntropyLoss" and not hasattr(torch.nn, name): + raise ValueError(f"unknown criterion {name!r}") + for weights in (self.loss_weights, self.class_weights): + for value in (weights or {}).values(): + values = value if isinstance(value, list) else [value] + if any(not math.isfinite(x) or x < 0 for x in values): + raise ValueError( + "loss and class weights must be finite and nonnegative" + ) + return self + + +class SupervisedLearningConfig(ConfigModel): + model_config = ConfigDict(extra="forbid") + datasets: dict[str, DatasetLearningConfig] = Field(default_factory=dict) + bert_spec: BERTSpecModel | None = None + next_occurrence_config: NextOccurrenceConfigModel | None = None + window_stride: int | None = Field(default=None, gt=0) diff --git a/src/sequifier/config/metadata.py b/src/sequifier/config/metadata.py index 29c9bbd0..a45090dc 100644 --- a/src/sequifier/config/metadata.py +++ b/src/sequifier/config/metadata.py @@ -191,7 +191,7 @@ def load_dataset_metadata(path: str) -> DatasetMetadata: def extract_inline_metadata( values: dict[str, Any], ) -> tuple[dict[str, Any], dict[str, Any] | None]: - """Split the historical ``skip_metadata`` representation into two mappings.""" + """Separate inline metadata facts from the authored request.""" authored = copy.deepcopy(values) window_view = authored.get("window_view") diff --git a/src/sequifier/config/naming.py b/src/sequifier/config/naming.py index ce1c7d47..03d86bef 100644 --- a/src/sequifier/config/naming.py +++ b/src/sequifier/config/naming.py @@ -1,122 +1,13 @@ -"""Canonical authored names and run-seed resolution. - -Removed names are diagnosed, never translated into accepted aliases. Arbitrary -user dictionaries (columns, constructor arguments, and environment config) are -not inspected as configuration schemas. -""" +"""Shared configuration defaults and run-seed resolution.""" import secrets from typing import Annotated, Any -from pydantic import BaseModel, BeforeValidator, model_validator - -RENAMED_FIELDS = { - "interface": "interfaces.", - "dataset": "dataset_training.", - "part": "parts.", - "policy_binding": "policy_bindings.", - "dim_model": "model_dim", - "n_heads": "num_heads", - "n_kv_heads": "num_kv_heads", - "embedding_size": "total_embedding_dim", - "positional_encoding_scope": "position_encoding_scope", - "post_conv_norm": "post_conv_normalization", - "frame_pool": "frame_pooling", - "image_frame_pool": "image_frame_pooling", - "support": "support_length", - "decoder_support": "decoder_support_length", - "device_max_concat_length": "max_concat_tensors", - "accumulation_steps": "accumulation_batches", - "minibatch_size": "batch_size", - "updates": "iterations", - "groups_per_update": "groups_per_iteration", - "rollout_length": "rollout_env_steps", - "log_interval": "log_interval_batches", - "early_stopping_epochs": "early_stopping_patience_epochs", - "gradient_clip": "gradient_clip_norm", - "value_clip": "value_clip_delta", - "target_kl": "old_policy_target_kl", - "kl_coefficient": "reference_kl_coefficient", - "max_weight": "max_advantage_weight", - "weight": "selection_weight", - "save_interval_batches": "snapshot_interval_batches", - "save_interval_minutes": "snapshot_interval_minutes", - "save_latest_interval_minutes": "latest_checkpoint_interval_minutes", - "save_interval_val_loss": "evaluate_on_snapshot", - "calculate_validation_loss_on_initialization": "evaluate_before_training", - "min_freq": "min_count", - "top_k": "max_categories", - "k": "max_categories", - "buckets": "num_buckets", - "column_types": "column_data_types", - "standard_deviation": "std", - "n_cores": "num_workers", - "image_path": "image_paths", - "preconditioning_image_path": "preconditioning_image_paths", - "manifest": "manifest_path", - "freezing_except": "freeze_except_groups", - "model_type": "task: forecast | policy | embedding", - "model_path": "model_paths (a list)", - "autoregressive": "autoregressive_generation", - "global_seed": "sampler_seed", - "model_config_write_path": "trial_config_directory", - "evaluation_script": "evaluation_script_path", - "evaluation_inference_config": "evaluation_inference_config_path", -} +from pydantic import BaseModel, BeforeValidator, ConfigDict class ConfigModel(BaseModel): - @model_validator(mode="before") - @classmethod - def reject_removed_names(cls, values: Any) -> Any: - if not isinstance(values, dict): - return values - fields = cls.model_fields - replacements = dict(RENAMED_FIELDS) - for key in ("evaluation_interval", "checkpoint_interval"): - unit = "iterations" if f"{key}_iterations" in fields else "epochs" - replacements[key] = f"{key}_{unit}" - if "direction" in fields: - replacements["mode"] = "direction: min | max" - if "pooling" in fields: - replacements["mode"] = "pooling" - if "hidden_dim" in fields: - replacements["dim"] = "hidden_dim" - if "max_depth_length" in fields: - replacements["context_length"] = "max_depth_length" - if "freeze_groups" in fields: - replacements["freeze"] = "freeze_groups" - elif "freeze_modules" in fields: - replacements["freeze"] = "freeze_modules" - if "paths" in fields and cls.__name__ == "PreconditioningImageConfig": - replacements["path"] = "paths (a view-to-path mapping)" - if "interface_metadata_paths" in fields: - replacements["metadata_config_path"] = "interface_metadata_paths" - if cls.__name__ == "PreprocessorModel": - replacements["window_length"] = "window_lengths (one value per split)" - if cls.__name__ in {"InferenceConfig", "ResolvedInferenceConfig"}: - replacements["preprocessing_data_path"] = "metadata_config_path" - if cls.__name__ == "SequifierConfig": - replacements["evaluation"] = "training_plan.phases[].evaluation" - for key in values: - if key not in fields and key in replacements: - replacement = replacements[key] - target = replacement.split(":")[0].split(".")[0].split()[0] - if target in fields: - raise ValueError( - f"{cls.__name__}.{key} was removed; use {replacement}" - ) - if cls.__name__ == "TrainingPlanModel" and "phases" not in values: - raise ValueError("Declare training phases under training_plan.phases") - if cls.__name__ == "ComponentSpec" and (extra := values.keys() - fields.keys()): - raise ValueError( - f"Put constructor arguments {sorted(extra)} under arguments" - ) - if "_project_root" in cls.__private_attributes__ and "project_root" in values: - raise ValueError( - "Source project_root was removed; use the run's project_root" - ) - return values + model_config = ConfigDict(extra="forbid") def resolve_run_seed(value: Any) -> int: diff --git a/src/sequifier/config/normalization_config.py b/src/sequifier/config/normalization_config.py new file mode 100644 index 00000000..0479c04f --- /dev/null +++ b/src/sequifier/config/normalization_config.py @@ -0,0 +1,29 @@ +"""Frozen interface normalization facts, independent of learning algorithms.""" + +import math +from typing import Literal + +from pydantic import ConfigDict, Field, model_validator + +from sequifier.config.naming import ConfigModel + + +class FrozenNormalizationConfig(ConfigModel): + model_config = ConfigDict(extra="forbid") + source: Literal["identity", "statistics"] + statistics: dict[str, dict[str, float]] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_statistics(self): + if self.source == "identity" and self.statistics: + raise ValueError("identity normalization cannot include statistics") + for column, values in self.statistics.items(): + if ( + set(values) != {"mean", "std"} + or not all(math.isfinite(value) for value in values.values()) + or values["std"] < 0 + ): + raise ValueError( + f"{column}: statistics require finite mean and nonnegative std" + ) + return self diff --git a/src/sequifier/config/optimizer_config.py b/src/sequifier/config/optimizer_config.py index e0d27bbc..3c847c50 100644 --- a/src/sequifier/config/optimizer_config.py +++ b/src/sequifier/config/optimizer_config.py @@ -8,38 +8,9 @@ from sequifier.config.components import ComponentSpec from sequifier.config.naming import ConfigModel +from sequifier.config.parameter_selection import ParameterSelector - -class OptimizerSelector(ConfigModel): - """Match parameter catalog descriptors; supplied conditions are conjunctive.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - semantic_groups: list[str] | None = None - component: Literal["ingestion", "backbone", "decoder"] | None = None - parameter_kind: Literal["weight", "bias", "other"] | None = None - ndim: int | None = Field(default=None, ge=0) - - @model_validator(mode="after") - def validate_selector(self) -> "OptimizerSelector": - if not any( - value is not None - for value in ( - self.semantic_groups, - self.component, - self.parameter_kind, - self.ndim, - ) - ): - raise ValueError( - "An optimizer selector must specify at least one condition." - ) - if self.semantic_groups is not None and ( - not self.semantic_groups - or any(not pattern for pattern in self.semantic_groups) - ): - raise ValueError("semantic_groups must contain nonempty patterns.") - return self +OptimizerSelector = ParameterSelector class OptimizerGroupSpec(ConfigModel): diff --git a/src/sequifier/config/ownership.py b/src/sequifier/config/ownership.py new file mode 100644 index 00000000..407ac2c3 --- /dev/null +++ b/src/sequifier/config/ownership.py @@ -0,0 +1,125 @@ +"""Run resources and durable model contracts, independent of learning algorithms.""" + +from typing import Literal + +from pydantic import ConfigDict, Field, field_validator, model_validator + +from sequifier.config.components import NextOccurrenceConfigModel, ResumeConfig +from sequifier.config.naming import ConfigModel, RunSeed +from sequifier.config.normalization_config import FrozenNormalizationConfig + + +class Settings(ConfigModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + + +class DistributionSettings(Settings): + strategy: Literal["local", "ddp", "fsdp"] = "local" + world_size: int = Field(default=1, gt=0) + backend: str = "nccl" + cpu_offload: bool | None = None + + @model_validator(mode="after") + def coherent(self): + if self.strategy == "local" and self.world_size != 1: + raise ValueError("local distribution requires world_size: 1") + if self.strategy != "fsdp" and self.cpu_offload is not None: + raise ValueError("cpu_offload is only valid with fsdp") + if self.strategy == "fsdp" and self.cpu_offload is None: + raise ValueError("fsdp requires explicit cpu_offload") + return self + + +class PrecisionSettings(Settings): + layer_type_dtypes: dict[str, str] | None = None + layer_autocast: bool = False + float32_matmul_precision: Literal["highest", "high", "medium"] = "highest" + + +class CompilationSettings(Settings): + torch_compile: Literal["outer", "inner", "none"] = "none" + + +class DataLoadingSettings(Settings): + num_workers: int = Field(default=0, ge=0) + load_full_data_to_ram: bool = True + max_ram_gb: float = Field(default=16, gt=0) + episode_cache_size: int = Field(default=8, gt=0) + + +class ExecutionSettings(Settings): + device: str = "cpu" + distribution: DistributionSettings = Field(default_factory=DistributionSettings) + precision: PrecisionSettings = Field(default_factory=PrecisionSettings) + compilation: CompilationSettings = Field(default_factory=CompilationSettings) + data_loading: DataLoadingSettings = Field(default_factory=DataLoadingSettings) + max_concat_tensors: int = Field(default=12, gt=0) + + +class ReproducibilitySettings(Settings): + seed: RunSeed = 1010 + enforce_determinism: bool = False + + +class RecoverySettings(Settings): + resume: ResumeConfig = Field(default_factory=ResumeConfig) + latest_checkpoint_interval_minutes: float | None = Field(default=None, gt=0) + + +class LoggingSettings(Settings): + log_interval_batches: int = Field(default=10, gt=0) + + +class OutputSettings(Settings): + model_path: str = "" + report_path: str = "" + phase_directory: str = "" + + +class ExportSettings(Settings): + generative_model: bool = True + embedding_model: bool = False + embedding_layer_names: list[str] = Field( + default_factory=lambda: ["backbone.final_norm"], min_length=1 + ) + onnx: bool = False + pt: bool = True + with_dropout: bool = False + batch_size: int = Field(default=1, gt=0) + + +class RunSettings(Settings): + project_root: str = "." + model_name: str + execution: ExecutionSettings = Field(default_factory=ExecutionSettings) + reproducibility: ReproducibilitySettings = Field( + default_factory=ReproducibilitySettings + ) + recovery: RecoverySettings = Field(default_factory=RecoverySettings) + logging: LoggingSettings = Field(default_factory=LoggingSettings) + outputs: OutputSettings = Field(default_factory=OutputSettings) + export: ExportSettings = Field(default_factory=ExportSettings) + + @field_validator("model_name") + @classmethod + def validate_model_name(cls, value): + if "embedding" in value: + raise ValueError("model_name cannot contain 'embedding'") + return value + + +class ModelExecutionSettings(Settings): + context_length: int | None = Field(default=None, gt=0) + target_offset: int = Field(default=1, ge=0) + training_objective: str | None = None + forward_attention: Literal["causal", "bidirectional"] | None = None + next_occurrence_config: NextOccurrenceConfigModel | None = None + + +class InputContract(Settings): + metadata_config_path: str | None = None + normalization: FrozenNormalizationConfig | None = None + + +class SplitSettings(Settings): + validation_fraction: float = Field(default=0.2, gt=0, lt=1) diff --git a/src/sequifier/config/parameter_selection.py b/src/sequifier/config/parameter_selection.py new file mode 100644 index 00000000..a819cb6e --- /dev/null +++ b/src/sequifier/config/parameter_selection.py @@ -0,0 +1,189 @@ +"""One catalog selector language for trainability and optimizer routing.""" + +from fnmatch import fnmatchcase +from typing import Literal + +from pydantic import ConfigDict, Field, model_validator + +from sequifier.config.naming import ConfigModel + + +class ParameterSelector(ConfigModel): + model_config = ConfigDict(extra="forbid", frozen=True) + semantic_groups: list[str] | None = None + exclude_modules: list[str] = Field(default_factory=list) + exclude_semantic_groups: list[str] = Field(default_factory=list) + component: Literal["ingestion", "backbone", "decoder"] | None = None + parameter_kind: Literal["weight", "bias", "other"] | None = None + ndim: int | None = Field(default=None, ge=0) + interface: str | None = None + branch_path: list[str] | None = None + module: str | None = None + + @model_validator(mode="after") + def conditions(self): + if all(getattr(self, key) in (None, []) for key in type(self).model_fields): + raise ValueError("a parameter selector requires at least one condition") + for key in ( + "semantic_groups", + "branch_path", + "exclude_modules", + "exclude_semantic_groups", + ): + value = getattr(self, key) + if value is not None and ( + (not value and key in {"semantic_groups", "branch_path"}) + or any(not x for x in value) + ): + raise ValueError(f"{key} requires nonempty entries") + for key in ("module", "interface"): + value = getattr(self, key) + if value is not None and ( + not value or any(not x for x in value.split(".")) + ): + raise ValueError(f"{key} must be nonempty") + return self + + def matches(self, descriptor): + names = list(descriptor.aliases) + for alias in descriptor.aliases: + parts = alias.split(".") + while len(parts) > 1 and (parts[0] == "network" or parts[0].isdigit()): + parts.pop(0) + normalized = ".".join(parts) + if normalized not in names: + names.append(normalized) + + def in_module(name, module): + return name == module or name.startswith(module + ".") + + if self.exclude_modules and descriptor.shared_parameter_id: + excluded = [ + any(in_module(name, path) for path in self.exclude_modules) + for name in descriptor.aliases + ] + if any(excluded) and not all(excluded): + raise ValueError( + f"conflicting selector exclusions for shared parameter {descriptor.parameter_id}" + ) + return ( + ( + self.semantic_groups is None + or any( + fnmatchcase(group, p) + for group in ( + descriptor.semantic_aliases or (descriptor.semantic_group,) + ) + for p in self.semantic_groups + ) + ) + and not any( + fnmatchcase(group, p) + for group in ( + descriptor.semantic_aliases or (descriptor.semantic_group,) + ) + for p in self.exclude_semantic_groups + ) + and not any(in_module(n, m) for n in names for m in self.exclude_modules) + and (self.component is None or descriptor.component == self.component) + and ( + self.parameter_kind is None + or descriptor.parameter_kind == self.parameter_kind + ) + and (self.ndim is None or len(descriptor.shape) == self.ndim) + and ( + self.branch_path is None + or tuple(self.branch_path) + == descriptor.branch_path[: len(self.branch_path)] + ) + and ( + self.interface is None + or any( + in_module(n, f"interfaces.{self.interface}") + or in_module(n, f"network.interfaces.{self.interface}") + for n in names + ) + ) + and (self.module is None or any(in_module(n, self.module) for n in names)) + ) + + +class RoleTrainability(ConfigModel): + model_config = ConfigDict(extra="forbid") + freeze: list[ParameterSelector] | None = None + freeze_except: list[ParameterSelector] | None = None + + @model_validator(mode="after") + def exclusive(self): + if self.freeze is not None and self.freeze_except is not None: + raise ValueError("freeze and freeze_except are mutually exclusive") + return self + + def resolve(self, module, *, eligible=None): + from sequifier.model.parameter_catalog import ParameterCatalog + + catalog = ParameterCatalog(module) + candidates = { + d.parameter_id + for d in catalog.descriptors() + if eligible is None or id(catalog.parameter(d.parameter_id)) in eligible + } + known_modules = set() + for name, _ in module.named_modules(remove_duplicate=False): + known_modules.add(name) + parts = name.split(".") + while len(parts) > 1 and (parts[0] == "network" or parts[0].isdigit()): + parts.pop(0) + known_modules.add(".".join(parts)) + selected = set() + for selector in ( + self.freeze if self.freeze is not None else self.freeze_except or [] + ): + paths = selector.exclude_modules + ( + [selector.module] if selector.module else [] + ) + if unknown := set(paths) - known_modules: + raise ValueError(f"unknown selector modules: {sorted(unknown)}") + matched = { + d.parameter_id + for d in catalog.descriptors() + if d.parameter_id in candidates and selector.matches(d) + } + if not matched: + raise ValueError( + f"trainability selector matched no parameters: {selector.model_dump(exclude_none=True)}" + ) + selected.update(matched) + if self.freeze_except is not None: + selected = candidates - selected + return frozenset(id(catalog.parameter(key)) for key in selected) + + +class RolePolicies(ConfigModel): + model_config = ConfigDict(extra="forbid") + roles: dict[str, RoleTrainability] = Field(default_factory=dict) + + +class TrainabilityConfig(RolePolicies): + by_dataset: dict[str, RolePolicies] = Field(default_factory=dict) + + def policy(self, role, dataset=None): + override = self.by_dataset.get(dataset) if dataset is not None else None + return ( + override.roles.get(role) + if override and role in override.roles + else self.roles.get(role) + ) or RoleTrainability() + + def validate_roles(self, roles, *, datasets=None): + for policy in [self, *self.by_dataset.values()]: + if unknown := policy.roles.keys() - set(roles): + raise ValueError(f"unsupported trainability roles: {sorted(unknown)}") + if self.by_dataset and datasets is None: + raise ValueError( + "source-dependent trainability requires supervised updates" + ) + if datasets is not None and (unknown := self.by_dataset.keys() - set(datasets)): + raise ValueError( + f"trainability overrides require training datasets: {sorted(unknown)}" + ) diff --git a/src/sequifier/config/phase_controls.py b/src/sequifier/config/phase_controls.py new file mode 100644 index 00000000..a6fa1e26 --- /dev/null +++ b/src/sequifier/config/phase_controls.py @@ -0,0 +1,109 @@ +"""Shared phase lifecycle declarations; algorithms only constrain capabilities.""" + +from typing import TYPE_CHECKING, Any, Literal + +from pydantic import ConfigDict, Field, model_validator + +from sequifier.config.naming import ConfigModel + + +class Cadence(ConfigModel): + model_config = ConfigDict(extra="forbid") + epochs: int | None = Field(default=None, gt=0) + iterations: int | None = Field(default=None, gt=0) + + @model_validator(mode="after") + def one_clock(self): + if (self.epochs is None) == (self.iterations is None): + raise ValueError("cadence requires exactly one of epochs or iterations") + return self + + +class EvaluationTriggers(ConfigModel): + model_config = ConfigDict(extra="forbid") + cadence: Cadence | None = None + before_phase: bool = False + on_snapshot: bool = False + batch_size: int = Field(default=1, gt=0) + + +class PhaseCheckpoints(ConfigModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + cadence: Cadence | None = None + snapshot_interval_minutes: float | None = Field(default=None, gt=0) + snapshot_interval_batches: int | None = Field(default=None, gt=0) + + +class ModelSelection(ConfigModel): + model_config = ConfigDict(extra="forbid") + handoff: Literal["best", "last"] = "last" + + +class StoppingConfig(ConfigModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + patience: int = Field(gt=0) + min_delta: float = Field(default=0.0, ge=0) + on_trigger: Literal["end_phase", "end_plan"] = "end_phase" + + +class CurriculumConfig(ConfigModel): + model_config = ConfigDict(extra="forbid") + column: str = Field(min_length=1) + + +class PhaseControls(ConfigModel): + if TYPE_CHECKING: + algorithm: Any + evaluation: Any + + model_config = ConfigDict(extra="forbid") + checkpoints: PhaseCheckpoints = Field(default_factory=PhaseCheckpoints) + model_selection: ModelSelection = Field(default_factory=ModelSelection) + stopping: StoppingConfig | None = None + + @property + def handoff(self): + return self.model_selection.handoff + + @property + def evaluation_interval_epochs(self): + cadence = self.evaluation.cadence if self.evaluation is not None else None + return cadence.epochs if cadence is not None else 1 + + @property + def checkpoint_interval_epochs(self): + return self.checkpoints.cadence.epochs if self.checkpoints.cadence else 1 + + @property + def evaluation_interval_iterations(self): + cadence = self.evaluation.cadence if self.evaluation is not None else None + return cadence.iterations if cadence is not None else 1 + + @property + def checkpoint_interval_iterations(self): + cadence = self.checkpoints.cadence + return cadence.iterations if cadence else 1 + + @model_validator(mode="after") + def validate_lifecycle(self): + clock = "epochs" if self.algorithm == "supervised" else "iterations" + for label, cadence in ( + ("checkpoints", self.checkpoints.cadence), + ("evaluation", self.evaluation.cadence if self.evaluation else None), + ): + if cadence is not None and getattr(cadence, clock) is None: + raise ValueError(f"{self.algorithm} {label}.cadence requires {clock}") + if self.evaluation is None and (self.stopping or self.handoff == "best"): + raise ValueError("stopping and best handoff require evaluation.monitor") + if ( + self.stopping + and getattr(self.evaluation, "monitor", None) is None + and len(getattr(self.evaluation, "sources", None) or []) != 1 + ): + raise ValueError("stopping requires evaluation.monitor") + if self.algorithm != "supervised": + if self.checkpoints.snapshot_interval_batches is not None: + raise ValueError( + "policy snapshots use committed iterations, not batches" + ) + return self diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 6b9c8036..955a42bf 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -1,12 +1,21 @@ """Built-in policy phase schemas; budgets and objectives are algorithm-specific.""" -import math from typing import Annotated, ClassVar, Literal from pydantic import ConfigDict, Field, field_validator, model_validator from sequifier.config.identifiers import identifier from sequifier.config.naming import ConfigModel +from sequifier.config.normalization_config import ( # noqa: F401 + FrozenNormalizationConfig, +) +from sequifier.config.parameter_selection import TrainabilityConfig +from sequifier.config.phase_controls import ( + Cadence, + EvaluationTriggers, + PhaseCheckpoints, + PhaseControls, +) from sequifier.config.policy_optimization import OptimizationConfig from sequifier.config.sources import TrainingSourceSpecModel @@ -23,29 +32,34 @@ class DPOEvaluationMonitor(ConfigModel): direction: Literal["min"] = "min" -class OfflineEvaluationConfig(ConfigModel): +class OfflineEvaluationConfig(EvaluationTriggers): model_config = ConfigDict(extra="forbid") sources: list[TrainingSourceSpecModel] | None = Field(default=None, min_length=1) monitor: OfflineEvaluationMonitor = Field(default_factory=OfflineEvaluationMonitor) -class DPOEvaluationConfig(ConfigModel): +class DPOEvaluationConfig(EvaluationTriggers): + @model_validator(mode="after") + def serial_evaluation(self): + if self.batch_size != 1: + raise ValueError( + "DPO evaluation scores one preference pair at a time; evaluation.batch_size must be 1" + ) + return self + model_config = ConfigDict(extra="forbid") sources: list[TrainingSourceSpecModel] | None = Field(default=None, min_length=1) monitor: DPOEvaluationMonitor = Field(default_factory=DPOEvaluationMonitor) -class PhaseConfig(ConfigModel): +class PhaseConfig(PhaseControls): model_config = ConfigDict(extra="forbid", allow_inf_nan=False) name: str optimizer_state: Literal["reset", "continue"] = "reset" - handoff: Literal["best", "last"] = "last" model_interface: str | None = None - freeze_modules: list[str] = Field(default_factory=list) + trainability: TrainabilityConfig = Field(default_factory=TrainabilityConfig) optimization_roles: ClassVar[tuple[str, ...]] = ("actor",) optimization: dict[str, OptimizationConfig] = Field(default_factory=dict) - evaluation_interval_iterations: int = Field(default=1, gt=0) - checkpoint_interval_iterations: int = Field(default=1, gt=0) def optimizer_step_budget(self, training_size=None): raise NotImplementedError("built-in phase must declare its optimizer budget") @@ -61,6 +75,7 @@ def optimization_configs(self): @model_validator(mode="after") def validate_optimization(self): + self.trainability.validate_roles(self.optimization_roles) for config in self.optimization_configs().values(): config.validate_budget(self.optimizer_step_budget()) return self @@ -95,6 +110,12 @@ def optimizer_step_budget(self, training_size=None): class IQLPhaseConfig(OfflinePhaseConfig): optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value", "critic") algorithm: Literal["iql"] + evaluation: OfflineEvaluationConfig = Field( + default_factory=lambda: OfflineEvaluationConfig(cadence=Cadence(iterations=100)) + ) + checkpoints: PhaseCheckpoints = Field( + default_factory=lambda: PhaseCheckpoints(cadence=Cadence(iterations=100)) + ) name: str = Field(default="offline_iql", min_length=1) iterations: int = Field(gt=0) batch_size: int = Field(default=64, gt=0) @@ -103,8 +124,6 @@ class IQLPhaseConfig(OfflinePhaseConfig): inverse_temperature: float = Field(default=3.0, gt=0, allow_inf_nan=False) max_advantage_weight: float = Field(default=100.0, ge=1, allow_inf_nan=False) target_update_rate: float = Field(default=0.005, gt=0, le=1, allow_inf_nan=False) - evaluation_interval_iterations: int = Field(default=100, gt=0) - checkpoint_interval_iterations: int = Field(default=100, gt=0) def optimizer_step_budget(self, training_size=None): return self.iterations @@ -132,27 +151,6 @@ def validate_factory(cls, value): return value -class FrozenNormalizationConfig(ConfigModel): - model_config = ConfigDict(extra="forbid") - source: Literal["identity", "statistics"] - statistics: dict[str, dict[str, float]] = Field(default_factory=dict) - - @model_validator(mode="after") - def validate_statistics(self): - if self.source == "identity" and self.statistics: - raise ValueError("identity normalization cannot include statistics") - for column, values in self.statistics.items(): - if ( - set(values) != {"mean", "std"} - or not all(math.isfinite(value) for value in values.values()) - or values["std"] < 0 - ): - raise ValueError( - f"{column}: statistics require finite mean and nonnegative std" - ) - return self - - class EvaluationEpisodesConfig(ConfigModel): model_config = ConfigDict(extra="forbid") environment: OnlineEnvironmentConfig @@ -178,7 +176,15 @@ class OnlineEvaluationMonitor(ConfigModel): direction: Literal["min", "max"] = "max" -class OnlineEvaluationConfig(EvaluationEpisodesConfig): +class OnlineEvaluationConfig(EvaluationEpisodesConfig, EvaluationTriggers): + @model_validator(mode="after") + def serial_evaluation(self): + if self.batch_size != 1: + raise ValueError( + "online evaluation runs serial episodes; evaluation.batch_size must be 1" + ) + return self + monitor: OnlineEvaluationMonitor = Field(default_factory=OnlineEvaluationMonitor) diff --git a/src/sequifier/config/preprocess_config.py b/src/sequifier/config/preprocess_config.py index 38ac3e52..397161d1 100644 --- a/src/sequifier/config/preprocess_config.py +++ b/src/sequifier/config/preprocess_config.py @@ -6,13 +6,14 @@ import numpy as np from pydantic import ConfigDict, Field, field_validator, model_validator +from sequifier.config.command_ownership import CommandRunSettings from sequifier.config.composition import ( load_composed_yaml_config, merge_config_fragments, ) from sequifier.config.depth_layout import DepthLayoutRegistryModel from sequifier.config.image_config import ImageSourceConfig, PreconditioningImageConfig -from sequifier.config.naming import ConfigModel, RunSeed +from sequifier.config.naming import ConfigModel from sequifier.config.sequence_import_config import SequenceImportConfig from sequifier.helpers import ( canonicalize_polars_dtype_name, @@ -30,11 +31,18 @@ def load_preprocessor_config( """Load preprocessing YAML plus CLI overrides.""" config_values = load_composed_yaml_config(config_path) + from sequifier.config.command_ownership import apply_run_overrides + + config_values, args_config = apply_run_overrides(config_values, args_config) config_values = merge_config_fragments((config_values, args_config)) if config_values.get("operation") == "import_sequences": return SequenceImportConfig.model_validate(config_values) - return try_catch_excess_keys(config_path, PreprocessorModel, config_values) + return try_catch_excess_keys( + config_path, + PreprocessorModel, + config_values, + ) class CardinalityHashingModel(ConfigModel): @@ -88,7 +96,7 @@ class PreprocessorModel(ConfigModel): depth_layouts: DepthLayoutRegistryModel = Field( default_factory=DepthLayoutRegistryModel ) - project_root: str + run: CommandRunSettings = Field(default_factory=CommandRunSettings) preprocessing_data_path: str static_data_path: Optional[str] = None image_sources: dict[str, ImageSourceConfig] = Field(default_factory=dict) @@ -120,8 +128,6 @@ class PreprocessorModel(ConfigModel): prediction_alignment_steps: int = Field(default=1, gt=0) target_offset: Optional[int] = Field(default=None, ge=0) max_rows: Optional[int] = None - seed: RunSeed = 1010 - num_workers: Optional[int] = None batches_per_file: int = 1024 process_by_file: bool = True continue_preprocessing: bool = False @@ -132,6 +138,26 @@ class PreprocessorModel(ConfigModel): mask_column: Optional[str] = None curriculum_column: Optional[Union[str, list[str]]] = None + @property + def project_root(self): + return self.run.project_root + + @property + def seed(self): + return self.run.reproducibility.seed + + @property + def num_workers(self): + loading = self.run.execution.data_loading + return ( + loading.num_workers if "num_workers" in loading.model_fields_set else None + ) + + @model_validator(mode="after") + def validate_resources(self): + self.run.validate_for("preprocess") + return self + @field_validator("image_sources") @classmethod def validate_image_source_names(cls, sources): diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index 7e7bdad0..f8992f09 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -43,7 +43,7 @@ class RolloutConfig(ConfigModel): environment: AdapterConfig bindings: list[InputBinding] trajectory_path: str = Field(min_length=1) - # Index over prepared windows in dataset/chunk order, before legacy seed filtering. + # Index over prepared windows in dataset/chunk order, before seed filtering. seed_window_index: int = Field(ge=0) max_episode_steps: int | None = Field(default=None, gt=0) diff --git a/src/sequifier/config/sequence_import_config.py b/src/sequifier/config/sequence_import_config.py index 299578a4..c8221309 100644 --- a/src/sequifier/config/sequence_import_config.py +++ b/src/sequifier/config/sequence_import_config.py @@ -5,6 +5,7 @@ from pydantic import Field +from sequifier.config.command_ownership import CommandRunSettings from sequifier.config.policy_config import ( PolicyBinding, PolicyConfigBase, @@ -15,13 +16,17 @@ class SequenceImportConfig(PolicyConfigBase): operation: Literal["import_sequences"] - project_root: str = "." + run: CommandRunSettings = Field(default_factory=CommandRunSettings) source: SequenceSourceConfig policy_binding: PolicyBinding initialize_from: str | None = None output_path: str episodes_per_shard: int = Field(default=64, gt=0) + @property + def project_root(self): + return self.run.project_root + def resolved_source(self) -> SequenceSourceConfig: from sequifier.config.policy_train_config import resolve_sequence_source diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 83580058..0d206270 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -1,14 +1,13 @@ """Canonical named training configuration and resolution. All cardinalities use named interfaces, datasets, parts and ordered phases. -Removed spellings are diagnosed rather than accepted as aliases. +Only the current ownership structure is accepted. """ from __future__ import annotations import copy import inspect -import math import os import warnings from dataclasses import dataclass @@ -16,15 +15,7 @@ from types import SimpleNamespace from typing import Annotated, Any, ClassVar, Literal, Optional -import torch -from pydantic import ( - ConfigDict, - Field, - PrivateAttr, - StrictStr, - field_validator, - model_validator, -) +from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator # Reusable schemas are independent of the canonical root configuration. from sequifier.config.components import ( @@ -39,10 +30,6 @@ ResumeConfig, ) from sequifier.config.depth_layout import DepthLayoutRegistryModel -from sequifier.config.freezing_config import ( - IngestionFreezingConfig, - LayerFreezingConfigFields, -) from sequifier.config.identifiers import identifier as _identifier from sequifier.config.image_config import ( ImageSourceConfig, @@ -51,13 +38,20 @@ bind_image_paths, image_views, ) +from sequifier.config.learning_config import SupervisedLearningConfig from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata -from sequifier.config.naming import ConfigModel, RunSeed +from sequifier.config.naming import ConfigModel +from sequifier.config.normalization_config import FrozenNormalizationConfig +from sequifier.config.parameter_selection import TrainabilityConfig +from sequifier.config.phase_controls import ( + CurriculumConfig, + EvaluationTriggers, + PhaseControls, +) from sequifier.config.policy_config import PolicyBinding, ResolvedPolicyBinding from sequifier.config.policy_optimization import SupervisedOptimizationConfig from sequifier.config.policy_plan import ( DPOPhaseConfig, - FrozenNormalizationConfig, GRPOPhaseConfig, IQLPhaseConfig, PolicyImitationPhaseConfig, @@ -71,7 +65,6 @@ ModelWindowView, StoredWindowLayout, derive_target_column_types, - metadata_config_path_from_preprocessing_data_path, normalize_path, resolve_window_view, ) @@ -119,72 +112,12 @@ def _unique_columns(value: list[str], usage: str) -> list[str]: return value -@beartype -def normalize_train_config_surface(values: Any) -> Any: - """Apply defaults without accepting alternate authoring structures.""" - if not isinstance(values, dict): - return values - normalized = copy.deepcopy(values) - model = normalized.get("model") - training = normalized.get("global_training", {}) - if isinstance(model, dict): - backbone = model.get("backbone") - architecture = ( - backbone.get("architecture") if isinstance(backbone, dict) else None - ) - if isinstance(architecture, dict) and isinstance(training, dict): - if training.get("context_length") is not None: - architecture.setdefault( - "max_context_length", training["context_length"] - ) - datasets = normalized.get("dataset_training", {}) - interfaces = model.get("interfaces", {}) if isinstance(model, dict) else {} - if isinstance(datasets, dict): - for dataset in datasets.values(): - if not isinstance(dataset, dict): - continue - if isinstance(interfaces, dict) and len(interfaces) == 1: - dataset.setdefault("model_interface", next(iter(interfaces))) - parts = dataset.get("parts", {}) - if isinstance(parts, dict): - for part in parts.values(): - if isinstance(part, dict): - part.setdefault("type", "prepared") - implicit_source = None - if isinstance(datasets, dict) and len(datasets) == 1: - dataset_name, dataset = next(iter(datasets.items())) - if isinstance(dataset, dict) and isinstance(dataset.get("parts"), dict): - parts = dataset["parts"] - if parts: - implicit_source = ( - f"{dataset_name}.{next(iter(parts))}" - if len(parts) == 1 - else dataset_name - ) - plan = normalized.get("training_plan") - if isinstance(plan, dict) and isinstance(plan.get("phases"), list): - for phase in plan["phases"]: - if isinstance(phase, dict): - phase.setdefault("algorithm", "supervised") - if implicit_source and phase["algorithm"] in { - "supervised", - "policy_imitation", - "iql", - }: - phase.setdefault("sources", [{"source": implicit_source}]) - return normalized - - -class GlobalTrainingSpecModel(ConfigModel): - """Run-wide data, precision, execution and distribution settings.""" +class TrainingRuntimeSettings(ConfigModel): + """Internal native execution projection; never an authored ownership surface.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - validation_fraction: float = Field(default=0.2, gt=0, lt=1) episode_cache_size: int = Field(default=8, gt=0) - interface_metadata_paths: dict[str, str] = Field(default_factory=dict) - normalization: FrozenNormalizationConfig | None = None - read_format: Literal["csv", "parquet", "pt"] = "parquet" training_objective: str | None = None forward_attention: Literal["causal", "bidirectional"] | None = None context_length: int | None = Field(default=None, gt=0) @@ -196,12 +129,7 @@ class GlobalTrainingSpecModel(ConfigModel): max_concat_tensors: int = Field(default=12, gt=0) log_interval_batches: int = Field(default=10, gt=0) - early_stopping_patience_epochs: Optional[int] = Field(default=None, gt=0) latest_checkpoint_interval_minutes: Optional[float] = None - snapshot_interval_minutes: Optional[float] = None - snapshot_interval_batches: Optional[int] = None - evaluate_on_snapshot: bool = True - evaluate_before_training: bool = True resume: Optional[ResumeConfig] = None enforce_determinism: bool = False distributed: bool = False @@ -264,11 +192,7 @@ def validate_distribution(self): raise ValueError("FSDP requires torch_compile 'none' or 'inner'") if self.data_parallelism == "ddp" and self.torch_compile == "inner": raise ValueError("DDP requires torch_compile 'none' or 'outer'") - for name in ( - "latest_checkpoint_interval_minutes", - "snapshot_interval_minutes", - "snapshot_interval_batches", - ): + for name in ("latest_checkpoint_interval_minutes",): value = getattr(self, name) if ( value is not None @@ -457,6 +381,7 @@ def validate_interface_names(cls, value): class DatasetPartSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") type: Literal["prepared"] = "prepared" + read_format: Literal["csv", "parquet", "pt"] = "parquet" metadata_config_path: str image_paths: MediaPath = None @@ -480,93 +405,15 @@ class PreferencePartSpecModel(ConfigModel): ] -class DatasetFreezingSpecModel(ConfigModel): - model_config = ConfigDict(extra="forbid") - - backbone: LayerFreezingConfigFields = Field( - default_factory=LayerFreezingConfigFields - ) - ingestion: IngestionFreezingConfig = Field(default_factory=IngestionFreezingConfig) - ingestion_adapter: bool = False - decoder: LayerFreezingConfigFields = Field( - default_factory=LayerFreezingConfigFields - ) - - @property - @beartype - def active(self) -> bool: - return self.ingestion_adapter or any( - value.has_freezing_policy - for value in (self.backbone, self.ingestion, self.decoder) - ) - - -class DatasetTrainingSpecModel(ConfigModel): - model_config = ConfigDict(extra="forbid") - - model_interface: str | None = None - parts: dict[str, DatasetPart] = Field(..., min_length=1) - criterion: dict[str, str] = Field(default_factory=dict) - class_weights: Optional[dict[str, list[float]]] = None - loss_weights: Optional[dict[str, float]] = None - class_share_log_columns: list[str] = Field(default_factory=list) - freeze: DatasetFreezingSpecModel = Field(default_factory=DatasetFreezingSpecModel) - - @field_validator("model_interface") - @classmethod - @beartype - def validate_interface_name(cls, value): - return ( - _identifier(value, "Model interface reference") - if value is not None - else None - ) - - @field_validator("parts") - @classmethod - @beartype - def validate_part_names(cls, value): - for name in value: - _identifier(name, "Dataset part name") - return value - - @field_validator("criterion") - @classmethod - @beartype - def validate_criteria(cls, value): - for name in value.values(): - if name != "AccurateCrossEntropyLoss" and not hasattr(torch.nn, name): - raise ValueError(f"Criterion {name!r} not found in torch.nn") - return value - - @field_validator("loss_weights") - @classmethod - @beartype - def validate_loss_weights(cls, value): - if value is None: - return value - invalid = { - target: weight - for target, weight in value.items() - if not math.isfinite(weight) or weight < 0 - } - if invalid: - raise ValueError( - "loss_weights must contain finite, non-negative values; " - f"found {invalid}." - ) - return value - - class EvaluationMonitorSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") source: str - metric: str = "loss" + metric: Literal["loss"] = "loss" direction: Literal["min", "max"] = "min" -class EvaluationSpecModel(ConfigModel): +class EvaluationSpecModel(EvaluationTriggers): model_config = ConfigDict(extra="forbid") sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) @@ -593,15 +440,15 @@ def validate_monitor(self): return self -class TrainingPhaseSpecModel(ConfigModel): +class TrainingPhaseSpecModel(PhaseControls): model_config = ConfigDict(extra="forbid") evaluation: EvaluationSpecModel | None = None - evaluation_interval_epochs: int = Field(default=1, gt=0) - checkpoint_interval_epochs: int = Field(default=1, gt=0) + learning: SupervisedLearningConfig = Field(default_factory=SupervisedLearningConfig) + trainability: TrainabilityConfig = Field(default_factory=TrainabilityConfig) + curriculum: CurriculumConfig | None = None algorithm: Literal["supervised"] = "supervised" - handoff: Literal["last", "best"] = "last" optimization_roles: ClassVar[tuple[str, ...]] = ("model",) optimizer_state: Literal["reset", "continue"] = "reset" @@ -629,6 +476,16 @@ def validate_name(cls, value): @model_validator(mode="after") @beartype def validate_mode_fields(self): + datasets = {ref.source.partition(".")[0] for ref in self.sources} + self.trainability.validate_roles(self.optimization_roles, datasets=datasets) + learning_datasets = datasets | { + ref.source.partition(".")[0] + for ref in (self.evaluation.sources if self.evaluation else []) + } + if unknown := self.learning.datasets.keys() - learning_datasets: + raise ValueError( + f"learning declarations require training/evaluation datasets: {sorted(unknown)}" + ) if self.mode == "sequential": if self.selection is not None: raise ValueError("Sequential phases cannot configure selection") @@ -663,21 +520,11 @@ def validate_mode_fields(self): class TrainingPlanModel(ConfigModel): model_config = ConfigDict(extra="forbid") - curriculum_training: bool = False - curriculum_column: Optional[str] = Field(default=None, min_length=1) phases: list[TrainingPhase] = Field(..., min_length=1) @model_validator(mode="after") @beartype def validate_unique_names(self): - if self.curriculum_column is not None and not self.curriculum_training: - raise ValueError( - "curriculum_column may only be set when curriculum_training is true" - ) - if self.curriculum_training and self.curriculum_column is None: - raise ValueError( - "curriculum_column must be set when curriculum_training is true" - ) names = [phase.name for phase in self.phases] if len(names) != len(set(names)): raise ValueError("Training phase names must be unique") @@ -703,12 +550,6 @@ def validate_unique_names(self): raise ValueError( "supervised optimizer continuation requires identical dataset ownership" ) - if getattr(previous, "freeze_modules", None) != getattr( - phase, "freeze_modules", None - ): - raise ValueError( - "optimizer continuation requires identical parameter selection" - ) if { role: value.continuation_settings() for role, value in previous.optimization_configs().items() @@ -729,241 +570,177 @@ def validate_unique_names(self): return self -class SequifierConfig(ConfigModel): - """Training configuration with canonical names and inferred defaults.""" - - model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - - project_root: str - model_name: str - device: str - seed: RunSeed = 1010 - global_training: GlobalTrainingSpecModel = Field( - default_factory=GlobalTrainingSpecModel - ) - model: ModelSpecModel | None = None - initialize_from: str | None = None - policy_bindings: dict[str, PolicyBinding] = Field(default_factory=dict) - model_path: str = "" - report_path: str = "" - phase_directory: str = "" - dataset_training: dict[str, DatasetTrainingSpecModel] = Field(default_factory=dict) - training_plan: TrainingPlanModel - - export_generative_model: bool = True - export_embedding_model: bool = False - embedding_layer_names: list[StrictStr] = Field( - default_factory=lambda: ["backbone.final_norm"], min_length=1 - ) - export_onnx: bool = True - export_pt: bool = False - export_with_dropout: bool = False - - @model_validator(mode="before") - @classmethod - @beartype - def apply_authored_defaults(cls, values): - return normalize_train_config_surface(values) - - @field_validator("dataset_training") - @classmethod - @beartype - def validate_dataset_names(cls, value): - for name in value: - _identifier(name, "Dataset name") - return value +def _validate_source(config, source, usage): + name, _, part = source.partition(".") + if name not in config.dataset_training or ( + part and part not in config.dataset_training[name].parts + ): + raise ValueError(f"unknown {usage} source {source!r}") - @field_validator("model_name") - @classmethod - @beartype - def validate_model_name(cls, value): - if "embedding" in value: - raise ValueError("model_name cannot contain 'embedding'") - return value - @model_validator(mode="after") - @beartype - def validate_relationships(self): - if self.model is None and self.initialize_from is None: - raise ValueError("a run requires model or initialize_from") - for phase in self.training_plan.phases: - evaluation = phase.evaluation - refs = list(getattr(phase, "sources", [])) + list( - getattr(evaluation, "sources", None) or [] - ) - preference = getattr(phase, "preferences", None) - if preference is not None: - refs.append(preference) - for ref in refs: - self._validate_source(ref.source, f"phase {phase.name!r}") - if phase.algorithm != "supervised" and ( - ref.selection_weight is not None - or ref.batches_per_selection is not None - ): - raise ValueError( - "policy source references cannot configure supervised interleaving" - ) - if phase.algorithm != "supervised": - continue - role = phase.optimization_configs()["model"] - needs_monitor = ( - phase.handoff == "best" - or role.scheduler.name == "ReduceLROnPlateau" - or self.global_training.early_stopping_patience_epochs is not None - or ( - evaluation is not None and self.global_training.evaluate_on_snapshot - ) - ) - if needs_monitor and (evaluation is None or evaluation.monitor is None): - raise ValueError( - f"{phase.name}: best handoff, plateau scheduling, validation saving, and early stopping require evaluation.monitor" - ) - if ( - role.scheduler.name == "ReduceLROnPlateau" - and phase.evaluation_interval_epochs != 1 - ): - raise ValueError( - "ReduceLROnPlateau requires evaluation_interval_epochs: 1" - ) - if ( - self.global_training.early_stopping_patience_epochs is not None - and phase.evaluation_interval_epochs != 1 +def validate_training_contract(config): + if config.model is None and config.initialize_from is None: + raise ValueError("a run requires model or initialize_from") + for phase in config.training_plan.phases: + evaluation = phase.evaluation + refs = list(getattr(phase, "sources", [])) + list( + getattr(evaluation, "sources", None) or [] + ) + preference = getattr(phase, "preferences", None) + if preference is not None: + refs.append(preference) + for ref in refs: + _validate_source(config, ref.source, f"phase {phase.name!r}") + if phase.algorithm != "supervised" and ( + ref.selection_weight is not None + or ref.batches_per_selection is not None ): raise ValueError( - "early_stopping_patience_epochs requires evaluation_interval_epochs: 1" + "policy source references cannot configure supervised interleaving" ) - if any(phase.algorithm != "supervised" for phase in self.training_plan.phases): - if self.global_training.distributed: - raise ValueError("policy phases currently require local execution") - return self - if self.initialize_from is not None: - return ( - self # Model-contract assertions are checked after artifact resolution. + if phase.algorithm != "supervised": + continue + role = phase.optimization_configs()["model"] + needs_monitor = ( + phase.handoff == "best" + or role.scheduler.name == "ReduceLROnPlateau" + or phase.stopping is not None + or (evaluation is not None and evaluation.on_snapshot) + ) + if needs_monitor and (evaluation is None or evaluation.monitor is None): + raise ValueError( + f"{phase.name}: best handoff, plateau scheduling, validation saving, and early stopping require evaluation.monitor" ) - model = self.model - assert model is not None if ( - self.global_training.training_objective is None - or self.global_training.context_length is None + role.scheduler.name == "ReduceLROnPlateau" + and phase.evaluation_interval_epochs != 1 ): + raise ValueError("ReduceLROnPlateau requires evaluation_interval_epochs: 1") + + if any(phase.algorithm != "supervised" for phase in config.training_plan.phases): + if config.runtime_settings.distributed: + raise ValueError("policy phases currently require local execution") + return config + if config.initialize_from is not None: + return ( + config # Model-contract assertions are checked after artifact resolution. + ) + model = config.model + assert model is not None + if ( + config.runtime_settings.training_objective is None + or config.runtime_settings.context_length is None + ): + raise ValueError( + "supervised training requires training_objective and context_length" + ) + for interface in model.interfaces.values(): + validate_embedding_layer_names( + config.run.export.embedding_layer_names, + SimpleNamespace( + backbone=model.backbone, + decoder=interface.decoder, + ), + ) + + referenced_interfaces = set() + for dataset_name, dataset in config.dataset_training.items(): + if dataset.model_interface not in model.interfaces: raise ValueError( - "supervised training requires training_objective and context_length" + f"Dataset {dataset_name!r} references unknown model interface " + f"{dataset.model_interface!r}." ) - for interface in model.interfaces.values(): - validate_embedding_layer_names( - self.embedding_layer_names, - SimpleNamespace( - backbone=model.backbone, - decoder=interface.decoder, - ), + referenced_interfaces.add(dataset.model_interface) + interface = model.interfaces[dataset.model_interface] + learning = config.dataset_learning(dataset_name) + if set(learning.criterion) != set(interface.target_columns): + raise ValueError( + f"Dataset {dataset_name!r} criterion keys must equal interface " + f"target_columns." ) - - referenced_interfaces = set() - for dataset_name, dataset in self.dataset_training.items(): - if dataset.model_interface not in model.interfaces: - raise ValueError( - f"Dataset {dataset_name!r} references unknown model interface " - f"{dataset.model_interface!r}." - ) - referenced_interfaces.add(dataset.model_interface) - interface = model.interfaces[dataset.model_interface] - if set(dataset.criterion) != set(interface.target_columns): - raise ValueError( - f"Dataset {dataset_name!r} criterion keys must equal interface " - f"target_columns." - ) - if dataset.loss_weights is not None and set(dataset.loss_weights) - set( - interface.target_columns - ): - raise ValueError( - f"Dataset {dataset_name!r} loss_weights references unknown targets." - ) - if dataset.loss_weights is not None and all( - dataset.loss_weights.get(target, 1.0) == 0.0 - for target in interface.target_columns - ): - raise ValueError( - f"Dataset {dataset_name!r} must have at least one target with " - "a positive loss weight." - ) - if dataset.class_weights is not None and set(dataset.class_weights) - set( - interface.target_columns - ): - raise ValueError( - f"Dataset {dataset_name!r} class_weights references unknown targets." - ) - unreferenced = set(model.interfaces) - referenced_interfaces - if unreferenced: - warnings.warn( - f"Unreferenced model interfaces: {sorted(unreferenced)}", - stacklevel=2, + if learning.loss_weights is not None and set(learning.loss_weights) - set( + interface.target_columns + ): + raise ValueError( + f"Dataset {dataset_name!r} loss_weights references unknown targets." ) - - for phase in self.training_plan.phases: - for source in phase.sources: - self._validate_source(source.source, f"training phase {phase.name!r}") - objective = get_objective_class(self.global_training.training_objective) - is_bert = issubclass(objective, BERTObjective) - is_next = issubclass(objective, NextOccurrenceObjective) - if (self.global_training.bert_spec is not None) != is_bert: - raise ValueError("bert_spec must be configured exactly for BERT training") - if (self.global_training.next_occurrence_config is not None) != is_next: + if learning.loss_weights is not None and all( + learning.loss_weights.get(target, 1.0) == 0.0 + for target in interface.target_columns + ): raise ValueError( - "next_occurrence_config must be configured exactly for " - "next_occurrence training" + f"Dataset {dataset_name!r} must have at least one target with " + "a positive loss weight." ) - if ( - len(self.dataset_training) > 1 - and self.global_training.data_parallelism == "fsdp" + if learning.class_weights is not None and set(learning.class_weights) - set( + interface.target_columns ): - raise ValueError("Multi-dataset training does not support FSDP") - - context_length = self.global_training.context_length - prefix_positions = int( - any( - route.static_conditioning is not None - or route.preconditioning_image_encoder is not None - for route in model.interfaces.values() + raise ValueError( + f"Dataset {dataset_name!r} class_weights references unknown targets." ) + unreferenced = set(model.interfaces) - referenced_interfaces + if unreferenced: + warnings.warn( + f"Unreferenced model interfaces: {sorted(unreferenced)}", + stacklevel=2, ) - if ( - context_length + prefix_positions - > model.backbone.architecture.max_context_length - ): + + for phase in config.training_plan.phases: + for source in phase.sources: + _validate_source(config, source.source, f"training phase {phase.name!r}") + objective = get_objective_class(config.runtime_settings.training_objective) + is_bert = issubclass(objective, BERTObjective) + is_next = issubclass(objective, NextOccurrenceObjective) + if (config.runtime_settings.bert_spec is not None) != is_bert: + raise ValueError("bert_spec must be configured exactly for BERT training") + if (config.runtime_settings.next_occurrence_config is not None) != is_next: + raise ValueError( + "next_occurrence_config must be configured exactly for " + "next_occurrence training" + ) + if ( + len(config.dataset_training) > 1 + and config.runtime_settings.data_parallelism == "fsdp" + ): + raise ValueError("Multi-dataset training does not support FSDP") + + context_length = config.runtime_settings.context_length + prefix_positions = int( + any( + route.static_conditioning is not None + or route.preconditioning_image_encoder is not None + for route in model.interfaces.values() + ) + ) + if ( + context_length + prefix_positions + > model.backbone.architecture.max_context_length + ): + raise ValueError( + "context_length plus static prefix exceeds backbone max_context_length" + ) + for name, interface in model.interfaces.items(): + if interface.decoder.support_length > context_length: raise ValueError( - "context_length plus static prefix exceeds backbone max_context_length" + f"Interface {name!r} decoder support exceeds context_length" ) - for name, interface in model.interfaces.items(): - if interface.decoder.support_length > context_length: - raise ValueError( - f"Interface {name!r} decoder support exceeds context_length" - ) - decoded_length = context_length - interface.decoder.support_length + 1 - if interface.decoder.prediction_length > decoded_length: - raise ValueError( - f"Interface {name!r} prediction_length exceeds decoded length" - ) - objective.validate_prediction_length( - interface.decoder.prediction_length, - context_length, - usage="training", + decoded_length = context_length - interface.decoder.support_length + 1 + if interface.decoder.prediction_length > decoded_length: + raise ValueError( + f"Interface {name!r} prediction_length exceeds decoded length" ) - if ( - not self.export_generative_model - and not self.export_embedding_model - and os.getenv("SEQUIFIER_PREVENT_EXPORT") is None - ): - raise ValueError("At least one model export must be enabled") - return self - - @beartype - def _validate_source(self, source: str, usage: str) -> None: - dataset_name, _, part_name = source.partition(".") - dataset = self.dataset_training.get(dataset_name) - if dataset is None: - raise ValueError(f"Unknown {usage} dataset source {dataset_name!r}") - if part_name and part_name not in dataset.parts: - raise ValueError(f"Unknown {usage} part source {source!r}") + objective.validate_prediction_length( + interface.decoder.prediction_length, + context_length, + usage="training", + ) + if ( + not config.run.export.generative_model + and not config.run.export.embedding_model + and os.getenv("SEQUIFIER_PREVENT_EXPORT") is None + ): + raise ValueError("At least one model export must be enabled") + return config class ResolvedDatasetPart(ConfigModel): @@ -976,6 +753,7 @@ class ResolvedDatasetPart(ConfigModel): preconditioning_image_paths: MediaPath = None training_data_path: str validation_data_path: Optional[str] = None + read_format: Literal["csv", "parquet", "pt"] = "parquet" storage_form: Literal["file", "folder"] file_order: Literal["shuffled", "name"] = "shuffled" @@ -1056,7 +834,6 @@ class ResolvedDatasetTrainingSpec(ConfigModel): class_weights: Optional[dict[str, list[float]]] = None loss_weights: Optional[dict[str, float]] = None class_share_log_columns: list[str] - freeze: DatasetFreezingSpecModel class ResolvedTrainingSource(ConfigModel): @@ -1069,17 +846,21 @@ class ResolvedTrainingSource(ConfigModel): batches_per_selection: int = 1 -class ResolvedTrainingPhase(ConfigModel): +class ResolvedTrainingPhase(PhaseControls): + resolved_seed: int | None = None + trainability_summary: dict[str, list[str]] = Field(default_factory=dict) + model_config = ConfigDict(extra="forbid") + evaluation: EvaluationSpecModel | None = None + learning: SupervisedLearningConfig = Field(default_factory=SupervisedLearningConfig) + trainability: TrainabilityConfig = Field(default_factory=TrainabilityConfig) + curriculum: CurriculumConfig | None = None evaluation_sources: list[ResolvedTrainingSource] = Field(default_factory=list) evaluation_monitor: EvaluationMonitorSpecModel | None = None - evaluation_interval_epochs: int = Field(default=1, gt=0) - checkpoint_interval_epochs: int = Field(default=1, gt=0) algorithm: Literal["supervised"] = "supervised" - handoff: Literal["last", "best"] = "last" optimizer_state: Literal["reset", "continue"] = "reset" batch_size: int = Field(default=32, gt=0) accumulation_batches: int = Field(default=1, gt=0) @@ -1092,11 +873,12 @@ class ResolvedTrainingPhase(ConfigModel): sources: list[ResolvedTrainingSource] -def resolve_supervised_phases(authored_phases): +def resolve_supervised_phases(authored_phases, *, seed=0): """Resolve role defaults and epoch budgets without accessing datasets.""" phases = [ ResolvedTrainingPhase( name=phase.name, + resolved_seed=seed + phase_index, epochs=phase.epochs, evaluation_sources=[ _source(ref.source, 1) for ref in phase.evaluation.sources @@ -1106,9 +888,13 @@ def resolve_supervised_phases(authored_phases): evaluation_monitor=phase.evaluation.monitor if phase.evaluation is not None else None, - evaluation_interval_epochs=phase.evaluation_interval_epochs, - checkpoint_interval_epochs=phase.checkpoint_interval_epochs, - handoff=phase.handoff, + evaluation=phase.evaluation, + learning=phase.learning, + trainability=phase.trainability, + curriculum=phase.curriculum, + checkpoints=phase.checkpoints, + model_selection=phase.model_selection, + stopping=phase.stopping, optimizer_state=phase.optimizer_state, batch_size=phase.batch_size, accumulation_batches=phase.accumulation_batches, @@ -1132,7 +918,7 @@ def resolve_supervised_phases(authored_phases): for source in phase.sources ], ) - for phase in authored_phases + for phase_index, phase in enumerate(authored_phases) ] # A continued epoch scheduler spans the entire contiguous ownership chain. for index, phase in enumerate(phases): @@ -1161,13 +947,19 @@ class ResolvedSequifierConfig(ConfigModel): _prepared_inputs: Any = PrivateAttr(default=None) + ownership: dict[str, Any] = Field(default_factory=dict) + source_validation_fractions: dict[str, float] = Field(default_factory=dict) + normalization_by_interface: dict[str, FrozenNormalizationConfig] = Field( + default_factory=dict + ) + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") project_root: str model_name: str device: str seed: int - global_training: GlobalTrainingSpecModel + runtime_settings: TrainingRuntimeSettings model: ModelSpecModel dataset_training: dict[str, ResolvedDatasetTrainingSpec] authored: SequifierConfig | None = Field(default=None, exclude=True) @@ -1179,6 +971,7 @@ class ResolvedSequifierConfig(ConfigModel): preference_sources: dict[str, str] = Field(default_factory=dict) policy_bindings: dict[str, ResolvedPolicyBinding] = Field(default_factory=dict) training_plan: list[ResolvedTrainingPhase | PolicyPhase] + active_phase_index: int = 0 curriculum_training: bool = False curriculum_column: Optional[str] = None @@ -1204,11 +997,11 @@ def evaluation_sources(self) -> list[ResolvedTrainingSource]: @model_validator(mode="after") @beartype def validate_next_occurrence_metadata(self): - objective = get_objective_class(self.global_training.training_objective) + objective = get_objective_class(self.runtime_settings.training_objective) if not issubclass(objective, NextOccurrenceObjective): return self - next_config = self.global_training.next_occurrence_config + next_config = self.runtime_settings.next_occurrence_config if next_config is None: raise ValueError( "next_occurrence_config must be set for next_occurrence training" @@ -1256,7 +1049,7 @@ def validate_model_execution_plans(self): if dataset.model_interface in validated_interfaces: continue if dataset.interface.depth_layouts: - if self.global_training.read_format != "pt": + if any(part.read_format != "pt" for part in dataset.parts.values()): raise ValueError( "Preprocessed depth inputs require read_format: pt" ) @@ -1301,7 +1094,7 @@ def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: @dataclass(frozen=True) class LoadedTrainConfig: - config: SequifierConfig + config: Any resolved: ResolvedSequifierConfig metadata: dict[str, DatasetMetadata] @@ -1348,7 +1141,7 @@ def _layout_for_part_path( @beartype -def _evaluated_parts(config: SequifierConfig) -> set[str]: +def _evaluated_parts(config: Any) -> set[str]: selected: set[str] = set() sources = [ ref @@ -1519,7 +1312,7 @@ def _resolve_interface( name: str, spec: ModelInterfaceSpecModel, metadata: DatasetMetadata, - global_spec: GlobalTrainingSpecModel, + global_spec: TrainingRuntimeSettings, storage_layout: StoredWindowLayout, ) -> ResolvedModelInterface: if global_spec.training_objective is None or global_spec.context_length is None: @@ -1750,18 +1543,17 @@ def _interface_semantics(interface: ResolvedModelInterface) -> dict[str, Any]: @beartype def resolve_sequifier_config( - config: SequifierConfig, + config: Any, metadata: DatasetMetadata | dict[str, DatasetMetadata], - *, - part_overrides: Optional[dict[str, dict[str, str]]] = None, ) -> ResolvedSequifierConfig: """Resolve every dataset part, compatibility contract, and source.""" + validate_training_contract(config) if config.model is None: raise ValueError("resolved training requires a model") if ( - config.global_training.context_length is None - or config.global_training.training_objective is None + config.runtime_settings.context_length is None + or config.runtime_settings.training_objective is None ): raise ValueError( "resolved training requires training_objective and context_length" @@ -1785,11 +1577,11 @@ def resolve_sequifier_config( raise ValueError(f"Missing metadata for parts: {sorted(missing_metadata)}") evaluated = _evaluated_parts(config) - overrides = part_overrides or {} resolved_datasets: dict[str, ResolvedDatasetTrainingSpec] = {} interface_semantics: dict[str, dict[str, Any]] = {} for dataset_name, dataset_spec in config.dataset_training.items(): + learning = config.dataset_learning(dataset_name) if dataset_spec.model_interface is None: raise ValueError(f"Dataset {dataset_name!r} requires a model interface") interface_spec = config.model.interfaces[dataset_spec.model_interface] @@ -1808,22 +1600,17 @@ def resolve_sequifier_config( _assert_compatible(expected_signature, signature, ref) split_paths = list(part_metadata.split_paths) - override = overrides.get(ref, {}) - training_path = override.get("data_path") or ( - split_paths[0] if split_paths else None - ) + training_path = split_paths[0] if split_paths else None if training_path is None: raise ValueError(f"Training part {ref!r} needs at least one split path") - validation_path = override.get("validation_data_path") or ( - split_paths[1] if len(split_paths) > 1 else None - ) + validation_path = split_paths[1] if len(split_paths) > 1 else None if ref in evaluated and validation_path is None: raise ValueError( f"Evaluation part {ref!r} requires a second split path" ) - training_path = normalize_path(training_path, config.project_root) + training_path = normalize_path(training_path, config.run.project_root) validation_path = ( - normalize_path(validation_path, config.project_root) + normalize_path(validation_path, config.run.project_root) if validation_path is not None else None ) @@ -1840,10 +1627,8 @@ def resolve_sequifier_config( ) resolved_parts[part_name] = ResolvedDatasetPart( name=part_name, - metadata_config_path=( - override.get("metadata_config_path") - or part_spec.metadata_config_path - ), + read_format=part_spec.read_format, + metadata_config_path=part_spec.metadata_config_path, metadata=part_metadata, image_paths=part_spec.image_paths, preconditioning_image_paths=part_spec.preconditioning_image_paths, @@ -1858,7 +1643,7 @@ def resolve_sequifier_config( interface_spec.target_columns, first_metadata.column_data_types, ) - for column in dataset_spec.class_share_log_columns: + for column in learning.class_share_log_columns: if column not in interface_spec.target_columns: raise ValueError(f"Class-share column {column!r} must be a target") if target_types[column] != "categorical": @@ -1868,11 +1653,11 @@ def resolve_sequifier_config( if column not in first_metadata.id_maps: raise ValueError(f"Class-share column {column!r} needs an id_map") view = ModelWindowView( - context_length=config.global_training.context_length, - objective=config.global_training.training_objective, + context_length=config.runtime_settings.context_length, + objective=config.runtime_settings.training_objective, target_offset=target_offset_for_objective( - config.global_training.training_objective, - config.global_training.target_offset, + config.runtime_settings.training_objective, + config.runtime_settings.target_offset, ), ) first_training_layout: StoredWindowLayout | None = None @@ -1883,7 +1668,7 @@ def resolve_sequifier_config( selected_paths.append(resolved_part.validation_data_path) for path in selected_paths: layout = _layout_for_part_path( - resolved_part.metadata, path, config.project_root + resolved_part.metadata, path, config.run.project_root ) resolve_window_view(layout, view) if first_training_layout is None: @@ -1893,7 +1678,7 @@ def resolve_sequifier_config( dataset_spec.model_interface, interface_spec, first_metadata, - config.global_training, + config.runtime_settings, first_training_layout, ) for part_name, resolved_part in resolved_parts.items(): @@ -1915,7 +1700,7 @@ def resolve_sequifier_config( f"window width at least {required_aligned_width}." ) normalized_splits = [ - normalize_path(path, config.project_root) + normalize_path(path, config.run.project_root) for path in aligned_metadata.split_paths ] aligned_paths = { @@ -1953,7 +1738,7 @@ def resolve_sequifier_config( else: interface_semantics[dataset_spec.model_interface] = semantic_contract - for target, criterion in dataset_spec.criterion.items(): + for target, criterion in learning.criterion.items(): if ( criterion == "AccurateCrossEntropyLoss" and interface.target_column_types[target] != "categorical" @@ -1962,8 +1747,8 @@ def resolve_sequifier_config( f"AccurateCrossEntropyLoss requires categorical target {target!r}." ) - if dataset_spec.class_weights is not None: - for column, weights in dataset_spec.class_weights.items(): + if learning.class_weights is not None: + for column, weights in learning.class_weights.items(): if column in interface.categorical_hashing: raise ValueError( f"class_weights[{column!r}] describes canonical classes, but hash heads " @@ -1984,7 +1769,7 @@ def resolve_sequifier_config( ) for column in interface.categorical_hashing: - if column in interface.target_columns and dataset_spec.criterion[ + if column in interface.target_columns and learning.criterion[ column ] not in {"CrossEntropyLoss", "AccurateCrossEntropyLoss"}: raise ValueError( @@ -1997,42 +1782,48 @@ def resolve_sequifier_config( model_interface=dataset_spec.model_interface, interface=interface, parts=resolved_parts, - criterion=dataset_spec.criterion, - class_weights=dataset_spec.class_weights, - loss_weights=dataset_spec.loss_weights, - class_share_log_columns=dataset_spec.class_share_log_columns, - freeze=dataset_spec.freeze, + criterion=learning.criterion, + class_weights=learning.class_weights, + loss_weights=learning.loss_weights, + class_share_log_columns=learning.class_share_log_columns, ) - phases = resolve_supervised_phases(config.training_plan.phases) + phases = resolve_supervised_phases( + config.training_plan.phases, seed=config.run.reproducibility.seed + ) resolved = ResolvedSequifierConfig( - project_root=config.project_root, - model_name=config.model_name, - device=config.device, - seed=config.seed, - global_training=config.global_training, - model=config.model, + ownership=config.model_dump(mode="json"), + authored=config, + source_validation_fractions={ + f"{name}.{part}": spec.split.validation_fraction + for name, spec in config.dataset_training.items() + for part in spec.parts + }, + project_root=config.run.project_root, + model_name=config.run.model_name, + device=config.run.execution.device, + seed=config.run.reproducibility.seed, + runtime_settings=config.runtime_settings, + model=config.model.architecture_model(), dataset_training=resolved_datasets, training_plan=phases, - curriculum_training=config.training_plan.curriculum_training, - curriculum_column=config.training_plan.curriculum_column, - export_generative_model=config.export_generative_model, - export_embedding_model=config.export_embedding_model, - embedding_layer_names=config.embedding_layer_names, - export_onnx=config.export_onnx, - export_pt=config.export_pt, - export_with_dropout=config.export_with_dropout, + curriculum_training=bool(config.training_plan.phases[0].curriculum), + curriculum_column=config.training_plan.phases[0].curriculum.column + if config.training_plan.phases[0].curriculum + else None, + export_generative_model=config.run.export.generative_model, + export_embedding_model=config.run.export.embedding_model, + embedding_layer_names=config.run.export.embedding_layer_names, + export_onnx=config.run.export.onnx, + export_pt=config.run.export.pt, + export_with_dropout=config.run.export.with_dropout, ) + from sequifier.config.input_contract_resolution import validate_input_contracts + + validate_input_contracts(config, resolved) return resolved -_SENSITIVE_OVERRIDES = { - "data_path", - "validation_data_path", - "metadata_config_path", - "preprocessing_data_path", - "input_columns", -} _INLINE_METADATA_KEYS = { "depth_layouts", "image_sources", @@ -2040,7 +1831,6 @@ def resolve_sequifier_config( "tensor_payload_version", "metadata_by_part", "column_data_types", - "column_types", "static_columns", "static_features_path", "n_classes", @@ -2061,39 +1851,9 @@ def resolve_sequifier_config( } -@beartype -def _override_part(config: SequifierConfig) -> str: - if len(config.dataset_training) != 1: - raise ValueError( - "Dataset-sensitive legacy CLI overrides require exactly one dataset" - ) - dataset_name, dataset = next(iter(config.dataset_training.items())) - if len(dataset.parts) == 1: - return f"{dataset_name}.{next(iter(dataset.parts))}" - refs = { - source.source - for phase in config.training_plan.phases - for source in phase.sources - if source.source.startswith(f"{dataset_name}.") - } - refs.update( - source.source - for phase in config.training_plan.phases - if phase.algorithm == "supervised" and phase.evaluation is not None - for source in phase.evaluation.sources - if source.source.startswith(f"{dataset_name}.") - ) - if len(refs) == 1: - return next(iter(refs)) - raise ValueError( - "Legacy metadata/data-path CLI override is ambiguous because multiple " - "dataset parts remain possible" - ) - - @beartype def _inline_metadata( - values: dict[str, Any], global_spec: GlobalTrainingSpecModel + values: dict[str, Any], global_spec: TrainingRuntimeSettings ) -> DatasetMetadata: layout = values.get("storage_layout") if isinstance(layout, StoredWindowLayout): @@ -2117,9 +1877,7 @@ def _inline_metadata( "depth_layouts": values.get("depth_layouts", {}), "tensor_payload_version": values.get("tensor_payload_version", 1), "split_paths": values.get("split_paths", []), - "column_data_types": values.get( - "column_data_types", values.get("column_types", {}) - ), + "column_data_types": values.get("column_data_types", {}), "static_columns": values.get("static_columns", []), "static_features_path": values.get("static_features_path"), "image_sources": values.get("image_sources", {}), @@ -2161,57 +1919,34 @@ def load_train_config_with_source( metadata_inline_values = { key: args.pop(key) for key in list(args) if key in _INLINE_METADATA_KEYS } - sensitive = { - key: args.pop(key) for key in list(args) if key in _SENSITIVE_OVERRIDES - } - for key in ("model_name", "seed", "device"): - if key in args: - raw[key] = args.pop(key) + from sequifier.config.command_ownership import apply_run_overrides + + raw, args = apply_run_overrides(raw, args) from sequifier.config.composition import merge_config_fragments overrides = { - key: args.pop(key) - for key in ("global_training", "training_plan") - if key in args + key: args.pop(key) for key in ("run", "model", "training_plan") if key in args } raw = merge_config_fragments((raw, overrides)) if args: raise ValueError(f"Unsupported training CLI overrides: {sorted(args)}") - config = try_catch_excess_keys(config_path, SequifierConfig, raw) + authored = try_catch_excess_keys(config_path, SequifierConfig, raw) + config = authored if ( config.initialize_from is not None or config.policy_bindings or any(phase.algorithm != "supervised" for phase in config.training_plan.phases) ): - if sensitive or metadata_inline_values or skip_metadata: + if metadata_inline_values or skip_metadata: raise ValueError( "policy/mixed plans require source declarations instead of inline metadata overrides" ) from sequifier.config.training_resolution import resolve_training_run return LoadedTrainConfig( - config=config, resolved=resolve_training_run(config), metadata={} + config=authored, resolved=resolve_training_run(config), metadata={} ) - part_overrides: dict[str, dict[str, str]] = {} - if sensitive: - selected_part = _override_part(config) - dataset_name = selected_part.partition(".")[0] - if "input_columns" in sensitive: - interface_name = config.dataset_training[dataset_name].model_interface - config.model.interfaces[interface_name].input_columns = sensitive.pop( - "input_columns" - ) - override = {} - if "preprocessing_data_path" in sensitive: - override["metadata_config_path"] = ( - metadata_config_path_from_preprocessing_data_path( - sensitive.pop("preprocessing_data_path") - ) - ) - override.update(sensitive) - part_overrides[selected_part] = override - metadata_by_part: dict[str, DatasetMetadata] = {} all_part_refs = [ (dataset_name, part_name, part) @@ -2225,29 +1960,27 @@ def load_train_config_with_source( ) if "metadata_by_part" in metadata_inline_values: for ref, values in metadata_inline_values["metadata_by_part"].items(): - metadata_by_part[ref] = _inline_metadata(values, config.global_training) + metadata_by_part[ref] = _inline_metadata( + values, config.runtime_settings + ) else: dataset_name, part_name, _ = all_part_refs[0] metadata_by_part[f"{dataset_name}.{part_name}"] = _inline_metadata( - metadata_inline_values, config.global_training + metadata_inline_values, config.runtime_settings ) else: for dataset_name, part_name, part in all_part_refs: ref = f"{dataset_name}.{part_name}" - configured_path = part_overrides.get(ref, {}).get( - "metadata_config_path", part.metadata_config_path - ) + configured_path = part.metadata_config_path if configured_path is None: raise ValueError(f"Part {ref!r} has no metadata_config_path") metadata_by_part[ref] = load_dataset_metadata( - normalize_path(configured_path, config.project_root) + normalize_path(configured_path, config.run.project_root) ) - resolved = resolve_sequifier_config( - config, metadata_by_part, part_overrides=part_overrides - ) + resolved = resolve_sequifier_config(config, metadata_by_part) return LoadedTrainConfig( - config=config, resolved=resolved, metadata=metadata_by_part + config=authored, resolved=resolved, metadata=metadata_by_part ) @@ -2275,7 +2008,7 @@ class SelectedInterfaceConfig: model_name: str device: str seed: int - global_training: GlobalTrainingSpecModel + runtime_settings: TrainingRuntimeSettings training_objective: str input_columns: list[str] static_columns: list[str] @@ -2333,8 +2066,8 @@ def interface_build_view( model_name=config.model_name, device=config.device, seed=config.seed, - global_training=config.global_training, - training_objective=config.global_training.training_objective, + runtime_settings=config.runtime_settings, + training_objective=config.runtime_settings.training_objective, static_columns=( interface.static_conditioning.columns if interface.static_conditioning @@ -2385,17 +2118,17 @@ def dataset_part_view( ) return SelectedDatasetPartConfig( - batch_size=config.training_plan[0].batch_size + batch_size=config.training_plan[config.active_phase_index].batch_size if config.training_plan - else config.global_training.inference_batch_size, + else config.runtime_settings.inference_batch_size, **interface_values, dataset_training=dataset, part=part, curriculum_training=config.curriculum_training, curriculum_column=config.curriculum_column, - epochs=sum(phase.epochs for phase in config.training_plan), - read_format=config.global_training.read_format, - window_stride=config.global_training.window_stride, + epochs=sum(getattr(phase, "epochs", 0) for phase in config.training_plan), + read_format=part.read_format, + window_stride=config.runtime_settings.window_stride, data_path=part.training_data_path, validation_data_path=part.validation_data_path, metadata_config_path=part.metadata_config_path, @@ -2403,3 +2136,278 @@ def dataset_part_view( TrainModel = ResolvedSequifierConfig + + +# Public authored API; native schemas above are resolved execution projections. +from sequifier.config.ownership import ( # noqa: E402 + InputContract, + ModelExecutionSettings, + RunSettings, + SplitSettings, +) + + +class OwnedModelInterfaceSpec(ModelInterfaceSpecModel): + input_contract: InputContract = Field(default_factory=InputContract) + + +class OwnedModelSpec(ConfigModel): + model_config = ConfigDict(extra="forbid") + backbone: BackboneComponentConfig | None = None + execution: ModelExecutionSettings = Field(default_factory=ModelExecutionSettings) + interfaces: dict[str, OwnedModelInterfaceSpec] = Field(default_factory=dict) + + def architecture_model(self): + if self.backbone is None: + raise ValueError("model backbone has not been resolved") + return ModelSpecModel.model_validate( + self.model_dump( + exclude={ + "execution": True, + "interfaces": {"__all__": {"input_contract"}}, + } + ) + ) + + +class DatasetTrainingSpecModel(ConfigModel): + model_config = ConfigDict(extra="forbid") + model_interface: str | None = None + parts: dict[str, DatasetPart] = Field(min_length=1) + split: SplitSettings = Field(default_factory=SplitSettings) + + +class SequifierConfig(ConfigModel): + _runtime_settings: TrainingRuntimeSettings | None = PrivateAttr(default=None) + """The authored training configuration, with one owner for each setting.""" + + model_config = ConfigDict(extra="forbid") + run: RunSettings + model: OwnedModelSpec | None = None + initialize_from: str | None = None + dataset_training: dict[str, DatasetTrainingSpecModel] = Field(default_factory=dict) + policy_bindings: dict[str, PolicyBinding] = Field(default_factory=dict) + training_plan: TrainingPlanModel + + @model_validator(mode="before") + @classmethod + def apply_defaults(cls, values): + if not isinstance(values, dict): + return values + values = copy.deepcopy(values) + model = values.get("model") + if isinstance(model, ConfigModel): + model = values["model"] = model.model_dump(exclude_unset=True) + model = model or {} + if not isinstance(model, dict): + return values + interfaces = model.get("interfaces", {}) + backbone = model.get("backbone") + context = model.get("execution", {}).get("context_length") + if isinstance(backbone, dict) and context is not None: + architecture = backbone.get("architecture") + if isinstance(architecture, dict): + architecture.setdefault("max_context_length", context) + datasets = values.get("dataset_training", {}) + if not isinstance(datasets, dict): + return values + for name, dataset in datasets.items(): + if isinstance(dataset, ConfigModel): + dataset = datasets[name] = dataset.model_dump(exclude_unset=True) + if not isinstance(dataset, dict): + continue + if isinstance(interfaces, dict) and len(interfaces) == 1: + dataset.setdefault("model_interface", next(iter(interfaces))) + for part in dataset.get("parts", {}).values(): + if isinstance(part, dict): + part.setdefault("type", "prepared") + plan = values.get("training_plan", {}) + if isinstance(plan, ConfigModel): + plan = values["training_plan"] = plan.model_dump(exclude_unset=True) + if not isinstance(plan, dict): + return values + implicit = None + if len(datasets) == 1: + name, dataset = next(iter(datasets.items())) + parts = dataset.get("parts", {}) + implicit = f"{name}.{next(iter(parts))}" if len(parts) == 1 else name + for phase in plan.get("phases", []): + phase.setdefault("algorithm", "supervised") + if implicit and phase["algorithm"] in { + "supervised", + "policy_imitation", + "iql", + }: + phase.setdefault("sources", [{"source": implicit}]) + return values + + @model_validator(mode="after") + def validate_contract(self): + if ( + self.model is None or self.model.backbone is None + ) and self.initialize_from is None: + raise ValueError("a run requires a complete model or initialize_from") + next_occurrence = ( + self.model.execution.next_occurrence_config if self.model else None + ) + for phase in self.training_plan.phases: + if ( + phase.algorithm == "supervised" + and phase.learning.next_occurrence_config is not None + ): + if ( + next_occurrence is not None + and next_occurrence != phase.learning.next_occurrence_config + ): + raise ValueError( + "phase next_occurrence_config differs from durable model.execution contract" + ) + next_occurrence = phase.learning.next_occurrence_config + for name, dataset in self.dataset_training.items(): + _identifier(name, "Dataset name") + for part in dataset.parts: + _identifier(part, "Part name") + for phase in self.training_plan.phases: + refs = list(getattr(phase, "sources", [])) + list( + getattr(phase.evaluation, "sources", None) or [] + ) + if phase.algorithm == "dpo": + refs.append(phase.preferences) + for ref in refs: + name, _, part = ref.source.partition(".") + if name not in self.dataset_training or ( + part and part not in self.dataset_training[name].parts + ): + raise ValueError( + f"{phase.name}: unknown dataset source {ref.source!r}" + ) + if phase.algorithm == "supervised": + for name in {ref.source.partition(".")[0] for ref in refs}: + if name not in phase.learning.datasets: + raise ValueError( + f"{phase.name}: declare learning.datasets.{name}" + ) + interface_name = self.dataset_training[name].model_interface + if self.model and interface_name in self.model.interfaces: + targets = set( + self.model.interfaces[interface_name].target_columns + ) + learning = phase.learning.datasets[name] + if set(learning.criterion) != targets: + raise ValueError( + f"{phase.name}: learning.datasets.{name}.criterion keys must equal interface targets" + ) + for label, weights in ( + ("class_weights", learning.class_weights), + ("loss_weights", learning.loss_weights), + ): + if weights is not None and weights.keys() - targets: + raise ValueError( + f"{phase.name}: {label} references unknown targets" + ) + if learning.loss_weights is not None and all( + learning.loss_weights.get(target, 1.0) == 0 + for target in targets + ): + raise ValueError( + f"{phase.name}: at least one target must have positive loss weight" + ) + if any(phase.algorithm != "supervised" for phase in self.training_plan.phases): + execution = self.run.execution + if execution.distribution.strategy != "local": + raise ValueError("policy phases currently require local execution") + if execution.compilation.torch_compile != "none": + raise ValueError( + "policy phases do not support run.execution.compilation" + ) + if execution.precision != type(execution.precision)(): + raise ValueError( + "policy phases do not support custom precision settings" + ) + if execution.max_concat_tensors != 12: + raise ValueError( + "policy phases do not support custom max_concat_tensors" + ) + if self.run.logging.log_interval_batches != 10: + raise ValueError("policy phases do not support batch logging intervals") + export = self.run.export + if ( + export.onnx + or export.embedding_model + or export.with_dropout + or not export.generative_model + or not export.pt + or export.batch_size != 1 + ): + raise ValueError( + "policy-containing plans require portable PT actor export; graph/embedding/dropout/batched exports are unsupported" + ) + if ( + execution.data_loading.num_workers + or not execution.data_loading.load_full_data_to_ram + or execution.data_loading.max_ram_gb != 16 + ): + raise ValueError( + "policy phases do not support worker or lazy-loading controls" + ) + return self + + def _build_runtime_settings(self): + """Resolve execution settings from their current owners.""" + run = self.run + execution = run.execution + contract = ( + self.model.execution.model_dump(exclude_unset=True) if self.model else {} + ) + training = { + **contract, + **execution.precision.model_dump(), + **execution.compilation.model_dump(), + **execution.data_loading.model_dump(), + **run.logging.model_dump(), + "max_concat_tensors": execution.max_concat_tensors, + "distributed": execution.distribution.strategy != "local", + "data_parallelism": None + if execution.distribution.strategy == "local" + else execution.distribution.strategy, + "world_size": execution.distribution.world_size, + "backend": execution.distribution.backend, + "fsdp_cpu_offload": execution.distribution.cpu_offload, + "resume": run.recovery.resume.model_dump(), + "latest_checkpoint_interval_minutes": run.recovery.latest_checkpoint_interval_minutes, + "enforce_determinism": run.reproducibility.enforce_determinism, + "inference_batch_size": run.export.batch_size, + } + supervised = [ + phase + for phase in self.training_plan.phases + if phase.algorithm == "supervised" + ] + if supervised: + training.update( + supervised[0].learning.model_dump( + exclude={"datasets"}, exclude_none=True + ) + ) + return TrainingRuntimeSettings.model_validate(training) + + @property + def runtime_settings(self): + if self._runtime_settings is None: + self._runtime_settings = self._build_runtime_settings() + return self._runtime_settings + + @runtime_settings.setter + def runtime_settings(self, value): + self._runtime_settings = value + + def dataset_learning(self, name): + for phase in self.training_plan.phases: + if phase.algorithm == "supervised" and name in phase.learning.datasets: + return phase.learning.datasets[name] + from sequifier.config.learning_config import DatasetLearningConfig + + return DatasetLearningConfig() + + +ResolvedSequifierConfig.model_rebuild() diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index ba2df6aa..90355f78 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -9,10 +9,11 @@ ) from sequifier.config.train_config import ( DatasetPartSpecModel, - GlobalTrainingSpecModel, + OwnedModelSpec, PreferencePartSpecModel, SequencePartSpecModel, SequifierConfig, + TrainingRuntimeSettings, _resolve_interface, resolve_sequifier_config, resolve_supervised_phases, @@ -20,6 +21,20 @@ from sequifier.special_tokens import SPECIAL_TOKEN_IDS +def _normalization(config, name): + if config.model and name in config.model.interfaces: + return config.model.interfaces[name].input_contract.normalization + return None + + +def _source_fractions(config): + return { + f"{name}.{part}": spec.split.validation_fraction + for name, spec in config.dataset_training.items() + for part in spec.parts + } + + def source_parts(config, reference): """Expand a reference without silently filtering out incompatible parts.""" dataset, _, part = reference.partition(".") @@ -154,15 +169,15 @@ def _inferred_metadata(config, name, binding): } counts[column] = SPECIAL_TOKEN_IDS.user_start + len(labels) types[column] = "Int64" - normalization = config.global_training.normalization + normalization = _normalization(config, name) statistics = ( normalization.statistics if normalization and normalization.source == "statistics" else {} ) return DatasetMetadata( - window_length=config.global_training.context_length - + max(1, config.global_training.target_offset), + window_length=config.runtime_settings.context_length + + max(1, config.runtime_settings.target_offset), column_data_types=types, n_classes=counts, id_maps=maps, @@ -193,6 +208,7 @@ def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): if phase is None or phase.algorithm not in {"policy_imitation", "iql", "dpo"}: return metadata interface = config.model.interfaces[name] + normalization = _normalization(config, name) static = ( interface.static_conditioning.columns if interface.static_conditioning else [] ) @@ -206,14 +222,12 @@ def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): - set(categorical) - metadata.selected_columns_statistics.keys() ) - if not missing_codecs and ( - not missing_statistics or config.global_training.normalization is not None - ): + if not missing_codecs and (not missing_statistics or normalization is not None): return metadata provisional = PolicyFittingSpec( **layout.binding_values(), binding=binding, - context_length=config.global_training.context_length, + context_length=config.runtime_settings.context_length, interface_name=name, ) sources, preferences = TrainingSources(config, root).resolve(prepared=False) @@ -226,11 +240,12 @@ def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): ): return metadata # Prepared sources already own frozen preprocessing metadata. run = SimpleNamespace( - validation_fraction=config.global_training.validation_fraction, - seed=config.seed, + validation_fraction=0.2, + source_validation_fractions=_source_fractions(config), + seed=config.run.reproducibility.seed, preferences=preferences, sources=sources, - episode_cache_size=config.global_training.episode_cache_size, + episode_cache_size=config.runtime_settings.episode_cache_size, ) inputs = prepared_inputs.resolve(run) inputs.revalidate(inputs.identity, f"interfaces.{name}.fitting_sources") @@ -241,7 +256,7 @@ def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): values["n_classes"].update(fitted.categorical_n_classes) values["id_maps"].update(fitted.id_maps) if ( - config.global_training.normalization is None + normalization is None and not metadata.depth_layouts and not metadata.image_sources ): @@ -287,37 +302,50 @@ def resolve_training_run(authored: SequifierConfig): from sequifier.rl.prepared_inputs import PreparedTrainingInputs prepared_inputs = PreparedTrainingInputs() - root = Path(authored.project_root).resolve() config = authored.model_copy(deep=True) - config.project_root = str(root) + root = Path(config.run.project_root).resolve() + config.run.project_root = str(root) + training = config.runtime_settings source = None if config.initialize_from: config.initialize_from = str((root / config.initialize_from).resolve()) source, _ = resolved_config_from_model_config( read_execution_artifact(config.initialize_from).model_config.values, - device=config.device, + device=config.run.execution.device, ) if config.model is not None: from sequifier.artifacts.model_config import extend_prediction_distributions extend_prediction_distributions(source, config.model) - config.model = source.model.model_copy(deep=True) + declared = config.model + values = source.model.model_dump() + values["execution"] = ( + declared.execution.model_dump(exclude_unset=True) if declared else {} + ) + if declared: + for name, interface in declared.interfaces.items(): + if name in values["interfaces"]: + values["interfaces"][name]["input_contract"] = ( + interface.input_contract.model_dump() + ) + config.model = OwnedModelSpec.model_validate(values) for field in ( "context_length", "training_objective", "target_offset", "forward_attention", ): - actual = getattr(source.global_training, field) + actual = getattr(source.runtime_settings, field) if ( - field in config.global_training.model_fields_set - and getattr(config.global_training, field) != actual + field in training.model_fields_set + and getattr(training, field) != actual ): raise ValueError( - f"global_training.{field} differs from initialize_from" + f"model.execution.{field} differs from initialize_from" ) - setattr(config.global_training, field, actual) + setattr(training, field, actual) assert config.model is not None + assert config.model.backbone is not None for dataset in config.dataset_training.values(): if dataset.model_interface is None: if len(config.model.interfaces) != 1: @@ -327,7 +355,7 @@ def resolve_training_run(authored: SequifierConfig): raise ValueError( f"unknown dataset model_interface {dataset.model_interface!r}" ) - training = config.global_training + training = config.runtime_settings if training.context_length is None: training.context_length = config.model.backbone.architecture.max_context_length if training.training_objective is None: @@ -335,16 +363,28 @@ def resolve_training_run(authored: SequifierConfig): phase.algorithm == "supervised" for phase in config.training_plan.phases ): raise ValueError( - "supervised phases require global_training.training_objective" + "supervised phases require model.execution.training_objective" ) training.training_objective = "policy" training.target_offset = 0 - training = GlobalTrainingSpecModel.model_validate(training.model_dump()) - config.global_training = training + training = TrainingRuntimeSettings.model_validate(training.model_dump()) + config.runtime_settings = training + for field in ( + "context_length", + "training_objective", + "target_offset", + "forward_attention", + "next_occurrence_config", + ): + setattr(config.model.execution, field, getattr(training, field)) # All declared metadata for an interface is an assertion of the same facts. metadata = {} - declarations = training.interface_metadata_paths + declarations = { + name: value.input_contract.metadata_config_path + for name, value in config.model.interfaces.items() + if value.input_contract.metadata_config_path + } for name, path in (declarations or {}).items(): if name not in config.model.interfaces: raise ValueError(f"unknown metadata model_interface {name!r}") @@ -408,13 +448,13 @@ def resolve_training_run(authored: SequifierConfig): name, interface, value, training, value.storage_layout ) resolved = execution_config( - config.model, + config.model.architecture_model(), interfaces, training, - device=config.device, + device=config.run.execution.device, project_root=str(root), - model_name=config.model_name, - seed=config.seed, + model_name=config.run.model_name, + seed=config.run.reproducibility.seed, ) for name, binding in config.policy_bindings.items(): if name not in interfaces: @@ -483,6 +523,13 @@ def resolve_training_run(authored: SequifierConfig): previous.prediction_distributions ) resolved = native + resolved.ownership = config.model_dump(mode="json") + resolved.source_validation_fractions = _source_fractions(config) + resolved.normalization_by_interface = { + name: value + for name in interfaces + if (value := _normalization(config, name)) is not None + } resolved.policy_bindings = { name: interface.policy for name, interface in interfaces.items() @@ -492,21 +539,30 @@ def resolve_training_run(authored: SequifierConfig): resolved.authored = config resolved.initialize_from = config.initialize_from resolved.model_path = str( - (root / (config.model_path or f"models/{config.model_name}/model.pt")).resolve() + ( + root + / ( + config.run.outputs.model_path + or f"models/{config.run.model_name}/model.pt" + ) + ).resolve() ) resolved.report_path = str( ( root / ( - config.report_path - or f"outputs/training/{config.model_name}/report.json" + config.run.outputs.report_path + or f"outputs/training/{config.run.model_name}/report.json" ) ).resolve() ) resolved.phase_directory = str( ( root - / (config.phase_directory or f"outputs/training/{config.model_name}/phases") + / ( + config.run.outputs.phase_directory + or f"outputs/training/{config.run.model_name}/phases" + ) ).resolve() ) resolved.sequence_sources = sequences @@ -525,15 +581,19 @@ def resolve_training_run(authored: SequifierConfig): resolved.training_plan = [ supervised.get(phase.name, phase) for phase in config.training_plan.phases ] - for key in ( - "export_generative_model", - "export_embedding_model", - "embedding_layer_names", - "export_onnx", - "export_pt", - "export_with_dropout", - ): - setattr(resolved, key, getattr(config, key)) + for index, phase in enumerate(resolved.training_plan): + if phase.algorithm == "supervised": + phase.resolved_seed = config.run.reproducibility.seed + index + export = config.run.export + resolved.export_generative_model = export.generative_model + resolved.export_embedding_model = export.embedding_model + resolved.embedding_layer_names = export.embedding_layer_names + resolved.export_onnx = export.onnx + resolved.export_pt = export.pt + resolved.export_with_dropout = export.with_dropout + from sequifier.config.input_contract_resolution import validate_input_contracts + + validate_input_contracts(config, resolved) return resolved @@ -572,7 +632,7 @@ def policy_run_config( binding = execution.interface(name).policy if binding is None: raise ValueError("selected interface has no policy binding") - training = config.global_training + training = config.runtime_settings resume = training.resume root = Path(config.project_root) checkpoint = ( @@ -595,12 +655,14 @@ def policy_run_config( execution_model=model_execution_config(execution), policy_interface=name, initialize_from=initialization, - normalization=training.normalization, + normalization=config.normalization_by_interface.get(name), units=binding.units, sources=config.sequence_sources, preferences=config.preference_sources, phases=phases, - validation_fraction=training.validation_fraction, + validation_fraction=0.2, + source_validation_fractions=config.source_validation_fractions, + latest_checkpoint_interval_minutes=training.latest_checkpoint_interval_minutes, episode_cache_size=training.episode_cache_size, resume=ResumeConfig( checkpoint_path=str(checkpoint), @@ -609,6 +671,7 @@ def policy_run_config( else (resume.policy if resume else "never"), mode=resume.mode if resume else "reset_environments", ), + embedding_layer_names=config.embedding_layer_names, model_path=str(destination or config.model_path), report_path=config.report_path if destination is None diff --git a/src/sequifier/docs.py b/src/sequifier/docs.py index c2fe5705..ac653633 100644 --- a/src/sequifier/docs.py +++ b/src/sequifier/docs.py @@ -11,6 +11,7 @@ "documentation/commands/visualize-training.md", "documentation/configs/hyperparameter-search.md", "documentation/configs/naming.md", + "documentation/configs/ownership.md", "documentation/training/multi-gpu-training.md", "documentation/policy-learning.md", "documentation/supervised-policy-learning.md", diff --git a/src/sequifier/evaluation/service.py b/src/sequifier/evaluation/service.py index 9c15b8ea..99b38915 100644 --- a/src/sequifier/evaluation/service.py +++ b/src/sequifier/evaluation/service.py @@ -74,7 +74,7 @@ def __init__(self, loss_service: LossService) -> None: def _baseline_definition(dataset: Any) -> str: """Identify inputs that affect raw baseline loss statistics.""" interface = dataset.config.interface - training = getattr(dataset.objective.config, "global_training", None) + training = getattr(dataset.objective.config, "runtime_settings", None) def model_value(value: Any) -> Any: dump = getattr(value, "model_dump", None) diff --git a/src/sequifier/export/preflight.py b/src/sequifier/export/preflight.py index a5c6cbef..3c767006 100644 --- a/src/sequifier/export/preflight.py +++ b/src/sequifier/export/preflight.py @@ -196,7 +196,7 @@ def _worker(request_path): request_path.parent / f"{index}-generative.onnx", interface_name=dataset.model_interface, batch_size=2, - context_length=config.global_training.context_length, + context_length=config.runtime_settings.context_length, training=request["dropout"], ) if request["embedding"]: @@ -209,7 +209,7 @@ def _worker(request_path): request_path.parent / f"{index}-embedding.onnx", interface_name=None, batch_size=2, - context_length=config.global_training.context_length, + context_length=config.runtime_settings.context_length, training=request["dropout"], ) print( diff --git a/src/sequifier/export/service.py b/src/sequifier/export/service.py index b7c2bc43..e55f7bee 100644 --- a/src/sequifier/export/service.py +++ b/src/sequifier/export/service.py @@ -74,9 +74,9 @@ def _export( del network export_config = self.config.model_copy(deep=True) export_config.device = "cpu" - export_config.global_training.distributed = False - export_config.global_training.data_parallelism = None - export_config.global_training.torch_compile = "none" + export_config.runtime_settings.distributed = False + export_config.runtime_settings.data_parallelism = None + export_config.runtime_settings.torch_compile = "none" export_network = build_transformer_network( export_config, device=torch.device("cpu"), initialize=False ).network @@ -136,8 +136,8 @@ def _export( dataset.interface, path, interface_name=dataset.model_interface, - batch_size=self.config.global_training.inference_batch_size, - context_length=self.config.global_training.context_length, + batch_size=self.config.runtime_settings.inference_batch_size, + context_length=self.config.runtime_settings.context_length, training=self.config.export_with_dropout, metadata={ MODEL_CONFIG_KEY: json.dumps( @@ -169,8 +169,8 @@ def _export( dataset.interface, path, interface_name=None, - batch_size=self.config.global_training.inference_batch_size, - context_length=self.config.global_training.context_length, + batch_size=self.config.runtime_settings.inference_batch_size, + context_length=self.config.runtime_settings.context_length, training=self.config.export_with_dropout, metadata={ MODEL_CONFIG_KEY: json.dumps( diff --git a/src/sequifier/hyperparameter_search.py b/src/sequifier/hyperparameter_search.py index 3fd48c97..23c44d1c 100644 --- a/src/sequifier/hyperparameter_search.py +++ b/src/sequifier/hyperparameter_search.py @@ -42,7 +42,7 @@ def _monitored_dataset(run_config: Any) -> tuple[str | None, int]: dataset_count = len(dataset_names) if dataset_count == 1: return None, dataset_count - evaluation = getattr(run_config, "evaluation", None) + evaluation = run_config.training_plan.phases[-1].evaluation monitor = getattr(evaluation, "monitor", None) if monitor is None: raise ValueError( @@ -87,7 +87,7 @@ def objective( """Run one Optuna trial through the CLI trainer and validation metrics.""" if run_config is None: run_config = config.sample_trial(trial, accepted_trials) - run_name = run_config.model_name + run_name = run_config.run.model_name monitored_dataset, dataset_count = _monitored_dataset(run_config) config_path = os.path.join( @@ -230,7 +230,7 @@ def consume_metrics( elif exit_code != 0: raise RuntimeError(f"Training failed with exit code {exit_code}") - model_type = "onnx" if run_config.export_onnx else "pt" + model_type = "onnx" if run_config.run.export.onnx else "pt" model_path, _last_epoch = get_best_model_path( config.project_root, run_name, diff --git a/src/sequifier/integration/callbacks.py b/src/sequifier/integration/callbacks.py index 3d85daa5..7e2d63cb 100644 --- a/src/sequifier/integration/callbacks.py +++ b/src/sequifier/integration/callbacks.py @@ -264,7 +264,7 @@ def validate_execution( if needs_eager and torch_compile != "none": raise ValueError( f"Integration {item.instance.integration_id!r} requires eager " - "execution; set global_training.torch_compile to 'none'." + "execution; set runtime_settings.torch_compile to 'none'." ) if self.distributed and needs_eager and item.rank_policy == "rank_zero": raise ValueError( diff --git a/src/sequifier/io/config.py b/src/sequifier/io/config.py deleted file mode 100644 index ac7aa894..00000000 --- a/src/sequifier/io/config.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Small compatibility boundary for standalone data-loader tests and callers.""" - -from typing import Any - -from sequifier.typechecking import beartype - - -@beartype -def global_training(config: Any) -> Any: - training = vars(config).get("global_training") - if training is not None: - return training - return config.training_spec diff --git a/src/sequifier/io/folder_dataset.py b/src/sequifier/io/folder_dataset.py index a5bdd0ac..41f75403 100644 --- a/src/sequifier/io/folder_dataset.py +++ b/src/sequifier/io/folder_dataset.py @@ -20,7 +20,6 @@ stored_window_layout_from_metadata, ) from sequifier.io.batch import SequifierBatch -from sequifier.io.config import global_training from sequifier.io.images import ImageStore from sequifier.io.iteration_state import ( read_shared_int, @@ -89,7 +88,7 @@ def _initialize_window_layout(self) -> None: @beartype def _calculate_total_batches(self, target_samples: int) -> int: - num_workers = global_training(self.config).num_workers + num_workers = self.config.runtime_settings.num_workers num_workers_to_use = num_workers if num_workers > 0 else 1 total_batches = 0 diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index 655f44e8..da88b945 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -197,6 +197,8 @@ def use_config(self, config): def prepare_preferences(run, phase, config, catalog, cache): + from sequifier.rl.offline_data import phase_validation_fraction + key = ( "preferences", phase.preferences.source, @@ -234,7 +236,7 @@ def prepare_preferences(run, phase, config, catalog, cache): ) else: pairs, validation = split_components( - pairs, data.components, run.validation_fraction, run.seed + pairs, data.components, phase_validation_fraction(run, phase), run.seed ) result = PreferenceData( PreferenceDataset(pairs, corpora, config), diff --git a/src/sequifier/io/sequifier_dataset_from_file.py b/src/sequifier/io/sequifier_dataset_from_file.py index 80a04781..54d9fed4 100644 --- a/src/sequifier/io/sequifier_dataset_from_file.py +++ b/src/sequifier/io/sequifier_dataset_from_file.py @@ -14,7 +14,7 @@ resolve_window_sampling_plan, ) from sequifier.io.batch import SequifierBatch -from sequifier.io.config import global_training +from sequifier.io.config import runtime_settings from sequifier.io.images import ImageStore from sequifier.io.iteration_state import ( read_shared_int, @@ -184,7 +184,7 @@ def set_start_batch(self, start_batch: int): @beartype def __len__(self) -> int: - num_workers = max(1, global_training(self.config).num_workers) + num_workers = max(1, runtime_settings(self.config).num_workers) total_batches = 0 for worker_id in range(num_workers): worker_samples = self.n_samples // num_workers + ( diff --git a/src/sequifier/io/yaml.py b/src/sequifier/io/yaml.py index 563db27f..bc7f3b52 100644 --- a/src/sequifier/io/yaml.py +++ b/src/sequifier/io/yaml.py @@ -10,14 +10,6 @@ def represent_sequifier_object(dumper, data): """Represent sequifier config objects as plain YAML mappings.""" values = data.model_dump() if isinstance(data, BaseModel) else vars(data).copy() - for field_name in ("freeze_groups", "freeze_except_groups"): - if values.get(field_name) is None: - values.pop(field_name, None) - for component_name in ("ingestion", "backbone", "decoder"): - for suffix in ("freeze_groups", "freeze_except_groups"): - field_name = f"{component_name}_{suffix}" - if values.get(field_name) is None: - values.pop(field_name, None) return dumper.represent_dict(values) diff --git a/src/sequifier/make.py b/src/sequifier/make.py index 351f79b0..9a01ea7b 100644 --- a/src/sequifier/make.py +++ b/src/sequifier/make.py @@ -4,8 +4,7 @@ from sequifier.typechecking import beartype -preprocess_config_string = """project_root: "." -preprocessing_data_path: PLEASE FILL +preprocess_config_string = """preprocessing_data_path: PLEASE FILL read_format: csv write_format: parquet selected_columns: @@ -24,16 +23,11 @@ - 49 - 49 - 49 +run: + project_root: . """ -train_config_string = """project_root: "." -model_name: PLEASE FILL -device: cuda -global_training: - training_objective: causal - context_length: 48 - inference_batch_size: 10 -model: +train_config_string = """model: backbone: architecture: model_dim: 128 @@ -55,6 +49,9 @@ decoder: type: linear prediction_length: 1 + execution: + context_length: 48 + training_objective: causal training_plan: phases: - name: supervised @@ -80,19 +77,57 @@ evaluation: sources: - source: default.default -export_generative_model: true -export_embedding_model: false + cadence: + epochs: 1 + batch_size: 10 + before_phase: true + on_snapshot: true + model_selection: + handoff: last + checkpoints: + cadence: + epochs: 1 + learning: + datasets: + default: + criterion: + EXAMPLE_TARGET_COLUMN_NAME: MSELoss dataset_training: default: - criterion: - EXAMPLE_TARGET_COLUMN_NAME: MSELoss parts: default: metadata_config_path: PLEASE FILL + read_format: parquet + split: + validation_fraction: 0.2 +run: + project_root: . + model_name: PLEASE FILL + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cuda + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: outer + recovery: {} + logging: {} + outputs: {} + export: + generative_model: true + embedding_model: false + onnx: true + pt: false + batch_size: 10 """ -infer_config_string = """project_root: "." -input_columns: +infer_config_string = """input_columns: - EXAMPLE_INPUT_COLUMN_NAME target_columns: - EXAMPLE_TARGET_COLUMN_NAME @@ -101,15 +136,18 @@ training_objective: causal output_probabilities: false decode_categories: true -device: cpu context_length: 48 -inference_batch_size: 10 generation_steps: 5 task: forecast model_paths: - PLEASE FILL metadata_config_path: configs/metadata_configs/PLEASE_FILL.json autoregressive_generation: true +run: + project_root: . + execution: + device: cpu + batch_size: 10 """ gitignore_string = """models/ diff --git a/src/sequifier/model/execution.py b/src/sequifier/model/execution.py index 61b5cb5f..63ce92a6 100644 --- a/src/sequifier/model/execution.py +++ b/src/sequifier/model/execution.py @@ -8,21 +8,18 @@ class ModelExecution: model: Any interfaces: dict - global_training: Any + runtime_settings: Any @classmethod def from_config(cls, config): interfaces = {} for dataset in config.dataset_training.values(): interfaces.setdefault(dataset.model_interface, dataset.interface) - return cls(config.model, interfaces, config.global_training) + return cls(config.model, interfaces, config.runtime_settings) def dataset(self, name): - """Adapt an interface for consumers of the historical training contract.""" - from sequifier.config.train_config import ( - DatasetFreezingSpecModel, - ResolvedDatasetTrainingSpec, - ) + """Construct the resolved loss and interface view needed by execution.""" + from sequifier.config.train_config import ResolvedDatasetTrainingSpec interface = self.interfaces[name] return ResolvedDatasetTrainingSpec( @@ -35,15 +32,14 @@ def dataset(self, name): for target, kind in interface.target_column_types.items() }, class_share_log_columns=[], - freeze=DatasetFreezingSpecModel(), ) def training_config(self, *, device="cpu", **options): - """Preserve the public resolved-config adapter and its serialized shape.""" + """Construct the resolved execution configuration.""" from sequifier.config.train_config import ResolvedSequifierConfig return ResolvedSequifierConfig( - global_training=self.global_training, + runtime_settings=self.runtime_settings, model=self.model, dataset_training={name: self.dataset(name) for name in self.interfaces}, **{ @@ -75,7 +71,7 @@ def retain_interfaces(self, config): selected = type(self)( self.model, {name: self.interfaces[name]}, - config.global_training, + config.runtime_settings, ) config.dataset_training[key] = ( selected.training_config() diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index 2017ca46..472759a8 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -123,7 +123,7 @@ def compile_unique_layers(layers: nn.ModuleList) -> None: @beartype def _apply_layer_dtypes(network: nn.Module, config: Any) -> None: - layer_config = config.global_training.layer_type_dtypes + layer_config = config.runtime_settings.layer_type_dtypes if not layer_config: return for name, module in network.named_modules(): @@ -269,8 +269,8 @@ def _build_composable_network( # A lean inference bundle may omit next-occurrence loss metadata. Its # forward attention policy remains the ordinary causal policy. if ( - config.global_training.training_objective == "next_occurrence" - and config.global_training.next_occurrence_config is None + config.runtime_settings.training_objective == "next_occurrence" + and config.runtime_settings.next_occurrence_config is None ): objectives[name] = CausalObjective(objective_view) else: @@ -293,7 +293,7 @@ def _build_composable_network( direct_real_dtype_provider=lambda: backbone.layers[ 0 ].ff.get_first_layer_dtype(), - max_concat_tensors=config.global_training.max_concat_tensors, + max_concat_tensors=config.runtime_settings.max_concat_tensors, load_pretrained=initialize, ) from sequifier.model.ingestions import EmbeddingFeatureIngestion @@ -432,22 +432,22 @@ def _build_composable_network( ) attention_objective = objective - if config.global_training.forward_attention is not None: + if config.runtime_settings.forward_attention is not None: attention_objective = ( CausalObjective - if config.global_training.forward_attention == "causal" + if config.runtime_settings.forward_attention == "causal" else BERTObjective ) network = ComposableTransformerNetwork( backbone=backbone, interfaces=routes, attention_mask_policy=attention_objective.build_attention_mask_policy( - config.global_training.context_length + config.runtime_settings.context_length ), attention_mask_policy_with_prefix=attention_objective.build_attention_mask_policy( - config.global_training.context_length + 1 + config.runtime_settings.context_length + 1 ), - context_length=config.global_training.context_length, + context_length=config.runtime_settings.context_length, ) if initialize: if len(routes) == 1: diff --git a/src/sequifier/model/parameter_catalog.py b/src/sequifier/model/parameter_catalog.py index 1e671a38..def23e0c 100644 --- a/src/sequifier/model/parameter_catalog.py +++ b/src/sequifier/model/parameter_catalog.py @@ -29,6 +29,7 @@ class ParameterDescriptor: shared_parameter_id: str | None depth_parameter: bool = False branch_path: tuple[str, ...] = () + semantic_aliases: tuple[str, ...] = () class ParameterCatalog: @@ -42,6 +43,42 @@ def __init__(self, model: nn.Module): for parameter in parameters } + semantic_aliases = { + identity: {group} for identity, group in semantic_by_identity.items() + } + # Component-local discovery preserves decoder and range-position semantics + # when the catalog is built for a complete multi-interface network. + from sequifier.model.decoders import TargetDecoderBranch, TargetDecoding + + for name, module in model.named_modules(): + component = name.rsplit(".", 1)[-1] + if ( + component not in {"backbone", "ingestion", "decoder"} + and not isinstance(module, (TargetDecoding, TargetDecoderBranch)) + and not any( + hasattr(module, key) + for key in ("range_position_projection", "range_projection") + ) + ): + continue + for group, component_parameters in semantic_parameter_groups( + module + ).items(): + for parameter in component_parameters: + identity = id(parameter) + previous = semantic_by_identity.get(identity) + group_aliases = semantic_aliases.setdefault(identity, set()) + if ( + previous + in {"fallback.linear", "fallback.convolution", "free_parameter"} + and previous != group + ): + group_aliases.discard(previous) + semantic_by_identity[identity] = group + elif previous is None: + semantic_by_identity[identity] = group + group_aliases.add(group) + aliases_by_identity: dict[int, list[str]] = {} parameter_by_identity: dict[int, nn.Parameter] = {} for name, parameter in model.named_parameters(remove_duplicate=False): @@ -67,6 +104,7 @@ def __init__(self, model: nn.Module): semantic_group=str( semantic_by_identity.get(identity, "free_parameter") ), + semantic_aliases=tuple(sorted(semantic_aliases.get(identity, ()))), parameter_kind=self._kind(canonical_name), shape=tuple(parameter.shape), dtype=parameter.dtype, @@ -143,7 +181,11 @@ def select( descriptor for descriptor in self._descriptors if (component is None or descriptor.component == component) - and (semantic_group is None or descriptor.semantic_group == semantic_group) + and ( + semantic_group is None + or semantic_group + in (descriptor.semantic_aliases or (descriptor.semantic_group,)) + ) ) @beartype diff --git a/src/sequifier/objectives.py b/src/sequifier/objectives.py index dbe087ea..9d130a1f 100644 --- a/src/sequifier/objectives.py +++ b/src/sequifier/objectives.py @@ -193,9 +193,7 @@ class NextOccurrenceObjective(CausalObjective): @beartype def __init__(self, config: Any) -> None: super().__init__(config) - training = getattr(config, "global_training", None) - if training is None: - training = config.training_spec + training = config.runtime_settings next_occurrence_config = training.next_occurrence_config self.next_occurrence_column = next_occurrence_config.column_name @@ -523,9 +521,7 @@ def apply_bert_masking( seeded_generator.manual_seed(eval_seed) generator = seeded_generator - training = getattr(config, "global_training", None) - if training is None: - training = config.training_spec + training = config.runtime_settings bert_spec = training.bert_spec if bert_spec is None: raise ValueError("bert_spec must be configured for BERT training") diff --git a/src/sequifier/optimizers/plan.py b/src/sequifier/optimizers/plan.py index c9c4d4a6..a335a258 100644 --- a/src/sequifier/optimizers/plan.py +++ b/src/sequifier/optimizers/plan.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from fnmatch import fnmatchcase from typing import Any, Iterable import torch @@ -24,21 +23,7 @@ class RoutedOptimizerGroup: def _matches(descriptor: ParameterDescriptor, selector: OptimizerSelector) -> bool: - return ( - ( - selector.semantic_groups is None - or any( - fnmatchcase(descriptor.semantic_group, pattern) - for pattern in selector.semantic_groups - ) - ) - and (selector.component is None or descriptor.component == selector.component) - and ( - selector.parameter_kind is None - or descriptor.parameter_kind == selector.parameter_kind - ) - and (selector.ndim is None or len(descriptor.shape) == selector.ndim) - ) + return selector.matches(descriptor) def route_optimizer_parameters( diff --git a/src/sequifier/rl/attachment.py b/src/sequifier/rl/attachment.py index b39308d4..fc91aa3a 100644 --- a/src/sequifier/rl/attachment.py +++ b/src/sequifier/rl/attachment.py @@ -8,11 +8,11 @@ def resolve_binding(execution, interface_name, binding): interface = execution.interface(interface_name) schema = ExecutionSchema.from_interface( - interface, execution.global_training.context_length + interface, execution.runtime_settings.context_length ) if ( -binding.decision_position - > execution.global_training.context_length + > execution.runtime_settings.context_length - interface.decoder.support_length + 1 ): @@ -75,7 +75,7 @@ def resolve_binding(execution, interface_name, binding): for name, stats in interface.selected_columns_statistics.items() if name in set(temporal) | set(static) and interface.normalize_real_columns }, - context_length=execution.global_training.context_length, + context_length=execution.runtime_settings.context_length, interface_name=interface_name, ) for column in binding.previous_action_inputs: diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index dbf18bbf..f94e7592 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -81,7 +81,16 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): or session.steps["actor"] != session.transaction ): raise ValueError("DPO sampler position disagrees with optimizer transaction") + if session.should_evaluate_initial(): + initial = evaluate_dpo(actor, reference, data.validation, phase) + session.record_evaluation( + initial["dpo_loss"], + initial=True, + **{f"validation_{key}": val for key, val in initial.items()}, + ) for update in range(session.transaction + 1, phase.iterations + 1): + if session.stopping.triggered: + break group = batches.next() session.zero_grad() loss_total = 0.0 diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 9d305396..9558b109 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -79,7 +79,9 @@ def evaluate(): {**result, "score_mean": score, "transaction": session.transaction} ) - while groups < phase.total_groups: + if session.should_evaluate_initial(): + evaluate() + while groups < phase.total_groups and not session.stopping.triggered: count = min(phase.groups_per_iteration, phase.total_groups - groups) buffer, executed = collect_groups( context, diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index c50dfff6..40ee6c50 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -83,7 +83,25 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): raise ValueError("IQL checkpoint exceeds phase update budget") if any(steps != session.transaction for steps in session.steps.values()): raise ValueError("IQL optimizer clocks disagree with committed updates") + if session.should_evaluate_initial(): + initial = _evaluate( + actor, + value, + q1, + q2, + data[1], + phase.evaluation.batch_size, + phase.gamma, + actor.device, + ) + session.record_evaluation( + initial["action_nll"], + initial=True, + **{f"validation_{key}": val for key, val in initial.items()}, + ) for update in range(session.transaction + 1, phase.iterations + 1): + if session.stopping.triggered: + break indices = torch.randint( len(data[0]), (phase.batch_size,), generator=session.sampler ) @@ -138,7 +156,7 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): q1, q2, data[1], - phase.batch_size, + phase.evaluation.batch_size, phase.gamma, actor.device, ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index d660055c..15eea467 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -3,6 +3,8 @@ from dataclasses import replace from pathlib import Path +import torch + from sequifier.artifacts.phase import PhaseArtifacts from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism @@ -37,6 +39,7 @@ def _plan_contract(config): values = config.model_dump(mode="json") values.pop("resume") + values.pop("checkpoint_path", None) for phase in values["phases"]: for role in phase["optimization"].values(): if role["scheduler"]["name"] == "OneCycleLR": @@ -61,6 +64,7 @@ def _initialize_actor(config): validate_execution_attachment( config.execution_model, model_execution_config(source.execution_config) ) + source.execution_config.embedding_layer_names = config.embedding_layer_names source.units = dict(config.units) return source else: @@ -99,6 +103,7 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: "policy training requires a resolved policy-only training plan" ) config = policy_run_config(config, config.training_plan) + torch.set_float32_matmul_precision("highest") configure_determinism(config.seed, config.enforce_determinism) plan = _plan_contract(config) inputs = PreparedTrainingInputs.for_run(config).resolve(config) @@ -158,7 +163,7 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: # Persist the complete fitted representation before the first update. recovery.save(checkpoint, actor.device) recovery.write_reports() - while recovery.phase_index < len(config.phases): + while recovery.phase_index < len(config.phases) and not recovery.stopped: phase_index = recovery.phase_index current_state = recovery.phase_state phase = config.phases[phase_index] diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index c751e7da..ddb5e143 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -72,7 +72,10 @@ def prepare_offline(run, phase, config, catalog, cache): validation = corpus([ref.source for ref in phase.evaluation.sources]) else: training, validation = split_by_episode( - training, run.validation_fraction, run.seed, group_identical=True + training, + phase_validation_fraction(run, phase), + run.seed, + group_identical=True, ) if not len(training) or not len(validation): raise ValueError( @@ -94,3 +97,19 @@ def prepare_iql(run, phase, config, catalog, cache): data = prepare_offline(run, phase, config, catalog, cache) data.prepare_transitions() return data + + +def phase_validation_fraction(run, phase): + """A combined leakage-safe split requires a single agreed fraction.""" + refs = [phase.preferences] if phase.algorithm == "dpo" else phase.sources + fractions = { + getattr(run, "source_validation_fractions", {}).get( + ref.source, run.validation_fraction + ) + for ref in refs + } + if len(fractions) != 1: + raise ValueError( + f"{phase.name}: automatic splitting of combined datasets requires equal dataset split fractions; declare evaluation.sources for explicit partitions" + ) + return next(iter(fractions)) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 17d81a35..196be41a 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -27,7 +27,7 @@ def policy_execution_config( model, device: str = "cpu", *, - global_training=None, + runtime_settings=None, statistics=None, execution_model=None, **run_options, diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index 5eee4de6..4123b98d 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -64,13 +64,21 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): } ) session.restore_random() - while env_steps < phase.total_env_steps: + if session.should_evaluate_initial(): + initial = evaluate_policy( + actor, phase.evaluation, forbidden_adapter=adapter + ) + session.observe(initial["selection_value"]) + evaluations.append( + {**initial, "transaction": session.transaction, "initial": True} + ) + while env_steps < phase.total_env_steps and not session.stopping.triggered: horizon = min(phase.rollout_env_steps, phase.total_env_steps - env_steps) buffer, episodes = collector.collect(horizon, session.transaction) if phase.archive_directory: buffer.archive( Path(phase.archive_directory) - / f"collection-{session.transaction+1:08d}.json", + / f"collection-{session.transaction + 1:08d}.json", include_scores=phase.archive_scores, ) record = update_ppo(session, value, buffer) diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index cd2a0e35..893003a0 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -101,6 +101,7 @@ def build_actor(config, fitted, statistics): model_name=config.model_name, seed=config.seed, ) + execution.embedding_layer_names = config.embedding_layer_names actor = Policy.from_network( build_transformer_network( execution, device=torch.device(config.device) @@ -159,14 +160,56 @@ def preflight_environments(config): def preflight_optimization(config, actor, data): """Resolve constructors and authored scheduler budgets before creating outputs.""" - budgets = [] - for phase, part in zip(config.phases, data): - for path in phase.freeze_modules: - actor.network.get_submodule(path) - size = len(part.training) if part is not None else None - budget = phase.optimizer_step_budget(size) - for optimization in phase.optimization_configs().values(): - optimization.validate_budget(budget) - optimization.build(actor.network, step_budget=budget) - budgets.append(budget) + from copy import deepcopy + + from sequifier.rl.models import HistoryCritic, make_iql_models + + baseline = { + name: parameter.requires_grad + for name, parameter in actor.network.named_parameters() + } + budgets, previous_masks = [], None + try: + for phase, part in zip(config.phases, data): + for name, parameter in actor.network.named_parameters(): + parameter.requires_grad_(baseline[name]) + modules = {"actor": actor.network} + if phase.algorithm == "iql": + value, q1, q2, _ = make_iql_models( + actor.execution_config, actor.interface_name, actor.config.actions + ) + modules.update(value=value, critic=torch.nn.ModuleList([q1, q2])) + elif phase.algorithm == "ppo": + modules["value"] = HistoryCritic( + deepcopy(actor.execution_config), actor.interface_name, None + ) + masks = {} + for role, module in modules.items(): + frozen = phase.trainability.policy(role).resolve(module) + for parameter in module.parameters(): + if id(parameter) in frozen: + parameter.requires_grad_(False) + masks[role] = sorted( + name + for name, parameter in module.named_parameters() + if parameter.requires_grad + ) + if not masks[role]: + raise ValueError( + f"{phase.name}: {role} leaves no trainable parameters" + ) + if phase.optimizer_state == "continue" and masks != previous_masks: + raise ValueError( + "optimizer continuation requires identical resolved parameter masks" + ) + previous_masks = masks + size = len(part.training) if part is not None else None + budget = phase.optimizer_step_budget(size) + for role, optimization in phase.optimization_configs().items(): + optimization.validate_budget(budget) + optimization.build(modules[role], step_budget=budget) + budgets.append(budget) + finally: + for name, parameter in actor.network.named_parameters(): + parameter.requires_grad_(baseline[name]) return budgets diff --git a/src/sequifier/rl/prepared_inputs.py b/src/sequifier/rl/prepared_inputs.py index a621e84e..626524fe 100644 --- a/src/sequifier/rl/prepared_inputs.py +++ b/src/sequifier/rl/prepared_inputs.py @@ -190,6 +190,9 @@ def prepare_phase(self, config, phase, policy, inputs, *, fitting=False): "preferences": preferences, "policy": _data_policy_contract(policy), "validation_fraction": config.validation_fraction, + "source_validation_fractions": getattr( + config, "source_validation_fractions", {} + ), "seed": config.seed, "episode_cache_size": config.episode_cache_size, } diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index c503cad7..cee4e5ed 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -56,6 +56,7 @@ class RunCheckpoint: completed: list[CompletedPhase] = field(default_factory=list) phase_state: dict | None = None continuation: dict | None = None + stopped: bool = False @property def results(self): @@ -78,33 +79,15 @@ def load(cls, path, config, plan, data_identity): if ( not isinstance(payload, dict) or payload.get("artifact_type") != "sequifier_policy_run" - or payload.get("format_version") not in {5, 6} + or payload.get("format_version") != 7 ): raise ValueError( - "policy resume requires a version-5 or version-6 policy run checkpoint" + "policy resume requires a version-7 policy run checkpoint; initialize from its model artifact to start a new run" ) try: saved_plan = payload["plan"] require_equal(plan, saved_plan, "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") - if payload["format_version"] == 5: - records = payload.pop("completed_artifacts") - results = payload.pop("completed_results") - require_equal(len(results), len(records), "resume.completed_phases") - for result, record in zip(results, records): - require_equal( - result["provenance"]["actor_path"], - record["path"], - "resume.actor_path", - ) - payload["completed"] = [ - { - "result": result, - "sha256": record["sha256"], - "result_fingerprint": record["result_fingerprint"], - } - for result, record in zip(results, records) - ] completed = [ CompletedPhase.from_dict(record) for record in payload["completed"] ] @@ -185,7 +168,7 @@ def validate(self, config): best = selection["best_transaction"] if best is not None and ( (not isinstance(best, int) or isinstance(best, bool)) - or not 1 <= best <= transaction + or not 0 <= best <= transaction ): raise ValueError( "checkpoint selected transaction is outside committed progress" @@ -230,7 +213,7 @@ def save(self, path, device, state=None): path, { "artifact_type": "sequifier_policy_run", - "format_version": 6, + "format_version": 7, **{ f.name: getattr(self, f.name) for f in fields(self) @@ -257,6 +240,7 @@ def finish_phase(self, actor, result): result, **PhaseArtifacts(Path(path)).record(result.as_dict()) ) ) + self.stopped = result.stop_scope == "end_plan" self.incoming_actor = weights self.phase_state = None diff --git a/src/sequifier/rl/run_config.py b/src/sequifier/rl/run_config.py index 315aea5a..18e2ca5b 100644 --- a/src/sequifier/rl/run_config.py +++ b/src/sequifier/rl/run_config.py @@ -2,11 +2,12 @@ from typing import Any -from pydantic import BaseModel, ConfigDict, PrivateAttr +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr from sequifier.config.components import ResumeConfig +from sequifier.config.normalization_config import FrozenNormalizationConfig from sequifier.config.policy_config import ResolvedPolicyBinding -from sequifier.config.policy_plan import FrozenNormalizationConfig, PolicyPhase +from sequifier.config.policy_plan import PolicyPhase from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.config.train_config import ModelSpecModel @@ -30,9 +31,14 @@ class ResolvedPolicyRun(BaseModel): preferences: dict[str, str] units: dict[str, str] phases: list[PolicyPhase] + latest_checkpoint_interval_minutes: float | None = None + source_validation_fractions: dict[str, float] = Field(default_factory=dict) validation_fraction: float episode_cache_size: int resume: ResumeConfig + embedding_layer_names: list[str] = Field( + default_factory=lambda: ["backbone.final_norm"] + ) model_path: str report_path: str checkpoint_path: str diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index c69c4e60..fed4c53d 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -3,7 +3,8 @@ import hashlib import json import math -from dataclasses import dataclass +import time +from dataclasses import asdict, dataclass from typing import TYPE_CHECKING, Callable, Generic, TypeVar import torch @@ -12,13 +13,14 @@ from sequifier.config.policy_plan import PhaseConfig from sequifier.rl.policy import Policy from sequifier.runtime.random_state import capture_rng_state, restore_rng_state +from sequifier.runtime.stopping import StoppingState if TYPE_CHECKING: from sequifier.rl.run_config import ResolvedPolicyRun PhaseType = TypeVar("PhaseType", bound=PhaseConfig) DataType = TypeVar("DataType") -PHASE_STATE_VERSION = 2 +PHASE_STATE_VERSION = 3 def clone_weights(module): @@ -111,6 +113,7 @@ class PolicyPhaseResult: clocks: dict metrics: tuple[dict, ...] provenance: dict + stop_scope: str | None = None def as_dict(self): from dataclasses import asdict @@ -223,9 +226,21 @@ def __init__(self, context, *, modules=None, frozen=None): } for name, parameter in actor.network.named_parameters(): parameter.requires_grad_(actor._phase_trainability[name]) - for path in context.phase.freeze_modules: - actor.network.get_submodule(path).requires_grad_(False) self.modules = {"actor": context.actor.network, **(modules or {})} + self.mask_ids = {} + for role, module in self.modules.items(): + eligible = {id(p) for p in module.parameters() if p.requires_grad} + mask = context.phase.trainability.policy(role).resolve(module) + if not eligible - mask: + raise ValueError( + f"{context.phase.name}: {role} leaves no trainable parameters" + ) + for parameter in module.parameters(): + if id(parameter) in mask: + parameter.requires_grad_(False) + self.mask_ids[role] = sorted( + name for name, p in module.named_parameters() if p.requires_grad + ) self.networks = {**self.modules, **(frozen or {})} self.configs = { role: config.resolve_budget(context.step_budget) @@ -259,6 +274,11 @@ def __init__(self, context, *, modules=None, frozen=None): sorted(continuation["networks"]), "optimization.roles", ) + require_equal( + self.mask_ids, + continuation["trainable_parameters"], + "optimization.trainability", + ) for name, module in self.networks.items(): module.load_state_dict(continuation["networks"][name], strict=True) for role, (optimizer, scheduler) in self.optimization.items(): @@ -272,6 +292,11 @@ def __init__(self, context, *, modules=None, frozen=None): self.sampler = torch.Generator().manual_seed(context.run.seed + context.index) self.state = context.resume_state or {} self.transaction = self.state.get("transaction", 0) + self.stopping = StoppingState(**self.state.get("stopping", {})) + self.initial_evaluated = self.state.get("initial_evaluated", False) + self._count_stopping = True + self.last_checkpoint_time = time.monotonic() + self.last_snapshot_time = self.last_checkpoint_time self.metrics = list(self.state.get("metrics", [])) selection = context.phase.evaluation.monitor monitor = selection.metric @@ -279,6 +304,9 @@ def __init__(self, context, *, modules=None, frozen=None): context.phase, monitor, selection.direction, self.state.get("selection") ) if self.state: + require_equal( + self.mask_ids, self.state["trainable_parameters"], "phase.trainability" + ) require_equal( sorted(self.networks), sorted(self.state["networks"]), "phase.networks" ) @@ -340,14 +368,39 @@ def step(self): self.zero_grad() return norms - def should_evaluate(self, final): + def snapshot_due(self): + minutes = self.context.phase.checkpoints.snapshot_interval_minutes return ( + minutes is not None + and time.monotonic() - self.last_snapshot_time >= minutes * 60 + ) + + def should_evaluate_initial(self): + due = self.context.phase.evaluation.before_phase and not self.initial_evaluated + if due: + self.initial_evaluated = True + self._count_stopping = False + return due + + def should_evaluate(self, final): + scheduled = ( final or self.transaction % self.context.phase.evaluation_interval_iterations == 0 ) + self._count_stopping = scheduled + return scheduled or ( + self.snapshot_due() and self.context.phase.evaluation.on_snapshot + ) def observe(self, metric): self.selection.observe(self.context.actor, metric, self.transaction) + if self._count_stopping: + self.stopping.observe( + self.context.phase.stopping, + metric, + self.selection.direction, + self.transaction, + ) def record_evaluation(self, metric, **record): self.observe(metric) @@ -366,6 +419,13 @@ def optimization_state(self, *, include_steps=False): def should_checkpoint(self, final): return ( final + or self.stopping.triggered + or self.snapshot_due() + or ( + self.context.run.latest_checkpoint_interval_minutes is not None + and time.monotonic() - self.last_checkpoint_time + >= self.context.run.latest_checkpoint_interval_minutes * 60 + ) or self.transaction % self.context.phase.checkpoint_interval_iterations == 0 ) @@ -383,6 +443,9 @@ def checkpoint(self, final, **extra): "optimization": self.optimization_state(include_steps=True), "sampler": self.sampler.get_state(), "selection": self.selection.state_dict(), + "stopping": asdict(self.stopping), + "initial_evaluated": self.initial_evaluated, + "trainable_parameters": self.mask_ids, "transaction": self.transaction, "metrics": self.metrics, "reference": clone_weights(self.context.reference.network) @@ -393,6 +456,30 @@ def checkpoint(self, final, **extra): } ) + retained = ( + final + or self.stopping.triggered + or self.snapshot_due() + or self.transaction % self.context.phase.checkpoint_interval_iterations == 0 + ) + if retained: + from pathlib import Path + from shutil import copyfile + + from sequifier.io.atomic import atomic_path + + destination = ( + Path(self.context.run.phase_directory) + / f"{self.context.index:03d}-{self.context.phase.name}" + / "checkpoints" + / f"transaction-{self.transaction:08d}.pt" + ) + with atomic_path(destination, durable=True) as temporary: + copyfile(self.context.run.checkpoint_path, temporary) + self.last_checkpoint_time = time.monotonic() + if self.snapshot_due(): + self.last_snapshot_time = self.last_checkpoint_time + def finish(self, clocks, provenance): next_index = self.context.index + 1 preserve = ( @@ -407,9 +494,30 @@ def finish(self, clocks, provenance): for name, module in self.networks.items() }, "optimization": self.optimization_state(), + "trainable_parameters": self.mask_ids, } elif hasattr(self.context.actor, "_phase_continuation"): del self.context.actor._phase_continuation - return self.selection.finish( - self.context, self.transaction, clocks, self.metrics, provenance + actor, result = self.selection.finish( + self.context, + self.transaction, + clocks, + self.metrics, + { + **provenance, + "trainable_parameters": self.mask_ids, + "resolved_seeds": { + "run": self.context.run.seed, + "phase_sampling": self.context.run.seed + self.context.index, + }, + }, + ) + + from dataclasses import replace + + return actor, replace( + result, + stop_scope=self.context.phase.stopping.on_trigger + if self.stopping.triggered + else None, ) diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 062e4989..c7c511f9 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -52,7 +52,12 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin raise ValueError("imitation checkpoint epoch/transaction clocks disagree") if batches.epoch == phase.epochs and batches.order is not None: raise ValueError("completed imitation checkpoint has a live sampler order") - while batches.epoch < phase.epochs: + if session.should_evaluate_initial(): + session.record_evaluation( + evaluate_imitation(actor, data.validation, phase.evaluation.batch_size), + initial=True, + ) + while batches.epoch < phase.epochs and not session.stopping.triggered: group = batches.next() session.zero_grad() total = 0.0 @@ -69,7 +74,9 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin session.transaction += 1 final = batches.epoch == phase.epochs if session.should_evaluate(final): - score = evaluate_imitation(actor, data.validation, phase.batch_size) + score = evaluate_imitation( + actor, data.validation, phase.evaluation.batch_size + ) session.record_evaluation( score, epochs_completed=batches.epoch, diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index f13006ab..4987c824 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -1,5 +1,12 @@ """Local categorical policy learning, inference, and environment contracts.""" +from sequifier.config.normalization_config import FrozenNormalizationConfig +from sequifier.config.parameter_selection import ( + ParameterSelector, + RoleTrainability, + TrainabilityConfig, +) +from sequifier.config.phase_controls import Cadence, StoppingConfig from sequifier.config.policy_config import ( ActionFieldConfig, PolicyBinding, @@ -13,7 +20,6 @@ from sequifier.config.policy_optimization import OptimizationConfig from sequifier.config.policy_plan import ( DPOPhaseConfig, - FrozenNormalizationConfig, GRPOEvaluationConfig, GRPOPhaseConfig, IQLPhaseConfig, @@ -65,6 +71,11 @@ from sequifier.trajectory.returns import remaining_return, return_to_go __all__ = [ + "ParameterSelector", + "RoleTrainability", + "TrainabilityConfig", + "Cadence", + "StoppingConfig", "ActorRepresentation", "CategoricalActionDistribution", "Policy", diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 1fc1adec..d775595f 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -37,9 +37,11 @@ from sequifier.training.metrics_service import MetricsService from sequifier.training.optimization import GradientPolicy, OptimizationRuntime from sequifier.training.runtime import ( - DatasetFreezingPlan, DatasetRuntimeRegistry, + PhaseTrainabilityPlan, build_dataset_runtimes, + configure_phase, + validate_phase_learning, ) from sequifier.training.scheduler_budgets import validate_supervised_scheduler_budgets from sequifier.training.state import RunState @@ -69,6 +71,14 @@ class TrainingRun: def start_phase(self, phase_index: int) -> None: """Enter the phase with its resolved reset or continuation policy.""" + if self.config.active_phase_index != phase_index: + configure_phase( + self.config, + self.network, + self.datasets, + phase_index, + self.distributed.device, + ) optimizer_step = self.optimization.optimizer_step if self.config.training_plan[phase_index].optimizer_state == "reset": self.optimization = self.optimization_factory(phase_index) @@ -91,17 +101,17 @@ def _strategy( ) -> DistributedStrategy: if not execution.distributed: return LocalStrategy(device=execution.device) - if config.global_training.data_parallelism == "fsdp": + if config.runtime_settings.data_parallelism == "fsdp": dtype = None - if config.global_training.layer_autocast: - layer_types = config.global_training.layer_type_dtypes or {} + if config.runtime_settings.layer_autocast: + layer_types = config.runtime_settings.layer_type_dtypes or {} dtype = get_torch_dtype(layer_types.get("linear", "bfloat16")) return FullyShardedStrategy( rank=execution.rank, local_rank=execution.local_rank, world_size=execution.world_size, device=execution.device, - cpu_offload=bool(config.global_training.fsdp_cpu_offload), + cpu_offload=bool(config.runtime_settings.fsdp_cpu_offload), mixed_precision_dtype=dtype, ) return DistributedDataParallelStrategy( @@ -131,7 +141,7 @@ def _warm_up(self, run: TrainingRun) -> None: for dataset in run.datasets.values(): schema = ExecutionSchema.from_interface( dataset.config.interface, - run.config.global_training.context_length, + run.config.runtime_settings.context_length, ) features, metadata = schema.bind( schema.example_inputs(batch_size, run.distributed.device) @@ -174,6 +184,8 @@ def build( else: state = RunState() + validate_phase_learning(config, execution.device) + from sequifier.export.preflight import ensure_export_preflight ensure_export_preflight(config, strategy) @@ -203,13 +215,24 @@ def build( load_revision(network.backbone, revision) state.backbone_parent_revision_id = revision["revision_id"] - freezing_plan = DatasetFreezingPlan.resolve(config, network) - freezing_plan.apply_permanent(network) + freezing_plan = PhaseTrainabilityPlan.resolve( + config, network, state.phase_index + ) + previous_masks = None + for phase_index, phase in enumerate(config.training_plan): + masks = PhaseTrainabilityPlan.resolve( + config, network, phase_index + ).parameter_ids_by_dataset + if phase.optimizer_state == "continue" and masks != previous_masks: + raise ValueError( + "optimizer continuation requires identical resolved parameter masks" + ) + previous_masks = masks compile_before_ddp = ( execution.distributed - and config.global_training.data_parallelism == "ddp" - and config.global_training.torch_compile == "outer" + and config.runtime_settings.data_parallelism == "ddp" + and config.runtime_settings.torch_compile == "outer" ) network_to_prepare = torch.compile(network) if compile_before_ddp else network prepared = strategy.prepare_network(network_to_prepare) @@ -229,6 +252,7 @@ def build( runtime_metadata=built.runtime_metadata, freezing_plan=freezing_plan, ) + configure_phase(config, network, datasets, state.phase_index, execution.device) for dataset in datasets.values(): for part in dataset.parts.values(): part.factory.batch_size = config.training_plan[ @@ -254,7 +278,7 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: selected_parameters, parameter_catalog=catalog, accumulation_batches=phase.accumulation_batches, - layer_type_dtypes=config.global_training.layer_type_dtypes, + layer_type_dtypes=config.runtime_settings.layer_type_dtypes, ) validate_supervised_scheduler_budgets(config, datasets, state.phase_index) @@ -267,13 +291,15 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: restorer.restore_optimization(loaded, optimization, network, strategy) callable_network = prepared.callable_network - if config.global_training.torch_compile == "inner": + if config.runtime_settings.torch_compile == "inner": from sequifier.model.encoder_stack import TransformerEncoderStack for stack in list(network.modules()): if isinstance(stack, TransformerEncoderStack): compile_unique_layers(stack.layers) - elif config.global_training.torch_compile == "outer" and not compile_before_ddp: + elif ( + config.runtime_settings.torch_compile == "outer" and not compile_before_ddp + ): callable_network = torch.compile(callable_network) context = RunContext( diff --git a/src/sequifier/runtime/stopping.py b/src/sequifier/runtime/stopping.py new file mode 100644 index 00000000..8324d36e --- /dev/null +++ b/src/sequifier/runtime/stopping.py @@ -0,0 +1,31 @@ +"""Serializable observation-based patience shared by native learning runners.""" + +import math +from dataclasses import dataclass + + +@dataclass +class StoppingState: + best: float | None = None + without_improvement: int = 0 + last_observation: int | None = None + triggered: bool = False + + def observe(self, config, value, direction, observation): + if config is None or value is None or not math.isfinite(value): + return self.triggered + if self.last_observation is not None and observation <= self.last_observation: + return self.triggered + self.last_observation = observation + improved = self.best is None or ( + value < self.best - config.min_delta + if direction == "min" + else value > self.best + config.min_delta + ) + if improved: + self.best = float(value) + self.without_improvement = 0 + else: + self.without_improvement += 1 + self.triggered = self.without_improvement >= config.patience + return self.triggered diff --git a/src/sequifier/sequifier.py b/src/sequifier/sequifier.py index 6d59f48f..4cb2816d 100644 --- a/src/sequifier/sequifier.py +++ b/src/sequifier/sequifier.py @@ -1,5 +1,5 @@ from argparse import ArgumentParser, ArgumentTypeError -from typing import Any, NoReturn +from typing import Any import yaml @@ -14,36 +14,12 @@ class ConfigArgumentParser(ArgumentParser): - """Diagnose removed flags without accepting them as aliases.""" + """Parse only explicitly declared command options.""" def __init__(self, *args: Any, **kwargs: Any): kwargs.setdefault("allow_abbrev", False) super().__init__(*args, **kwargs) - def error(self, message: str) -> NoReturn: - replacements = { - "--model-type": "--task forecast|policy|embedding", - "--model-path": "--model-paths", - "--randomize": "--seed random", - "--data-path": "--preprocessing-data-path for preprocessing", - "-dp": "--preprocessing-data-path (preprocess) or --data-path (infer)", - "-mp": "--model-paths", - "-ic": "--input-columns", - "-mc": "--metadata-config-path", - "-sc": "--selected-columns", - "-mn": "--model-name", - "-sm": "--skip-metadata", - "-s": "--seed", - "-r": "--seed random", - } - prefix = "unrecognized arguments: " - if message.startswith(prefix): - for token in message[len(prefix) :].split(): - flag = token.partition("=")[0] - if flag in replacements: - message += f"\n{flag} was removed; use {replacements[flag]}" - super().error(message) - def parse_seed(value: str) -> int | str: if value == "random": @@ -156,7 +132,8 @@ def setup_parser() -> ArgumentParser: help="Space-separated list of selected columns", ) parser_train.add_argument("--model-name", type=str) - parser_train.add_argument("--global-training", type=yaml.safe_load) + for subparser in (parser_train, parser_infer, parser_preprocess): + subparser.add_argument("--run", type=yaml.safe_load) parser_train.add_argument("--training-plan", type=yaml.safe_load) parser_infer.add_argument("--model-paths", type=str, nargs="+") diff --git a/src/sequifier/templates/infer-policy.yaml b/src/sequifier/templates/infer-policy.yaml index 6c6663ab..2593e25b 100644 --- a/src/sequifier/templates/infer-policy.yaml +++ b/src/sequifier/templates/infer-policy.yaml @@ -1,11 +1,14 @@ -project_root: "." data_path: PLEASE_FILL read_format: csv write_format: csv -device: cpu desired_return: 10.0 -enforce_determinism: true action_selection: deterministic task: policy model_paths: - models/policy/model.pt +run: + project_root: . + execution: + device: cpu + reproducibility: + enforce_determinism: true diff --git a/src/sequifier/templates/train-dpo.yaml b/src/sequifier/templates/train-dpo.yaml index 7e942620..8d438d0a 100644 --- a/src/sequifier/templates/train-dpo.yaml +++ b/src/sequifier/templates/train-dpo.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: dpo-control -device: cpu -seed: 1010 model: backbone: architecture: @@ -33,10 +29,11 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 4 - normalization: - source: identity + input_contract: + normalization: + source: identity + execution: + context_length: 4 dataset_training: decisions: parts: @@ -54,11 +51,15 @@ dataset_training: cue: next_cue group: contextId model_interface: actor + split: + validation_fraction: 0.2 rankings: parts: default: type: preferences manifest_path: data/control-preferences.json + split: + validation_fraction: 0.2 training_plan: phases: - name: preferences @@ -67,6 +68,17 @@ training_plan: source: rankings.default beta: 0.1 iterations: 100 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 1 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -80,3 +92,29 @@ policy_bindings: values: - left - right +run: + project_root: . + model_name: dpo-control + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: {} + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/src/sequifier/templates/train-grpo.yaml b/src/sequifier/templates/train-grpo.yaml index 9b459f2f..9a43f6a1 100644 --- a/src/sequifier/templates/train-grpo.yaml +++ b/src/sequifier/templates/train-grpo.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: grpo-control -device: cpu -seed: 1010 model: backbone: architecture: @@ -33,10 +29,11 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 4 - normalization: - source: identity + input_contract: + normalization: + source: identity + execution: + context_length: 4 training_plan: phases: - name: group_improvement @@ -66,6 +63,16 @@ training_plan: monitor: metric: score_mean direction: max + cadence: + iterations: 1 + batch_size: 1 + before_phase: false + on_snapshot: false + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 policy_bindings: actor: previous_action_inputs: @@ -79,3 +86,29 @@ policy_bindings: values: - left - right +run: + project_root: . + model_name: grpo-control + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: {} + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/src/sequifier/templates/train-iql.yaml b/src/sequifier/templates/train-iql.yaml index 3eeadd7a..322412cb 100644 --- a/src/sequifier/templates/train-iql.yaml +++ b/src/sequifier/templates/train-iql.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: offline-iql -device: cpu -seed: 1010 model: backbone: architecture: @@ -32,11 +28,8 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 16 - validation_fraction: 0.2 - resume: - policy: if_exists + execution: + context_length: 16 dataset_training: demonstrations: parts: @@ -52,6 +45,8 @@ dataset_training: successors: OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN model_interface: actor + split: + validation_fraction: 0.2 training_plan: phases: - name: offline_improvement @@ -60,6 +55,17 @@ training_plan: - source: demonstrations.recorded batch_size: 64 iterations: 10000 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 100 + evaluation: + cadence: + iterations: 100 + batch_size: 64 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -73,3 +79,31 @@ policy_bindings: values: - 0.0 - 1.0 +run: + project_root: . + model_name: offline-iql + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: + resume: + policy: if_exists + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/src/sequifier/templates/train-policy-from-supervised.yaml b/src/sequifier/templates/train-policy-from-supervised.yaml index f72f805e..185b22d1 100644 --- a/src/sequifier/templates/train-policy-from-supervised.yaml +++ b/src/sequifier/templates/train-policy-from-supervised.yaml @@ -1,11 +1,4 @@ -project_root: "." -model_name: posttrained -device: cpu initialize_from: models/supervised/model.pt -global_training: - resume: - policy: if_exists - mode: reset_environments dataset_training: demonstrations: parts: @@ -14,6 +7,8 @@ dataset_training: paths: - data/demonstrations.parquet format: table + split: + validation_fraction: 0.2 training_plan: phases: - name: imitation @@ -21,6 +16,17 @@ training_plan: sources: - source: demonstrations.default epochs: 5 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 32 + before_phase: false + on_snapshot: false policy_bindings: default: actions: @@ -33,3 +39,32 @@ policy_bindings: type: real scale: fixed: 0.5 +run: + project_root: . + model_name: posttrained + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: + resume: + policy: if_exists + mode: reset_environments + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/src/sequifier/templates/train-policy.yaml b/src/sequifier/templates/train-policy.yaml index ceccac2b..dedcb47f 100644 --- a/src/sequifier/templates/train-policy.yaml +++ b/src/sequifier/templates/train-policy.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: policy -device: cpu -seed: 1010 model: backbone: architecture: @@ -36,12 +32,8 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 4 - validation_fraction: 0.2 - episode_cache_size: 8 - resume: - policy: never + execution: + context_length: 4 dataset_training: demonstrations: parts: @@ -50,6 +42,8 @@ dataset_training: paths: - PLEASE_FILL.csv model_interface: actor + split: + validation_fraction: 0.2 training_plan: phases: - algorithm: policy_imitation @@ -61,6 +55,17 @@ training_plan: optimization: actor: learning_rate: 0.005 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 64 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -74,3 +79,32 @@ policy_bindings: values: - 0.0 - 1.0 +run: + project_root: . + model_name: policy + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: + episode_cache_size: 8 + compilation: + torch_compile: none + recovery: + resume: + policy: never + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/src/sequifier/templates/train-ppo.yaml b/src/sequifier/templates/train-ppo.yaml index 950df086..66e89bcb 100644 --- a/src/sequifier/templates/train-ppo.yaml +++ b/src/sequifier/templates/train-ppo.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: online-ppo -device: cpu -seed: 1010 model: backbone: architecture: @@ -33,13 +29,11 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 4 - normalization: - source: identity - resume: - policy: if_exists - mode: exact + input_contract: + normalization: + source: identity + execution: + context_length: 4 training_plan: phases: - name: online_control @@ -69,9 +63,17 @@ training_plan: - 71 max_episode_steps: 8 timeout_seconds: 30 + cadence: + iterations: 2 + batch_size: 1 + before_phase: false + on_snapshot: false batch_size: 64 - evaluation_interval_iterations: 2 - checkpoint_interval_iterations: 2 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 2 policy_bindings: actor: previous_action_inputs: @@ -85,3 +87,32 @@ policy_bindings: values: - left - right +run: + project_root: . + model_name: online-ppo + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: none + recovery: + resume: + policy: if_exists + mode: exact + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/src/sequifier/templates/train-supervised-to-policy.yaml b/src/sequifier/templates/train-supervised-to-policy.yaml index c8318128..ea2c9262 100644 --- a/src/sequifier/templates/train-supervised-to-policy.yaml +++ b/src/sequifier/templates/train-supervised-to-policy.yaml @@ -1,7 +1,3 @@ -project_root: "." -model_name: supervised-to-policy -device: cpu -seed: 1010 model: backbone: architecture: @@ -36,14 +32,9 @@ model: support_length: 1 categorical_decoder_special_tokens: action: [] -global_training: - context_length: 4 - validation_fraction: 0.2 - episode_cache_size: 8 - resume: - policy: never - training_objective: causal - torch_compile: none + execution: + context_length: 4 + training_objective: causal dataset_training: demonstrations: parts: @@ -52,13 +43,16 @@ dataset_training: paths: - PLEASE_FILL.csv model_interface: actor + split: + validation_fraction: 0.2 observations: model_interface: actor parts: train: metadata_config_path: configs/metadata_configs/observations.json - criterion: - action: CrossEntropyLoss + read_format: parquet + split: + validation_fraction: 0.2 training_plan: phases: - name: supervised @@ -82,6 +76,16 @@ training_plan: gamma: 0.99 scheduler_step_on: epoch gradient_clip_norm: 1.0 + model_selection: + handoff: last + checkpoints: + cadence: + epochs: 1 + learning: + datasets: + observations: + criterion: + action: CrossEntropyLoss - algorithm: policy_imitation epochs: 80 sources: @@ -91,6 +95,17 @@ training_plan: optimization: actor: learning_rate: 0.005 + model_selection: + handoff: last + checkpoints: + cadence: + iterations: 1 + evaluation: + cadence: + iterations: 1 + batch_size: 64 + before_phase: false + on_snapshot: false policy_bindings: actor: previous_action_inputs: @@ -104,3 +119,32 @@ policy_bindings: values: - 0.0 - 1.0 +run: + project_root: . + model_name: supervised-to-policy + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: + episode_cache_size: 8 + compilation: + torch_compile: none + recovery: + resume: + policy: never + logging: {} + outputs: {} + export: + onnx: false + pt: true + batch_size: 1 + generative_model: true + embedding_model: false + with_dropout: false diff --git a/src/sequifier/templates/train-supervised.yaml b/src/sequifier/templates/train-supervised.yaml index af8591d7..4778a308 100644 --- a/src/sequifier/templates/train-supervised.yaml +++ b/src/sequifier/templates/train-supervised.yaml @@ -1,10 +1,3 @@ -project_root: "." -model_name: supervised -device: cpu -global_training: - training_objective: causal - context_length: 48 - inference_batch_size: 10 model: backbone: architecture: @@ -27,6 +20,9 @@ model: decoder: type: linear prediction_length: 1 + execution: + context_length: 48 + training_objective: causal training_plan: phases: - name: supervised @@ -52,12 +48,51 @@ training_plan: evaluation: sources: - source: default.default -export_generative_model: true -export_embedding_model: false + cadence: + epochs: 1 + batch_size: 10 + before_phase: true + on_snapshot: true + model_selection: + handoff: last + checkpoints: + cadence: + epochs: 1 + learning: + datasets: + default: + criterion: + EXAMPLE_TARGET_COLUMN_NAME: MSELoss dataset_training: default: - criterion: - EXAMPLE_TARGET_COLUMN_NAME: MSELoss parts: default: metadata_config_path: PLEASE FILL + read_format: parquet + split: + validation_fraction: 0.2 +run: + project_root: . + model_name: supervised + reproducibility: + seed: 1010 + enforce_determinism: false + execution: + device: cpu + distribution: + strategy: local + world_size: 1 + backend: nccl + precision: {} + data_loading: {} + compilation: + torch_compile: outer + recovery: {} + logging: {} + outputs: {} + export: + generative_model: true + embedding_model: false + onnx: true + pt: false + batch_size: 10 diff --git a/src/sequifier/train.py b/src/sequifier/train.py index f7750f5b..3771d588 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -92,37 +92,37 @@ def train_worker( config.model_name, global_rank, dataset_names=tuple(config.dataset_training), - rank_specific=config.global_training.distributed, + rank_specific=config.runtime_settings.distributed, ) initialized_distributed = False - if config.global_training.distributed: + if config.runtime_settings.distributed: if config.device.startswith("cuda"): torch.cuda.set_device(local_rank) setup_distributed_env( global_rank, local_rank, world_size, - config.global_training.backend, + config.runtime_settings.backend, ) initialized_distributed = True - configure_determinism(config.seed, config.global_training.enforce_determinism) + configure_determinism(config.seed, config.runtime_settings.enforce_determinism) execution = ExecutionEnvironment( rank=global_rank, local_rank=local_rank, world_size=world_size, device=_execution_device(config, local_rank), - distributed=config.global_training.distributed, + distributed=config.runtime_settings.distributed, ) integrations = IntegrationManager( specs=integration_specs, instances=integration_instances, rank=global_rank, world_size=world_size, - distributed=config.global_training.distributed, + distributed=config.runtime_settings.distributed, ) integrations.validate_execution( - torch_compile=config.global_training.torch_compile, - data_parallelism=config.global_training.data_parallelism, + torch_compile=config.runtime_settings.torch_compile, + data_parallelism=config.runtime_settings.data_parallelism, ) run = None try: @@ -189,7 +189,7 @@ def run_training( return run_learning_plan(config) if not isinstance(config, TrainModel): raise TypeError("Training requires a canonical resolved config.") - spec = config.global_training + spec = config.runtime_settings if spec.distributed and integration_instances: raise ValueError( "Distributed runs require IntegrationSpec; direct instances cannot " @@ -300,7 +300,7 @@ def _tensor_batches( column_data_types: dict[str, torch.dtype], ): categorical = set(model.categorical_columns) - layer_types = model.config.global_training.layer_type_dtypes or {} + layer_types = model.config.runtime_settings.layer_type_dtypes or {} reference_dtype = get_torch_dtype(layer_types.get("linear", "float32")) for index, features in enumerate(x): values = { diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index 02d75504..016c7bc7 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -62,13 +62,7 @@ def _resume_settings( execution = copy.deepcopy(dict(model_config or model_execution_config(config))) execution.pop("embedding_layer_names", None) - execution.setdefault( - "forward_attention", - "bidirectional" if execution["training_objective"] == "bert" else "causal", - ) - # Initialization is strict run provenance, compared from resolved training - # configs rather than older portable artifacts that omitted it. def strip_provenance(value: Any) -> Any: if isinstance(value, dict): return { @@ -91,23 +85,20 @@ def strip_provenance(value: Any) -> Any: # Paths locate data for inference; an exact resume may use a new # project directory containing the same splits. interface.pop("split_paths", None) - interface.setdefault("depth_layouts", {}) - interface.setdefault("prediction_distributions", {}) - interface.setdefault("tensor_payload_version", 1) - training = config.global_training + training = config.runtime_settings datasets = { name: { "model_interface": dataset.model_interface, "parts": list(dataset.parts), - "criterion": dataset.criterion, - "class_weights": dataset.class_weights, - "loss_weights": dataset.loss_weights, - "freeze": dataset.freeze.model_dump(mode="python"), + "part_formats": { + name: part.read_format for name, part in dataset.parts.items() + }, } for name, dataset in config.dataset_training.items() } phases = [phase.model_dump(mode="python") for phase in config.training_plan] for phase in phases: + phase.pop("trainability_summary", None) for role in phase["optimization"].values(): if role["scheduler"]["name"] == "OneCycleLR": # Recomputed from the phase/data budget before optimizer construction. @@ -123,9 +114,14 @@ def strip_provenance(value: Any) -> Any: for name, interface in config.model.interfaces.items() }, }, + "owned_execution": config.ownership.get("run", {}).get("execution", {}), + "normalization_by_interface": { + name: value.model_dump(mode="python") + for name, value in config.normalization_by_interface.items() + }, + "source_validation_fractions": config.source_validation_fractions, "resume_training": { "seed": config.seed, - "read_format": training.read_format, "distributed": training.distributed, "data_parallelism": training.data_parallelism, "world_size": training.world_size, diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 748f1a4d..8c8d58c6 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -55,12 +55,12 @@ def _supervised_config(config, phase, index, incoming, resume): extend_prediction_distributions(source, config.model) model = source.model - forward_attention = source.global_training.forward_attention or ( + forward_attention = source.runtime_settings.forward_attention or ( "bidirectional" - if source.global_training.training_objective == "bert" + if source.runtime_settings.training_objective == "bert" else "causal" ) - training = config.global_training.model_copy(deep=True) + training = config.runtime_settings.model_copy(deep=True) if training.distributed: raise ValueError("mixed learning plans currently require local phase execution") if forward_attention is not None: @@ -68,32 +68,44 @@ def _supervised_config(config, phase, index, incoming, resume): from sequifier.config.components import ResumeConfig training.resume = ResumeConfig(policy="if_exists" if resume else "never") - authored = SequifierConfig.model_validate( - { - "project_root": config.project_root, - "model_name": f"{config.model_name}-phase-{index:03d}-{phase.name}", - "device": config.device, - "seed": config.seed + index, - "model": model.model_dump(), - "global_training": training.model_dump(), - "dataset_training": { - name: value.model_dump() - for name, value in authored.dataset_training.items() - if name - in { - ref.source.partition(".")[0] - for ref in [*phase.sources, *phase.evaluation_sources] - } - }, - "training_plan": {"phases": [phase_spec.model_dump()]}, - "export_pt": True, - "export_onnx": config.export_onnx, - "export_generative_model": config.export_generative_model, - "export_embedding_model": config.export_embedding_model, - "embedding_layer_names": config.embedding_layer_names, - "export_with_dropout": config.export_with_dropout, - } - ) + values = authored.model_dump(mode="python") + run = values["run"] + run["project_root"] = config.project_root + run["model_name"] = f"{config.model_name}-phase-{index:03d}-{phase.name}" + run["execution"]["device"] = config.device + run["reproducibility"]["seed"] = config.seed + index + run["recovery"]["resume"] = training.resume.model_dump() + run["export"]["pt"] = True + model_values = model.model_dump() + model_values["execution"] = { + key: getattr(training, key) + for key in ( + "context_length", + "training_objective", + "target_offset", + "forward_attention", + "next_occurrence_config", + ) + } + if training.next_occurrence_config is not None: + model_values["execution"]["next_occurrence_config"] = ( + training.next_occurrence_config.model_dump() + ) + for name, interface in authored.model.interfaces.items(): + if name in model_values["interfaces"]: + model_values["interfaces"][name]["input_contract"] = ( + interface.input_contract.model_dump() + ) + values["model"] = model_values + names = {ref.dataset for ref in [*phase.sources, *phase.evaluation_sources]} + values["dataset_training"] = { + name: value + for name, value in values["dataset_training"].items() + if name in names + } + values["training_plan"] = {"phases": [phase_spec.model_dump()]} + values["initialize_from"] = None + authored = SequifierConfig.model_validate(values) metadata = { f"{name}.{part_name}": load_dataset_metadata( str(Path(config.project_root) / part.metadata_config_path) @@ -102,6 +114,9 @@ def _supervised_config(config, phase, index, incoming, resume): for part_name, part in dataset.parts.items() } resolved = resolve_sequifier_config(authored, metadata) + resolved.ownership = config.ownership + resolved.source_validation_fractions = config.source_validation_fractions + resolved.normalization_by_interface = config.normalization_by_interface if source is not None: # Preserve policy bindings and their model-owned scale modules on return # to supervised learning. Dataset resolution owns the data/window views. @@ -132,7 +147,7 @@ def _run_supervised(config, phase, index, incoming, destination, resume): resolved = _supervised_config(config, phase, index, incoming, resume) configure_logger(config.project_root, resolved.model_name, 0) configure_determinism( - config.seed + index, resolved.global_training.enforce_determinism + config.seed + index, resolved.runtime_settings.enforce_determinism ) environment = ExecutionEnvironment( rank=0, @@ -152,6 +167,10 @@ def _run_supervised(config, phase, index, incoming, destination, resume): save_model_artifact(build_model_artifact(run.network, resolved), destination) return { "completion_reason": result.completion_reason, + "resolved_seed": config.seed + index, + "stop_scope": phase.stopping.on_trigger + if run.state.stopping_state.get("triggered") + else None, "optimizer_steps": run.state.optimizer_step, "handoff": phase.handoff, } @@ -181,7 +200,7 @@ def _run_policy(config, phase, index, incoming, destination, resume): def run_learning_plan(config) -> LearningRunResult: phases = config.training_plan - resume = config.global_training.resume + resume = config.runtime_settings.resume from sequifier.config.components import ResumeConfig resume = resume or ResumeConfig() @@ -189,11 +208,12 @@ def run_learning_plan(config) -> LearningRunResult: resume.checkpoint_path or f"checkpoints/{config.model_name}/learning-run.pt" ) plan = config.model_dump(mode="json") - plan["global_training"].pop("resume", None) + plan["runtime_settings"].pop("resume", None) + plan.get("ownership", {}).get("run", {}).get("recovery", {}).pop("resume", None) identity = fingerprint(plan) state: dict[str, Any] = { "artifact_type": "sequifier_learning_run", - "format_version": 1, + "format_version": 2, "plan": identity, "completed": [], "incoming": config.initialize_from, @@ -206,7 +226,7 @@ def run_learning_plan(config) -> LearningRunResult: state = torch.load(checkpoint, map_location="cpu", weights_only=False) if ( state.get("artifact_type") != "sequifier_learning_run" - or state.get("format_version") != 1 + or state.get("format_version") != 2 or state.get("plan") != identity ): raise ValueError("learning run checkpoint differs from the requested plan") @@ -274,6 +294,8 @@ def run_learning_plan(config) -> LearningRunResult: if any(contains_path(directory, source) for source in protected): raise ValueError("phase_directory overlaps a learning input") for index in range(len(state["completed"]), len(phases)): + if state.get("stopped"): + break phase = phases[index] incoming = state["incoming"] verify_artifact( @@ -302,6 +324,7 @@ def run_learning_plan(config) -> LearningRunResult: runner = _run_supervised if phase.algorithm == "supervised" else _run_policy report = runner(config, phase, index, incoming, destination, resuming) artifacts.write_receipt(phase_identity, report) + state["stopped"] = report.get("stop_scope") == "end_plan" state["completed"].append( { "name": phase.name, diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 3e1968be..cdb60297 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -6,7 +6,7 @@ import time from contextlib import nullcontext from copy import deepcopy -from dataclasses import dataclass +from dataclasses import asdict, dataclass from pathlib import Path from typing import Any @@ -29,6 +29,7 @@ ) from sequifier.logging_paths import model_log_directory from sequifier.runtime.random_state import RandomState +from sequifier.runtime.stopping import StoppingState from sequifier.training.checkpoint_service import CheckpointRequest from sequifier.training.metrics_service import TrainingMetrics from sequifier.training.optimization import OptimizationBoundaryState, UpdatePolicy @@ -164,7 +165,12 @@ def _check_pruning(self, run: Any) -> None: raise SystemExit(143) def _update_monitor( - self, run: Any, result: EvaluationResult, phase: Any + self, + run: Any, + result: EvaluationResult, + phase: Any, + *, + count_patience: bool = True, ) -> tuple[float, bool]: monitor = phase.evaluation_monitor if monitor is None or monitor.source not in result.sources: @@ -174,6 +180,10 @@ def _update_monitor( "inf" ): run.state.best_validation_loss = float("-inf") + import math + + if not math.isfinite(value): + return value, False improved = ( value < run.state.best_validation_loss if monitor.direction == "min" @@ -181,11 +191,12 @@ def _update_monitor( ) if improved: run.state.best_validation_loss = value - run.state.epochs_without_improvement = 0 + if count_patience: + run.state.epochs_without_improvement = 0 state = run.distributed.capture_model_state(run.network) if run.distributed.rank == 0: run.state.best_model_state_dict = state - else: + elif count_patience: run.state.epochs_without_improvement += 1 return value, improved @@ -234,8 +245,11 @@ def run(self, run: Any) -> RunResult: config = run.config state = run.state time_checkpoint_intervals_enabled = bool( - config.global_training.latest_checkpoint_interval_minutes is not None - or config.global_training.snapshot_interval_minutes is not None + config.runtime_settings.latest_checkpoint_interval_minutes is not None + or any( + p.checkpoints.snapshot_interval_minutes is not None + for p in config.training_plan + ) ) defer_ddp_gradient_sync = bool( isinstance( @@ -254,7 +268,7 @@ def run(self, run: Any) -> RunResult: metric_windows: dict[str, dict[str, Any]] = {} last_latest = time.monotonic() last_snapshot = time.monotonic() - last_snapshot_step = state.global_batch_step + last_snapshot_step = state.last_snapshot_batch_step access = run.optimization.access(run.network) run.integrations.emit(ModelReady(access=access)) run.optimization.optimizer.zero_grad(set_to_none=True) @@ -331,30 +345,10 @@ def flush_metrics( current_batch = sum(state.iterator_positions.values()) current_batches_total = 1 try: - if ( - state.global_batch_step == 0 - and config.training_plan[0].evaluation_sources - and config.global_training.evaluate_before_training - ): - first_sources = [ - build_source_runtime(source, run.datasets) - for source in config.training_plan[0].sources - ] - self._evaluate( - run, - self._evaluation_context( - run, - kind="initial", - phase_index=0, - phase_epoch=0, - epoch=0, - batch=0, - batches_total=sum( - source.num_batches() for source in first_sources - ), - ), - ) for phase_index, phase in enumerate(config.training_plan): + if state.plan_stopped: + completion_reason = "early_stopping" + break if phase_index < state.phase_index: continue if phase_index != state.phase_index: @@ -371,6 +365,43 @@ def flush_metrics( state.best_validation_loss = float("inf") state.best_model_state_dict = None state.epochs_without_improvement = 0 + state.stopping_state = {} + last_snapshot = time.monotonic() + last_snapshot_step = state.global_batch_step + state.last_snapshot_batch_step = last_snapshot_step + phase_stopping = StoppingState(**state.stopping_state) + phase_stop = phase_stopping.triggered + if phase_stop: + continue + if run.optimization_phase_index != phase_index: + from sequifier.training.runtime import configure_phase + + configure_phase( + config, + run.network, + run.datasets, + phase_index, + run.distributed.device, + ) + if ( + phase.evaluation + and phase.evaluation.before_phase + and phase_index not in state.initial_evaluated_phases + ): + initial = self._evaluate( + run, + self._evaluation_context( + run, + kind="initial", + phase_index=phase_index, + phase_epoch=0, + epoch=state.epoch, + batch=0, + batches_total=1, + ), + ) + self._update_monitor(run, initial, phase, count_patience=False) + state.initial_evaluated_phases.append(phase_index) for dataset in run.datasets.values(): for part in dataset.parts.values(): if part.factory.batch_size != phase.batch_size: @@ -453,7 +484,7 @@ def flush_metrics( if run.integrations.enabled: run.integrations.emit(prepared_event) snapshot_interval_batches = ( - config.global_training.snapshot_interval_batches + phase.checkpoints.snapshot_interval_batches ) batch_snapshot_due = bool( snapshot_interval_batches is not None @@ -552,7 +583,7 @@ def flush_metrics( window["part"] = None if ( window["batches"] - >= config.global_training.log_interval_batches + >= config.runtime_settings.log_interval_batches ): flush_metrics( dataset.name, @@ -567,19 +598,19 @@ def flush_metrics( flush() now = time.monotonic() snapshot_due = bool( - config.global_training.snapshot_interval_batches is not None + phase.checkpoints.snapshot_interval_batches is not None and state.global_batch_step - last_snapshot_step - >= config.global_training.snapshot_interval_batches + >= phase.checkpoints.snapshot_interval_batches ) or bool( - config.global_training.snapshot_interval_minutes is not None + phase.checkpoints.snapshot_interval_minutes is not None and now - last_snapshot - >= config.global_training.snapshot_interval_minutes * 60 + >= phase.checkpoints.snapshot_interval_minutes * 60 ) latest_due = bool( - config.global_training.latest_checkpoint_interval_minutes + config.runtime_settings.latest_checkpoint_interval_minutes is not None and now - last_latest - >= config.global_training.latest_checkpoint_interval_minutes + >= config.runtime_settings.latest_checkpoint_interval_minutes * 60 ) if ( @@ -608,10 +639,11 @@ def flush_metrics( last_latest = time.monotonic() if snapshot_due: if ( - config.global_training.evaluate_on_snapshot - and phase.evaluation_sources + phase.evaluation_sources + and phase.evaluation is not None + and phase.evaluation.on_snapshot ): - self._evaluate( + snapshot_evaluation = self._evaluate( run, self._evaluation_context( run, @@ -623,7 +655,14 @@ def flush_metrics( batches_total=current_batches_total, ), ) + self._update_monitor( + run, + snapshot_evaluation, + phase, + count_patience=False, + ) run.callable_network.train() + state.last_snapshot_batch_step = state.global_batch_step run.checkpoints.save( CheckpointRequest( f"epoch-{current_epoch}-batch-{current_batch}", @@ -676,26 +715,32 @@ def flush_metrics( run.optimization.step_scheduler(metric) state.iterator_positions = {} state.source_scheduler_state = scheduler.state_dict() - self._update_monitor(run, evaluation, phase) + monitored, _ = self._update_monitor(run, evaluation, phase) + phase_stop = phase_stopping.observe( + phase.stopping, + monitored, + phase.evaluation_monitor.direction + if phase.evaluation_monitor + else "min", + phase_epoch, + ) + state.stopping_state = asdict(phase_stopping) + if phase_stop: + state.plan_stopped = phase.stopping.on_trigger == "end_plan" + stop_requested = state.plan_stopped + completion_reason = "early_stopping" boundary = capture_runtime_boundary(run) boundary_identity = last_identity if ( - phase_epoch % phase.checkpoint_interval_epochs == 0 + phase_stop + or phase_epoch % phase.checkpoint_interval_epochs == 0 or phase_epoch == phase.epochs ): run.checkpoints.save( CheckpointRequest(f"epoch-{current_epoch}", last_identity), run, ) - patience = config.global_training.early_stopping_patience_epochs - if ( - patience is not None - and phase.evaluation_monitor is not None - and state.epochs_without_improvement >= patience - ): - stop_requested = True - completion_reason = "early_stopping" - if stop_requested: + if stop_requested or phase_stop: break if stop_requested: break diff --git a/src/sequifier/training/metrics.py b/src/sequifier/training/metrics.py index cf54636e..eada6a65 100644 --- a/src/sequifier/training/metrics.py +++ b/src/sequifier/training/metrics.py @@ -16,12 +16,9 @@ from sequifier.logging_paths import dataset_artifact_prefix from sequifier.typechecking import beartype -METRICS_SCHEMA_VERSION = 1 -FULL_METRICS_SCHEMA_VERSION = 2 TOTAL_TARGET = "__total__" COMMON_FIELDS = [ - "schema_version", "run_id", "session_id", "timestamp_utc", @@ -219,7 +216,6 @@ def _common( part: str | None = None, ) -> dict[str, Any]: return { - "schema_version": METRICS_SCHEMA_VERSION, "run_id": run_id, "session_id": session_id, "timestamp_utc": timestamp, @@ -268,7 +264,6 @@ def write_training( for field, value in common.items() if field in FULL_COMMON_FIELDS } - full_common["schema_version"] = FULL_METRICS_SCHEMA_VERSION measurements = [(TOTAL_TARGET, total_loss), *target_losses.items()] rows = [ { @@ -346,7 +341,6 @@ def write_validation( for field, value in common.items() if field in FULL_COMMON_FIELDS }, - "schema_version": FULL_METRICS_SCHEMA_VERSION, "evaluation_id": evaluation_id, "evaluation_kind": evaluation_kind, "learning_rate": float(learning_rate), @@ -447,8 +441,7 @@ def write_validation( ) if class_rows and not self.class_share_columns: raise ValueError( - "Class-share rows were provided without configured class-share " - "columns." + "Class-share rows were provided without configured class-share columns." ) if class_rows and self._class_shares is None: self._class_shares = _CsvAppender(self.class_share_path, CLASS_SHARE_FIELDS) diff --git a/src/sequifier/training/metrics_service.py b/src/sequifier/training/metrics_service.py index f920f00c..14271122 100644 --- a/src/sequifier/training/metrics_service.py +++ b/src/sequifier/training/metrics_service.py @@ -35,7 +35,17 @@ def __init__(self, context: Any, config: Any) -> None: config.project_root, config.model_name, context.rank, - class_share_columns=dataset.class_share_log_columns, + class_share_columns=list( + dict.fromkeys( + column + for phase in config.training_plan + for column in ( + phase.learning.datasets[name].class_share_log_columns + if name in phase.learning.datasets + else [] + ) + ) + ), dataset_name=name, dataset_count=dataset_count, validation_enabled=name in evaluated, diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 6f3a264c..1ebff218 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -3,7 +3,6 @@ from __future__ import annotations import random -import warnings from collections.abc import Iterator, Sequence from dataclasses import dataclass, field, replace from typing import Any, Literal, Optional, Protocol, runtime_checkable @@ -14,7 +13,6 @@ from torch.utils.data import DataLoader from sequifier.config.train_config import ( - DatasetFreezingSpecModel, ResolvedDatasetTrainingSpec, ResolvedSequifierConfig, ResolvedTrainingPhase, @@ -36,7 +34,6 @@ ) from sequifier.model.network import ComposableTransformerNetwork from sequifier.model.parameter_catalog import ParameterCatalog -from sequifier.model.parameter_groups import semantic_parameter_groups from sequifier.objectives import Objective from sequifier.training.accurate_cross_entropy import AccurateCrossEntropyLoss from sequifier.typechecking import beartype @@ -57,6 +54,7 @@ def _training_plan_uses_part( source.dataset == dataset_name and (source.part is None or source.part == part_name) for phase in config.training_plan + if isinstance(phase, ResolvedTrainingPhase) for source in phase.sources ) @@ -78,13 +76,17 @@ class PartLoaderFactory: @beartype def build(self, split: Literal["training", "validation"]) -> DataLoader: - global_spec = self.config.global_training + global_spec = self.config.runtime_settings if split in self.loaders: return self.loaders[split] + phase = self.config.training_plan[self.config.active_phase_index] + evaluation = phase.evaluation dataset_config: Any = replace( dataset_part_view(self.config, self.dataset_name, self.part_name), - batch_size=self.batch_size, + batch_size=self.batch_size + if split == "training" + else (evaluation.batch_size if evaluation is not None else self.batch_size), ) if split != "training" or not _training_plan_uses_part( self.config, self.dataset_name, self.part_name @@ -122,14 +124,14 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: ), ) dataset = SequifierDatasetFromFile(path, dataset_config, shuffle=shuffle) - elif global_spec.read_format == "pt": + elif dataset_config.read_format == "pt": cls = ( SequifierDatasetFromFolderPt if global_spec.load_full_data_to_ram else SequifierDatasetFromFolderPtLazy ) dataset = cls(path, dataset_config, shuffle=shuffle) - elif global_spec.read_format == "parquet": + elif dataset_config.read_format == "parquet": cls = ( SequifierDatasetFromFolderParquet if global_spec.load_full_data_to_ram @@ -138,7 +140,7 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: dataset = cls(path, dataset_config, shuffle=shuffle) else: raise ValueError( - f"Folder parts do not support read_format={global_spec.read_format!r}" + f"Folder parts do not support read_format={dataset_config.read_format!r}" ) generator = self.generators.get(split) if generator is None: @@ -262,10 +264,6 @@ class DatasetRuntime: runtime_metadata: Any metrics: DatasetMetrics = field(default_factory=DatasetMetrics) - @property - def freeze_policy(self) -> DatasetFreezingSpecModel: - return self.config.freeze - class DatasetRuntimeRegistry: def __init__(self, datasets: dict[str, DatasetRuntime]) -> None: @@ -293,8 +291,8 @@ def items(self): @dataclass(frozen=True) -class DatasetFreezingPlan: - """Dataset freezing selections expressed as stable parameter catalog IDs.""" +class PhaseTrainabilityPlan: + """Phase-owned update masks expressed as stable parameter catalog IDs.""" parameter_ids_by_dataset: dict[str, frozenset[str]] @@ -304,24 +302,54 @@ def resolve( cls, config: ResolvedSequifierConfig, network: ComposableTransformerNetwork, - ) -> DatasetFreezingPlan: + phase_index: int = 0, + ) -> PhaseTrainabilityPlan: catalog = ParameterCatalog(network) parameter_id_by_identity = { id(catalog.parameter(descriptor.parameter_id)): descriptor.parameter_id for descriptor in catalog.descriptors() } parameter_ids_by_dataset = {} - for name, dataset in config.dataset_training.items(): - frozen = frozen_parameter_ids( - network, dataset.model_interface, dataset.freeze + phase = config.training_plan[phase_index] + if not isinstance(phase, ResolvedTrainingPhase): + raise ValueError( + "native supervised trainability requires a resolved supervised phase" ) + active_datasets = {ref.dataset for ref in phase.sources} + phase_parameters = {id(p) for p in network.backbone.parameters()} + for name in active_datasets: + phase_parameters.update( + id(p) + for p in network.interfaces[ + config.dataset_training[name].model_interface + ].parameters() + ) + default_mask = phase.trainability.policy("model").resolve( + network, eligible=phase_parameters + ) + for name, dataset in config.dataset_training.items(): route = network.interfaces[dataset.model_interface] active_parameter_ids = { id(parameter) for module in (network.backbone, route) for parameter in module.parameters() } - if frozen >= active_parameter_ids: + override = phase.trainability.by_dataset.get(name) + if name not in active_datasets: + frozen = frozenset() + elif override is not None and "model" in override.roles: + frozen = override.roles["model"].resolve( + network, eligible=active_parameter_ids + ) + else: + frozen = default_mask & active_parameter_ids + trainable = { + id(p) + for module in (network.backbone, route) + for p in module.parameters() + if p.requires_grad + } + if name in active_datasets and not trainable - frozen: raise ValueError(f"Dataset {name!r} leaves no trainable parameters") missing = frozen.difference(parameter_id_by_identity) if missing: @@ -331,24 +359,13 @@ def resolve( parameter_ids_by_dataset[name] = frozenset( parameter_id_by_identity[identity] for identity in frozen ) + phase.trainability_summary = { + name: sorted(ids) + for name, ids in parameter_ids_by_dataset.items() + if name in active_datasets + } return cls(parameter_ids_by_dataset) - @property - def permanently_frozen_parameter_ids(self) -> frozenset[str]: - selections = tuple( - set(parameter_ids) - for parameter_ids in self.parameter_ids_by_dataset.values() - ) - if not selections: - return frozenset() - return frozenset(set.intersection(*selections)) - - @beartype - def apply_permanent(self, network: ComposableTransformerNetwork) -> None: - catalog = ParameterCatalog(network) - for parameter_id in self.permanently_frozen_parameter_ids: - catalog.parameter(parameter_id).requires_grad_(False) - @beartype def bind(self, network: ComposableTransformerNetwork) -> dict[str, frozenset[int]]: catalog = ParameterCatalog(network) @@ -422,117 +439,6 @@ def iter_batches( ) -> Iterator[Any]: ... -@beartype -def _policy_parameter_ids(module: nn.Module, policy: Any, usage: str) -> set[int]: - if policy.freeze_groups is None and policy.freeze_except_groups is None: - return set() - groups = semantic_parameter_groups(module) - configured = set( - policy.freeze_groups - if policy.freeze_groups is not None - else policy.freeze_except_groups or [] - ) - matched = {group for group in configured if groups.get(group)} - unmatched = configured - matched - if unmatched and policy.freeze_except_groups is not None: - raise ValueError( - f"{usage} freeze_except_groups groups matched no parameters: " - f"{', '.join(sorted(unmatched))}" - ) - if unmatched: - warnings.warn( - f"{usage} freezing groups matched no parameters: {sorted(unmatched)}", - stacklevel=2, - ) - selected = { - id(parameter) for group in configured for parameter in groups.get(group, ()) - } - if policy.freeze_groups is not None: - return selected - return {id(parameter) for parameter in module.parameters()} - selected - - -def _ingestion_policy_ids( - module, - policy, - usage, - static_encoder=None, - image_encoder=None, - prefix_projection=None, -): - decisions = {} - root = nn.Module() - root.add_module("ingestion", module) - if static_encoder is not None: - root.add_module("static_condition_encoder", static_encoder) - if image_encoder is not None: - root.add_module("preconditioning_image_encoder", image_encoder) - if prefix_projection is not None: - root.add_module("prefix_projection", prefix_projection) - root_frozen = _policy_parameter_ids(root, policy, usage) - - def visit(node, selection, inherited, path, *, is_root=False): - own = {id(p) for p in node.parameters()} - if is_root: - local = {identity: identity in root_frozen for identity in own} - elif selection is not None and ( - selection.freeze_groups is not None - or selection.freeze_except_groups is not None - ): - frozen = _policy_parameter_ids(node, selection, path) - local = {identity: identity in frozen for identity in own} - else: - local = {identity: inherited.get(identity, False) for identity in own} - branches = getattr(node, "branches", {}) - child_policies = getattr(selection, "branches", {}) - unknown = set(child_policies) - set(branches) - if unknown: - raise ValueError(f"{path}: unknown freezing branches {sorted(unknown)}") - child_ids = {id(p) for child in branches.values() for p in child.parameters()} - for identity in own - child_ids: - decision = local[identity] - if identity in decisions and decisions[identity] != decision: - raise ValueError( - f"{path}: conflicting freezing decisions for a shared parameter" - ) - decisions[identity] = decision - for name, child in branches.items(): - visit(child, child_policies.get(name), local, f"{path}.branches.{name}") - - visit(module, policy, {}, usage, is_root=True) - frozen = {identity for identity, decision in decisions.items() if decision} - for extra in (static_encoder, image_encoder, prefix_projection): - if extra is not None: - frozen.update(root_frozen.intersection(id(p) for p in extra.parameters())) - return frozen - - -@beartype -def frozen_parameter_ids( - network: ComposableTransformerNetwork, - interface_name: str, - freeze: DatasetFreezingSpecModel, -) -> frozenset[int]: - route = network.interfaces[interface_name] - frozen = _policy_parameter_ids(network.backbone, freeze.backbone, "backbone") - frozen.update( - _ingestion_policy_ids( - route.ingestion, - freeze.ingestion, - "ingestion", - route.static_condition_encoder, - route.preconditioning_image_encoder, - route.prefix_projection, - ) - ) - if freeze.ingestion_adapter: - frozen.update( - id(parameter) for parameter in route.ingestion_adapter.parameters() - ) - frozen.update(_policy_parameter_ids(route.decoder, freeze.decoder, "decoder")) - return frozenset(frozen) - - @beartype def _criterion_modules( dataset: ResolvedDatasetTrainingSpec, device: torch.device @@ -582,11 +488,10 @@ def build_dataset_runtimes( *, objectives: dict[str, Objective], runtime_metadata: Any, - freezing_plan: DatasetFreezingPlan | None = None, + freezing_plan: PhaseTrainabilityPlan | None = None, ) -> DatasetRuntimeRegistry: if freezing_plan is None: - freezing_plan = DatasetFreezingPlan.resolve(config, network) - freezing_plan.apply_permanent(network) + freezing_plan = PhaseTrainabilityPlan.resolve(config, network) frozen_parameter_ids_by_dataset = freezing_plan.bind(network) runtimes = {} @@ -811,3 +716,56 @@ def iter_epoch(self, epoch: int) -> Iterator[RuntimeBatch]: yield batch if self.pending_batches == 0: self.pending_source = None + + +def configure_phase(config, network, datasets, phase_index, device): + """Apply phase learning intent without replacing loader recovery identities.""" + from sequifier.config.train_config import interface_build_view + from sequifier.objectives import create_objective + + phase = config.training_plan[phase_index] + config.active_phase_index = phase_index + config.runtime_settings.bert_spec = phase.learning.bert_spec + config.runtime_settings.next_occurrence_config = ( + phase.learning.next_occurrence_config + or config.runtime_settings.next_occurrence_config + ) + config.runtime_settings.window_stride = phase.learning.window_stride + config.curriculum_training = phase.curriculum is not None + config.curriculum_column = phase.curriculum.column if phase.curriculum else None + masks = PhaseTrainabilityPlan.resolve(config, network, phase_index).bind(network) + for name, dataset in datasets.items(): + learning = phase.learning.datasets.get(name) + if learning is not None: + for field, value in learning.model_dump().items(): + setattr(dataset.config, field, value) + dataset.criteria = _criterion_modules(dataset.config, device) + dataset.class_weights = learning.class_weights + dataset.loss_weights = learning.loss_weights + dataset.frozen_parameter_ids = masks[name] + dataset.objective = create_objective( + interface_build_view(config, dataset.config.interface) + ) + for part in dataset.parts.values(): + part.factory.loaders.clear() + part.factory.batch_size = phase.batch_size + + +def validate_phase_learning(config, device): + """Validate every phase's losses before native outputs are created.""" + for phase in config.training_plan: + for name, learning in phase.learning.datasets.items(): + dataset = config.dataset_training[name].model_copy(deep=True) + for key, value in learning.model_dump().items(): + setattr(dataset, key, value) + targets = set(dataset.interface.target_columns) + if set(dataset.criterion) != targets: + raise ValueError( + f"{phase.name}: {name} criterion keys must equal targets" + ) + for column in dataset.class_share_log_columns: + if dataset.interface.target_column_types.get(column) != "categorical": + raise ValueError( + f"{phase.name}: class share columns must be categorical targets" + ) + _criterion_modules(dataset, device) diff --git a/src/sequifier/training/scheduler_budgets.py b/src/sequifier/training/scheduler_budgets.py index 23260de2..303535c0 100644 --- a/src/sequifier/training/scheduler_budgets.py +++ b/src/sequifier/training/scheduler_budgets.py @@ -44,11 +44,14 @@ def iter_batches(self, split="training"): def _phase_update_budget(config, datasets, phase_index): phase = config.training_plan[phase_index] + config.active_phase_index = phase_index + config.runtime_settings.window_stride = phase.learning.window_stride + config.curriculum_training = phase.curriculum is not None + config.curriculum_column = phase.curriculum.column if phase.curriculum else None for dataset in datasets.values(): for part in dataset.parts.values(): - if part.factory.batch_size != phase.batch_size: - part.factory.loaders.clear() - part.factory.batch_size = phase.batch_size + part.factory.loaders.clear() + part.factory.batch_size = phase.batch_size sources = [build_source_runtime(source, datasets) for source in phase.sources] counted = [ _CountedSource(source.config, source.dataset, source.num_batches()) @@ -110,8 +113,14 @@ def validate_supervised_scheduler_budgets(config, datasets, active_phase_index): role.scheduler = role.scheduler.model_copy( update={"arguments": {**role.scheduler.arguments, "total_steps": budget}} ) + phase = config.training_plan[active_phase_index] + config.active_phase_index = active_phase_index + config.runtime_settings.window_stride = phase.learning.window_stride + config.curriculum_training = phase.curriculum is not None + config.curriculum_column = phase.curriculum.column if phase.curriculum else None for dataset in datasets.values(): for part in dataset.parts.values(): + part.factory.loaders.clear() if ( part.factory.batch_size != config.training_plan[active_phase_index].batch_size diff --git a/src/sequifier/training/state.py b/src/sequifier/training/state.py index ea0c9893..27cb477c 100644 --- a/src/sequifier/training/state.py +++ b/src/sequifier/training/state.py @@ -19,6 +19,9 @@ class RunStateSnapshot: class RunState: run_id: str = field(default_factory=lambda: uuid.uuid4().hex) session_id: str = field(default_factory=lambda: uuid.uuid4().hex) + stopping_state: dict[str, Any] = field(default_factory=dict) + plan_stopped: bool = False + initial_evaluated_phases: list[int] = field(default_factory=list) phase_index: int = 0 phase_epoch: int = 0 phase_epoch_complete: bool = False @@ -27,6 +30,7 @@ class RunState: iterator_positions: dict[str, int] = field(default_factory=dict) epoch: int = 0 batch: int = 0 + last_snapshot_batch_step: int = 0 global_batch_step: int = 0 optimizer_step: int = 0 accumulation_index: int = 0 diff --git a/src/sequifier/training_api.py b/src/sequifier/training_api.py index 61414a18..691fc697 100644 --- a/src/sequifier/training_api.py +++ b/src/sequifier/training_api.py @@ -1,5 +1,12 @@ """Stable update-aware primitives for training integrations.""" +from sequifier.config.ownership import RunSettings +from sequifier.config.parameter_selection import ( + ParameterSelector, + RoleTrainability, + TrainabilityConfig, +) +from sequifier.config.phase_controls import Cadence, StoppingConfig from sequifier.config.train_config import ResolvedSequifierConfig, SequifierConfig from sequifier.integration.contexts import StepIdentity from sequifier.integration.controls import TrainingDirective @@ -14,6 +21,12 @@ from sequifier.training.state import RunState __all__ = [ + "RunSettings", + "ParameterSelector", + "RoleTrainability", + "TrainabilityConfig", + "Cadence", + "StoppingConfig", "DistributedDataParallelStrategy", "DistributedStrategy", "FullyShardedStrategy", diff --git a/tools/convert_folder.sh b/tools/convert_folder.sh deleted file mode 100644 index 32a9e759..00000000 --- a/tools/convert_folder.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Check if a directory is provided -if [ -z "$1" ]; then - echo "Usage: $0 " - exit 1 -fi - -# Find all yaml files recursively and loop through them -find "$1" -type f -name "*.yaml" | while read -r filepath; do - # Get just the filename for checking patterns - filename=$(basename "$filepath") - - # logic: check strict exclusion first - if [[ "$filename" == *"train"* && "$filename" == *"infer"* ]]; then - echo "Skipping ambiguous file (contains both train/infer): $filepath" - continue - fi - - if [[ "$filename" == *"infer"* ]]; then - echo "Converting INFER config: $filepath" - python tools/convert_v0_config_to_v1.py "infer" "$filepath" - elif [[ "$filename" == *"train"* ]]; then - echo "Converting TRAIN config: $filepath" - python tools/convert_v0_config_to_v1.py "train" "$filepath" - fi -done diff --git a/tools/convert_v0_config_to_v1.py b/tools/convert_v0_config_to_v1.py deleted file mode 100644 index b64885d6..00000000 --- a/tools/convert_v0_config_to_v1.py +++ /dev/null @@ -1,224 +0,0 @@ -import argparse - -import yaml - - -# Helper to rename keys while preserving insertion order -def rename_key(d, old, new, transform=None): - if old not in d: - return - # Create a new dict to preserve order - new_d = {} - for k, v in d.items(): - if k == old: - new_d[new] = transform(v) if transform is not None else v - else: - new_d[k] = v - d.clear() - d.update(new_d) - - -def convert_preprocess(config): - rename_key(config, "project_path", "project_root") - rename_key(config, "data_path", "preprocessing_data_path") - rename_key(config, "column_types", "column_data_types") - - assert "group_proportions" in config, "group_proportions missing" - rename_key(config, "group_proportions", "split_ratios") - - if "seq_step_sizes" in config: - steps = config.pop("seq_step_sizes") - if not isinstance(steps, list) or not steps or len(set(steps)) != 1: - raise ValueError( - "seq_step_sizes must contain one repeated stride; choose a scalar window_stride manually" - ) - config["window_stride"] = steps[0] - - # Rename and validate merge settings - if "combine_into_single_file" in config: - config.pop("combine_into_single_file") - - # Enforce new v1.0 logic for merge_output based on format - write_format = config.get("write_format", "parquet") - if write_format == "pt": - config["merge_output"] = False - elif write_format == "parquet": - # Keep the user's choice, defaulting to True if not present to mimic old behavior - config["merge_output"] = config.get("merge_output", True) - else: - config["merge_output"] = True - - if "seed" not in config: - config["seed"] = 1010 - - return config - - -def convert_model(ms, is_hp_search=False): - # Mapping old keys to new keys - if "d_model" in ms: - val = ms["d_model"] - rename_key(ms, "d_model", "dim_model") - ms["initial_embedding_dim"] = val - - if is_hp_search: - ms["joint_embedding_dim"] = [None] * len(val) - - if "d_model_by_column" in ms: - rename_key(ms, "d_model_by_column", "feature_embedding_dims") - - if "nhead" in ms: - rename_key(ms, "nhead", "n_heads") - - if "d_hid" in ms: - rename_key(ms, "d_hid", "dim_feedforward") - - if "nlayers" in ms: - rename_key(ms, "nlayers", "num_layers") - - if "inference_size" in ms: - rename_key(ms, "inference_size", "prediction_length") - elif is_hp_search: - ms["prediction_length"] = 1 - - # Add v1 defaults - if is_hp_search: - ms["activation"] = ["swiglu"] - ms["normalization"] = ["rmsnorm"] - ms["positional_encoding"] = ["learned"] - ms["attention_type"] = ["mha"] - ms["norm_first"] = [True] - ms["n_kv_heads"] = [None] - ms["rope_theta"] = [10000.0] - else: - ms["activation"] = "swiglu" - ms["normalization"] = "rmsnorm" - ms["positional_encoding"] = "learned" - ms["attention_type"] = "mha" - ms["norm_first"] = True - ms["rope_theta"] = 10000.0 - - return ms - - -def convert_training_spec(ts, is_hp_search=False): - if "lr" in ts: - rename_key(ts, "lr", "learning_rate") - - if "iter_save" in ts: - rename_key(ts, "iter_save", "save_interval_epochs") - - if "scheduler_step_iter" in ts: - rename_key(ts, "scheduler_step_iter", "scheduler_step_on") - else: - ts["scheduler_step_on"] = "epoch" - - if not is_hp_search: - ts["enforce_determinism"] = False - ts["max_ram_gb"] = 16 - - return ts - - -def convert_train(config): - rename_key(config, "project_path", "project_root") - rename_key(config, "ddconfig_path", "metadata_config_path") - rename_key(config, "selected_columns", "input_columns") - - if "seed" not in config: - config["seed"] = 1010 - - assert "model_spec" in config, "model_spec missing" - rename_key(config, "model_spec", "model", convert_model) - - assert "training_spec" in config, "training_spec missing" - config["training_spec"] = convert_training_spec(config["training_spec"]) - for field_name in ("training_objective", "device"): - if field_name in config["training_spec"]: - config[field_name] = config["training_spec"].pop(field_name) - - return config - - -def convert_infer(config): - rename_key(config, "project_path", "project_root") - rename_key(config, "ddconfig_path", "metadata_config_path") - rename_key(config, "selected_columns", "input_columns") - rename_key( - config, - "autoregression_extra_steps", - "generation_steps", - lambda val: val + 1 if val is not None else val, - ) - - if "inference_size" in config: - rename_key(config, "inference_size", "prediction_length") - - if "seed" not in config: - config["seed"] = 1010 - - config["deterministic"] = False - - return config - - -def convert_hp_search(config): - rename_key(config, "project_path", "project_root") - rename_key(config, "ddconfig_path", "metadata_config_path") - rename_key(config, "selected_columns", "input_columns") - - # Add new export flags - config["export_generative_model"] = True - config["export_embedding_model"] = False - config["override_input"] = False - - assert ( - "model_hyperparameter_sampling" in config - ), "model_hyperparameter_sampling missing" - config["model_hyperparameter_sampling"] = convert_model( - config["model_hyperparameter_sampling"], is_hp_search=True - ) - - assert ( - "training_hyperparameter_sampling" in config - ), "training_hyperparameter_sampling missing" - config["training_hyperparameter_sampling"] = convert_training_spec( - config["training_hyperparameter_sampling"], is_hp_search=True - ) - - return config - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("config_type", type=str) - parser.add_argument("file_path", type=str) - args = parser.parse_args() - - assert args.config_type in [ - "preprocess", - "train", - "infer", - "hyperparameter-search", - ], "Invalid config-type" - - with open(args.file_path, "r") as f: - config = yaml.safe_load(f) - - if args.config_type == "preprocess": - new_config = convert_preprocess(config) - elif args.config_type == "train": - new_config = convert_train(config) - elif args.config_type == "infer": - new_config = convert_infer(config) - elif args.config_type == "hyperparameter-search": - new_config = convert_hp_search(config) - else: - new_config = {} - - with open(args.file_path, "w") as f: - f.write(yaml.dump(new_config, sort_keys=False, default_flow_style=False)) - - -if __name__ == "__main__": - main() diff --git a/tools/extract_validation_losses_to_table.py b/tools/extract_validation_losses_to_table.py index b42736d1..1cfb3837 100644 --- a/tools/extract_validation_losses_to_table.py +++ b/tools/extract_validation_losses_to_table.py @@ -14,7 +14,6 @@ def process_metrics(file_path: str) -> Path: losses["measurement"] = losses["metric"] + "." + losses["target"] context_columns = [ - "schema_version", "run_id", "session_id", "timestamp_utc", From dca04e2df6e6d00e344761bdbbe3b83bca331287 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 12 Nov 2026 12:09:13 +0100 Subject: [PATCH 31/71] clean up schema --- .../configs/hyperparameter-search.md | 24 +-- documentation/configs/naming.md | 3 +- documentation/configs/ownership.md | 18 +- documentation/configs/train.md | 15 ++ .../example_inputs/configs/train-bc-toy.yaml | 16 +- .../configs/train-iql-from-table.yaml | 16 +- .../example_inputs/configs/train-iql-toy.yaml | 16 +- .../configs/train-multiphase-control.yaml | 16 +- .../configs/train-policy-from-table.yaml | 16 +- .../configs/train-return-conditioned.yaml | 15 +- src/sequifier/artifacts/model_config.py | 1 - src/sequifier/artifacts/model_export.py | 5 - .../canonical_hyperparameter_search_config.py | 30 ++-- src/sequifier/config/command_ownership.py | 13 +- src/sequifier/config/composition.py | 6 +- src/sequifier/config/infer_config.py | 1 + src/sequifier/config/ownership.py | 3 +- src/sequifier/config/policy_infer_config.py | 1 + .../config/sequence_import_config.py | 11 +- src/sequifier/config/train_config.py | 162 +++++++++--------- src/sequifier/config/training_resolution.py | 7 +- src/sequifier/hyperparameter_search.py | 26 +-- src/sequifier/model/factory.py | 9 +- src/sequifier/templates/train-dpo.yaml | 16 +- src/sequifier/templates/train-grpo.yaml | 16 +- src/sequifier/templates/train-iql.yaml | 16 +- .../train-policy-from-supervised.yaml | 16 +- src/sequifier/templates/train-policy.yaml | 15 +- src/sequifier/templates/train-ppo.yaml | 16 +- .../templates/train-supervised-to-policy.yaml | 15 +- src/sequifier/training/coordinator.py | 5 - src/sequifier/training/runtime.py | 8 +- 32 files changed, 202 insertions(+), 351 deletions(-) diff --git a/documentation/configs/hyperparameter-search.md b/documentation/configs/hyperparameter-search.md index 50fde83c..fda886b3 100644 --- a/documentation/configs/hyperparameter-search.md +++ b/documentation/configs/hyperparameter-search.md @@ -17,8 +17,10 @@ search spaces using the same paths as the training schema. ```yaml base_config_path: train.yaml -name: transformer-width-search -trial_config_directory: configs/hp-search +run: + study_name: transformer-width-search + outputs: + trial_config_directory: configs/hp-search method: bayesian trials: 40 parameters: @@ -57,7 +59,7 @@ parameters: `base_config_path` resolves relative to the hyperparameter-search entry file. The base may itself be a composed training config. If the search config supplies -`project_root`, that value is used in every generated trial; otherwise the base +`run.project_root`, that value is used in every generated trial; otherwise the base training config's root is inherited. Use the same named paths as the training base: `model.interfaces.`, @@ -70,8 +72,8 @@ references, incompatible component types, and cross-field violations fail with their canonical validation paths. `model_name` and `project_root` cannot appear in `parameters`. Generated model -names use `[name]-run-[index]`, and `project_root` is controlled by the -top-level search field. +names use `[study_name]-run-[index]`, and `run.project_root` is controlled by the +search run field. ## Parameter expressions @@ -157,10 +159,10 @@ parameters: | --- | --- | --- | --- | --- | | `base_config_path` | `str` | Yes | - | Canonical training config used as the base. | | `parameters` | `mapping` | Yes | - | Recursive fixed values and search spaces. May be empty. | -| `project_root` | `str` | No | Base value | Root written to every generated training config. | -| `additional_config_paths` | `list[str]` or `null` | No | `null` | Complementary direct fragments of the search config. Relative paths resolve against the entry config's `project_root`; recursive composition and duplicate fields are rejected. | -| `name` | `str` | Yes | - | Study name and generated-run prefix. | -| `trial_config_directory` | `str` | Yes | - | Directory, under `project_root`, for generated trial configs. | +| `run.project_root` | `str` | No | Base value | Root used by the search and written to every generated training config. | +| `additional_config_paths` | `list[str]` or `null` | No | `null` | Complementary direct fragments of the search config. Relative paths resolve against the entry config's `run.project_root` (or `.` if omitted); recursive composition and duplicate fields are rejected. | +| `run.study_name` | `str` | Yes | - | Study name and generated-run prefix. | +| `run.outputs.trial_config_directory` | `str` | Yes | - | Directory, under `run.project_root`, for generated trial configs. | | `method` | `bayesian`, `random`, or `grid` | No | `bayesian` | Optuna TPE, random, or exhaustive finite-grid sampling. | | `trials` | `int` | Except grid | - | Target total number of completed or pruned runs in the persisted study. | | `sampler_seed` | `int` or `null` | No | `null` | Sampler seed. Training seeds belong in canonical `parameters`. | @@ -211,6 +213,6 @@ loading metadata-derived values, so all required authored fields must already be present. Generated canonical training configs are written below -`trial_config_directory`. The Optuna SQLite study is persisted at -`state/optuna/[name].db` under `project_root`, allowing later +`run.outputs.trial_config_directory`. The Optuna SQLite study is persisted at +`state/optuna/[study_name].db` under `run.project_root`, allowing later invocations to continue toward the configured total. diff --git a/documentation/configs/naming.md b/documentation/configs/naming.md index 8d97b782..b9dd688a 100644 --- a/documentation/configs/naming.md +++ b/documentation/configs/naming.md @@ -139,7 +139,8 @@ and model-owned versus fixed policy `scale`: these are meaningful distinctions. | --- | | `sampler_seed` | | `method: random` | -| `trial_config_directory` | +| `run.study_name` | +| `run.outputs.trial_config_directory` | | `evaluation_script_path` | | `evaluation_inference_config_path` | | `$choices` | diff --git a/documentation/configs/ownership.md b/documentation/configs/ownership.md index f35e7023..b4690658 100644 --- a/documentation/configs/ownership.md +++ b/documentation/configs/ownership.md @@ -19,6 +19,12 @@ remain on the command. Inference batching is `run.execution.batch_size`; preprocessing workers are `run.execution.data_loading.num_workers`. Unsupported execution controls fail validation. Component initialization and hash seeds stay on their components; evaluation seeds define a benchmark and stay in evaluation. +Unsupported fields must be omitted even when their values match the defaults. +Inference does not accept `run.execution.data_loading`, `distribution`, +`precision`, `compilation`, or `max_concat_tensors`. Preprocessing accepts +only `run.execution.data_loading.num_workers` within execution, and only `seed` +within reproducibility. Sequence import accepts `run.project_root` but no +execution or reproducibility settings. ## Run settings @@ -49,6 +55,13 @@ batched exports are rejected. Runs default to PT export. Supervised runs can req request those capabilities. Random run seeds resolve once; derived phase streams and resolved phase declarations are recorded in native recovery/provenance. +For policy-containing plans, omit `run.execution.distribution`, `precision`, +`compilation`, and `max_concat_tensors`, as well as `run.logging.log_interval_batches`. +Data loading may configure `episode_cache_size`, but not `num_workers`, +`load_full_data_to_ram`, or `max_ram_gb`. Omit the fixed export controls +`onnx`, `pt`, `generative_model`, `embedding_model`, `with_dropout`, and +`batch_size`; the PT actor export is selected automatically. + ## Dataset and model contracts Prepared parts own `read_format`; sequence parts own `format`. Raw datasets own @@ -61,8 +74,9 @@ training partitions only. `model.backbone.architecture.max_context_length` is capacity. Frozen input contracts and artifact assertions must agree. Phases cannot redefine immutable window, attention or codec facts. `learning.bert_spec` and `learning.next_occurrence_config` -configure objective behavior; reconstruction-required next-occurrence facts also -belong to the durable model contract and must agree across phases. +configure objective behavior for each supervised phase. Every next-occurrence +phase declares its trigger column and values; these define training and evaluation +targets and are not part of the portable execution contract. ## Phase settings diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 106bb2b8..8d0a4733 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -451,6 +451,21 @@ several future tokens as in LLM multi-token prediction. Training currently computes loss over all valid decoded positions, regardless of `prediction_length`. +For `model.execution.training_objective: next_occurrence`, declare the trigger +in each supervised phase's `learning` settings: + +```yaml +learning: + next_occurrence_config: + column_name: event_type + target_values: [purchase] +``` + +The trigger column must be a categorical target in every dataset used by that +phase, and its values must exist in that dataset's preprocessing metadata. +Different phases may select different triggers. These settings define training +and evaluation targets; they are not needed to execute an exported model. + ## Ownership and resolution - `run` owns execution resources, reproducibility, recovery, logging, outputs and export. diff --git a/documentation/example_inputs/configs/train-bc-toy.yaml b/documentation/example_inputs/configs/train-bc-toy.yaml index 7e2595a3..50a2ddc3 100644 --- a/documentation/example_inputs/configs/train-bc-toy.yaml +++ b/documentation/example_inputs/configs/train-bc-toy.yaml @@ -80,21 +80,7 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: {} logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index 56a5f498..e6ab19ff 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -98,23 +98,9 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: resume: policy: if_exists logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml index 485b3127..483cb33d 100644 --- a/documentation/example_inputs/configs/train-iql-toy.yaml +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -88,23 +88,9 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: resume: policy: if_exists logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index 03030a62..9e212492 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -207,21 +207,7 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: {} logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/documentation/example_inputs/configs/train-policy-from-table.yaml b/documentation/example_inputs/configs/train-policy-from-table.yaml index 7e637a97..adf8198a 100644 --- a/documentation/example_inputs/configs/train-policy-from-table.yaml +++ b/documentation/example_inputs/configs/train-policy-from-table.yaml @@ -85,21 +85,7 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: {} logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/documentation/example_inputs/configs/train-return-conditioned.yaml b/documentation/example_inputs/configs/train-return-conditioned.yaml index de7b565a..5106f224 100644 --- a/documentation/example_inputs/configs/train-return-conditioned.yaml +++ b/documentation/example_inputs/configs/train-return-conditioned.yaml @@ -90,15 +90,8 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} data_loading: episode_cache_size: 8 - compilation: - torch_compile: none recovery: resume: policy: never @@ -106,10 +99,4 @@ run: outputs: model_path: models/policy/actor.pt report_path: outputs/policy/training-report.json - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 5bca16e5..d9ec9908 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -112,7 +112,6 @@ def resolved_config_from_model_config( target_offset=target_offset, inference_batch_size=1, layer_type_dtypes=values.get("layer_type_dtypes"), - next_occurrence_config=values.get("next_occurrence_config"), torch_compile="none", ) authored_interfaces = {} diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index d3039062..793989c8 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -41,11 +41,6 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: or ("bidirectional" if spec.training_objective == "bert" else "causal"), "context_length": spec.context_length, "target_offset": spec.target_offset, - "next_occurrence_config": ( - spec.next_occurrence_config.model_dump(mode="python") - if spec.next_occurrence_config is not None - else None - ), "backbone": training_config.model.backbone.model_dump( mode="python", exclude={"repository"}, diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index 2b680962..a87a62be 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -628,6 +628,16 @@ def _compile_value(expression: Any, base: Any, path: ConfigPath) -> _CompiledVal return _LiteralValue(expression) +class SearchOutputSettings(ConfigModel): + trial_config_directory: str = Field(min_length=1) + + +class SearchRunSettings(ConfigModel): + project_root: str = Field(min_length=1) + study_name: str = Field(min_length=1) + outputs: SearchOutputSettings + + class CanonicalHyperparameterSearchConfig(ConfigModel): """Search controls and a recursive sampler over one canonical train config.""" @@ -639,16 +649,13 @@ class CanonicalHyperparameterSearchConfig(ConfigModel): base_config_path: str = Field(min_length=1) parameters: dict[str, Any] - project_root: str = Field(min_length=1) - - name: str = Field(min_length=1) + run: SearchRunSettings method: Literal["bayesian", "random", "grid"] = "bayesian" sampler_seed: RunSeed | None = None trials: int | None = Field(default=None, gt=0) prune_trials: bool = True pruning_warmup_epochs: int | None = Field(default=None, ge=0) pruning_warmup_batches: int | None = Field(default=None, ge=0) - trial_config_directory: str = Field(min_length=1) evaluation_inference_config_path: str | None = None evaluation_script_path: str | None = None @@ -700,16 +707,16 @@ def validate_search_controls(self): stacklevel=2, ) if self.evaluation_script_path is not None and not os.path.exists( - os.path.join(self.project_root, self.evaluation_script_path) + os.path.join(self.run.project_root, self.evaluation_script_path) ): raise ValueError( f"evaluation_script_path {self.evaluation_script_path!r} does not exist " - f"under project_root {self.project_root!r}" + f"under run.project_root {self.run.project_root!r}" ) if self.evaluation_inference_config_path is not None: inference_path = self.evaluation_inference_config_path if not os.path.isabs(inference_path): - inference_path = os.path.join(self.project_root, inference_path) + inference_path = os.path.join(self.run.project_root, inference_path) if not os.path.exists(inference_path): raise ValueError( f"evaluation_inference_config_path {self.evaluation_inference_config_path!r} " @@ -792,10 +799,10 @@ def _materialized_values(self, trial: Any | None, run_index: int) -> dict[str, A "range_concat", }: architecture["position_encoding_scope"] = "global" - values["run"]["project_root"] = self.project_root + values["run"]["project_root"] = self.run.project_root trial_count = self.trials if self.trials is not None else self.grid_size() values["run"]["model_name"] = ( - f"{self.name}-run-{run_index:0{len(str(trial_count))}d}" + f"{self.run.study_name}-run-{run_index:0{len(str(trial_count))}d}" ) return values @@ -865,13 +872,14 @@ def compile_canonical_hyperparameter_search_config( search_values = copy.deepcopy(config_values) search_values["base_config_path"] = base_config_path - search_values.setdefault("project_root", base_config.run.project_root) + if isinstance(search_values.get("run"), dict): + search_values["run"].setdefault("project_root", base_config.run.project_root) try: base_values = base_config.model_dump(mode="python") search_config = CanonicalHyperparameterSearchConfig.model_validate( search_values ) - base_values["run"]["project_root"] = search_config.project_root + base_values["run"]["project_root"] = search_config.run.project_root search_config._compiled_config = _compile_value( search_config.parameters, base_values, diff --git a/src/sequifier/config/command_ownership.py b/src/sequifier/config/command_ownership.py index 7c1e5987..f798d4c2 100644 --- a/src/sequifier/config/command_ownership.py +++ b/src/sequifier/config/command_ownership.py @@ -26,24 +26,21 @@ class CommandRunSettings(Settings): def validate_for(self, command): execution = self.execution - defaults = CommandExecutionSettings() for field in ("distribution", "precision", "compilation", "max_concat_tensors"): - if getattr(execution, field) != getattr(defaults, field): + if field in execution.model_fields_set: raise ValueError(f"{command} does not support run.execution.{field}") loading = execution.data_loading - if command == "infer" and loading != defaults.data_loading: + if command == "infer" and "data_loading" in execution.model_fields_set: raise ValueError("inference does not support data_loading controls") if command == "preprocess": if ( - execution.device != "cpu" - or execution.batch_size != 1 - or self.reproducibility.enforce_determinism + {"device", "batch_size"} & execution.model_fields_set + or "enforce_determinism" in self.reproducibility.model_fields_set ): raise ValueError( "preprocessing supports run execution workers and a seed; device/batching/determinism controls are unsupported" ) - other = loading.model_dump(exclude={"num_workers"}) - if other != defaults.data_loading.model_dump(exclude={"num_workers"}): + if loading.model_fields_set - {"num_workers"}: raise ValueError("preprocessing only supports data_loading.num_workers") return self diff --git a/src/sequifier/config/composition.py b/src/sequifier/config/composition.py index 3821734f..41521625 100644 --- a/src/sequifier/config/composition.py +++ b/src/sequifier/config/composition.py @@ -95,11 +95,7 @@ def load_composed_yaml_config( if not additional_paths: return entry_values - project_root = ( - entry_values.get("project_root") - if "base_config_path" in entry_values - else entry_values.get("run", {}).get("project_root", ".") - ) + project_root = entry_values.get("run", {}).get("project_root", ".") if not isinstance(project_root, str) or not project_root.strip(): raise ValueError( f"Config '{entry_path}' must define a non-empty run.project_root " diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 61fe93fa..29191773 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -833,6 +833,7 @@ def resolve_inference_config( "Prediction aligned inference requires window_stride: null" ) values = config.model_dump(mode="python") + values["run"] = config.run.model_dump(mode="python", exclude_unset=True) values.update( { "metadata_config_path": _effective_metadata_config_path(config), diff --git a/src/sequifier/config/ownership.py b/src/sequifier/config/ownership.py index 407ac2c3..628abc41 100644 --- a/src/sequifier/config/ownership.py +++ b/src/sequifier/config/ownership.py @@ -4,7 +4,7 @@ from pydantic import ConfigDict, Field, field_validator, model_validator -from sequifier.config.components import NextOccurrenceConfigModel, ResumeConfig +from sequifier.config.components import ResumeConfig from sequifier.config.naming import ConfigModel, RunSeed from sequifier.config.normalization_config import FrozenNormalizationConfig @@ -113,7 +113,6 @@ class ModelExecutionSettings(Settings): target_offset: int = Field(default=1, ge=0) training_objective: str | None = None forward_attention: Literal["causal", "bidirectional"] | None = None - next_occurrence_config: NextOccurrenceConfigModel | None = None class InputContract(Settings): diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 7b550f5c..a484c714 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -151,6 +151,7 @@ def policy_inference_config( in artifacts[model_path].model_config.values["interfaces"][selected], ) authored = request.model_dump() + authored["run"] = request.run.model_dump(mode="python", exclude_unset=True) authored["model_paths"] = [model_path] authored["model_interface"] = selected config = ResolvedPolicyInferenceConfig.model_validate(authored) diff --git a/src/sequifier/config/sequence_import_config.py b/src/sequifier/config/sequence_import_config.py index c8221309..60a55197 100644 --- a/src/sequifier/config/sequence_import_config.py +++ b/src/sequifier/config/sequence_import_config.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Literal -from pydantic import Field +from pydantic import Field, model_validator from sequifier.config.command_ownership import CommandRunSettings from sequifier.config.policy_config import ( @@ -23,6 +23,15 @@ class SequenceImportConfig(PolicyConfigBase): output_path: str episodes_per_shard: int = Field(default=64, gt=0) + @model_validator(mode="after") + def validate_resources(self): + unsupported = self.run.model_fields_set & {"execution", "reproducibility"} + if unsupported: + raise ValueError( + f"sequence import does not support run settings: {sorted(unsupported)}" + ) + return self + @property def project_root(self): return self.run.project_root diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 0d206270..6932b21c 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -690,14 +690,8 @@ def validate_training_contract(config): _validate_source(config, source.source, f"training phase {phase.name!r}") objective = get_objective_class(config.runtime_settings.training_objective) is_bert = issubclass(objective, BERTObjective) - is_next = issubclass(objective, NextOccurrenceObjective) if (config.runtime_settings.bert_spec is not None) != is_bert: raise ValueError("bert_spec must be configured exactly for BERT training") - if (config.runtime_settings.next_occurrence_config is not None) != is_next: - raise ValueError( - "next_occurrence_config must be configured exactly for " - "next_occurrence training" - ) if ( len(config.dataset_training) > 1 and config.runtime_settings.data_parallelism == "fsdp" @@ -997,44 +991,59 @@ def evaluation_sources(self) -> list[ResolvedTrainingSource]: @model_validator(mode="after") @beartype def validate_next_occurrence_metadata(self): - objective = get_objective_class(self.runtime_settings.training_objective) - if not issubclass(objective, NextOccurrenceObjective): - return self + self.validate_next_occurrence_phases() + return self - next_config = self.runtime_settings.next_occurrence_config - if next_config is None: - raise ValueError( - "next_occurrence_config must be set for next_occurrence training" - ) - column = next_config.column_name - for dataset_name, dataset in self.dataset_training.items(): - interface = dataset.interface - if column not in interface.target_columns: - raise ValueError( - "next_occurrence_config.column_name must be one of " - f"target_columns for dataset {dataset_name!r}, got {column!r}" - ) - if interface.target_column_types.get(column) != "categorical": - raise ValueError( - "next_occurrence_config.column_name must refer to a " - "categorical target column" - ) - if column not in interface.id_maps: + def validate_next_occurrence_phases(self, dataset_interfaces=None): + objective = get_objective_class(self.runtime_settings.training_objective) + is_next = issubclass(objective, NextOccurrenceObjective) + for phase in self.training_plan: + if phase.algorithm != "supervised": + continue + next_config = phase.learning.next_occurrence_config + if (next_config is not None) != is_next: raise ValueError( - "next_occurrence_config.column_name must have a " - f"preprocessing id_map, got {column!r}" + f"{phase.name}: learning.next_occurrence_config must be set " + "exactly for next_occurrence training" ) - missing = [ - value - for value in next_config.target_values - if value not in interface.id_maps[column] - ] - if missing: - raise ValueError( - "next_occurrence_config.target_values must match keys in " - f"id_maps[{column!r}] exactly, missing {missing!r}" + if next_config is None: + continue + column = next_config.column_name + names = { + source.dataset for source in phase.sources + phase.evaluation_sources + } + for dataset_name in names: + key = ( + dataset_interfaces[dataset_name] + if dataset_interfaces is not None + else dataset_name ) - return self + interface = self.dataset_training[key].interface + if column not in interface.target_columns: + raise ValueError( + f"{phase.name}: next_occurrence_config.column_name must be " + f"a target column for dataset {dataset_name!r}, got {column!r}" + ) + if interface.target_column_types.get(column) != "categorical": + raise ValueError( + f"{phase.name}: next_occurrence_config.column_name must refer " + "to a categorical target column" + ) + if column not in interface.id_maps: + raise ValueError( + f"{phase.name}: next_occurrence_config.column_name must have " + f"a preprocessing id_map, got {column!r}" + ) + missing = [ + value + for value in next_config.target_values + if value not in interface.id_maps[column] + ] + if missing: + raise ValueError( + f"{phase.name}: next_occurrence_config.target_values must " + f"match keys in id_maps[{column!r}], missing {missing!r}" + ) @model_validator(mode="after") @beartype @@ -2247,22 +2256,6 @@ def validate_contract(self): self.model is None or self.model.backbone is None ) and self.initialize_from is None: raise ValueError("a run requires a complete model or initialize_from") - next_occurrence = ( - self.model.execution.next_occurrence_config if self.model else None - ) - for phase in self.training_plan.phases: - if ( - phase.algorithm == "supervised" - and phase.learning.next_occurrence_config is not None - ): - if ( - next_occurrence is not None - and next_occurrence != phase.learning.next_occurrence_config - ): - raise ValueError( - "phase next_occurrence_config differs from durable model.execution contract" - ) - next_occurrence = phase.learning.next_occurrence_config for name, dataset in self.dataset_training.items(): _identifier(name, "Dataset name") for part in dataset.parts: @@ -2314,41 +2307,42 @@ def validate_contract(self): ) if any(phase.algorithm != "supervised" for phase in self.training_plan.phases): execution = self.run.execution - if execution.distribution.strategy != "local": - raise ValueError("policy phases currently require local execution") - if execution.compilation.torch_compile != "none": - raise ValueError( - "policy phases do not support run.execution.compilation" - ) - if execution.precision != type(execution.precision)(): - raise ValueError( - "policy phases do not support custom precision settings" - ) - if execution.max_concat_tensors != 12: + unsupported_execution = execution.model_fields_set & { + "distribution", + "compilation", + "precision", + "max_concat_tensors", + } + if unsupported_execution: raise ValueError( - "policy phases do not support custom max_concat_tensors" + "policy phases do not support run.execution settings: " + f"{sorted(unsupported_execution)}" ) - if self.run.logging.log_interval_batches != 10: + if "log_interval_batches" in self.run.logging.model_fields_set: raise ValueError("policy phases do not support batch logging intervals") export = self.run.export - if ( - export.onnx - or export.embedding_model - or export.with_dropout - or not export.generative_model - or not export.pt - or export.batch_size != 1 - ): + unsupported_export = export.model_fields_set & { + "onnx", + "embedding_model", + "with_dropout", + "generative_model", + "pt", + "batch_size", + } + if unsupported_export: raise ValueError( - "policy-containing plans require portable PT actor export; graph/embedding/dropout/batched exports are unsupported" + "policy-containing plans do not support run.export settings: " + f"{sorted(unsupported_export)}" ) - if ( - execution.data_loading.num_workers - or not execution.data_loading.load_full_data_to_ram - or execution.data_loading.max_ram_gb != 16 - ): + unsupported_loading = execution.data_loading.model_fields_set & { + "num_workers", + "load_full_data_to_ram", + "max_ram_gb", + } + if unsupported_loading: raise ValueError( - "policy phases do not support worker or lazy-loading controls" + "policy phases do not support run.execution.data_loading settings: " + f"{sorted(unsupported_loading)}" ) return self diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 90355f78..2ce02fdc 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -374,7 +374,6 @@ def resolve_training_run(authored: SequifierConfig): "training_objective", "target_offset", "forward_attention", - "next_occurrence_config", ): setattr(config.model.execution, field, getattr(training, field)) @@ -581,6 +580,12 @@ def resolve_training_run(authored: SequifierConfig): resolved.training_plan = [ supervised.get(phase.name, phase) for phase in config.training_plan.phases ] + resolved.validate_next_occurrence_phases( + { + name: dataset.model_interface + for name, dataset in config.dataset_training.items() + } + ) for index, phase in enumerate(resolved.training_plan): if phase.algorithm == "supervised": phase.resolved_seed = config.run.reproducibility.seed + index diff --git a/src/sequifier/hyperparameter_search.py b/src/sequifier/hyperparameter_search.py index 23c44d1c..49d65b90 100644 --- a/src/sequifier/hyperparameter_search.py +++ b/src/sequifier/hyperparameter_search.py @@ -91,7 +91,9 @@ def objective( monitored_dataset, dataset_count = _monitored_dataset(run_config) config_path = os.path.join( - config.project_root, config.trial_config_directory, f"{run_name}.yaml" + config.run.project_root, + config.run.outputs.trial_config_directory, + f"{run_name}.yaml", ) os.makedirs(os.path.dirname(config_path), exist_ok=True) @@ -103,12 +105,12 @@ def objective( sort_keys=False, ) - log_dir = model_log_directory(config.project_root, run_name) + log_dir = model_log_directory(config.run.project_root, run_name) log_dir.mkdir(parents=True, exist_ok=True) validation_path = ( str( dataset_artifact_prefix( - config.project_root, + config.run.project_root, run_name, dataset_name=monitored_dataset, dataset_count=dataset_count, @@ -198,7 +200,7 @@ def consume_metrics( try: try: timedelta = get_last_training_batch_timedelta( - run_name, 0, config.project_root + run_name, 0, config.run.project_root ) timeout_val = (timedelta * 2) + 30 except (ValueError, FileNotFoundError): @@ -232,7 +234,7 @@ def consume_metrics( model_type = "onnx" if run_config.run.export.onnx else "pt" model_path, _last_epoch = get_best_model_path( - config.project_root, + config.run.project_root, run_name, model_type, dataset_name=(monitored_dataset if model_type == "onnx" else None), @@ -246,7 +248,7 @@ def consume_metrics( evaluation_inference_config_path ): evaluation_inference_config_path = os.path.join( - config.project_root, + config.run.project_root, evaluation_inference_config_path, ) subprocess.run( @@ -264,7 +266,7 @@ def consume_metrics( cmd = [sys.executable, eval_script_path, evaluation_id] eval_process = subprocess.run( - cmd, capture_output=True, text=True, cwd=config.project_root + cmd, capture_output=True, text=True, cwd=config.run.project_root ) if eval_process.returncode != 0: @@ -273,7 +275,7 @@ def consume_metrics( ) eval_json_path = os.path.join( - config.project_root, + config.run.project_root, "outputs", "evaluations", f"{evaluation_id}.json", @@ -411,11 +413,11 @@ def hyperparameter_search(config_path: str, skip_metadata: bool) -> None: """Load config, create Optuna study, and optimize trials.""" config = load_hyperparameter_search_config(config_path, skip_metadata) - os.makedirs(os.path.join(config.project_root, "state", "optuna"), exist_ok=True) + os.makedirs(os.path.join(config.run.project_root, "state", "optuna"), exist_ok=True) sampler = create_sampler(config) storage_path = os.path.join( - config.project_root, "state", "optuna", f"{config.name}.db" + config.run.project_root, "state", "optuna", f"{config.run.study_name}.db" ) is_multivariate = ( @@ -425,7 +427,7 @@ def hyperparameter_search(config_path: str, skip_metadata: bool) -> None: if is_multivariate: assert config.evaluation_metric_directions is not None study = optuna.create_study( - study_name=config.name, + study_name=config.run.study_name, directions=[ {"min": "minimize", "max": "maximize"}[value] for value in config.evaluation_metric_directions @@ -444,7 +446,7 @@ def hyperparameter_search(config_path: str, skip_metadata: bool) -> None: else "min" ) study = optuna.create_study( - study_name=config.name, + study_name=config.run.study_name, direction={"min": "minimize", "max": "maximize"}[direction], sampler=sampler, storage=f"sqlite:///{storage_path}", diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index 472759a8..c17de510 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -266,12 +266,9 @@ def _build_composable_network( objectives = {} for name, interface in resolved_interfaces.items(): objective_view = interface_build_view(config, interface) - # A lean inference bundle may omit next-occurrence loss metadata. Its - # forward attention policy remains the ordinary causal policy. - if ( - config.runtime_settings.training_objective == "next_occurrence" - and config.runtime_settings.next_occurrence_config is None - ): + # Next-occurrence target construction belongs to each training phase; + # its forward attention policy is the ordinary causal policy. + if config.runtime_settings.training_objective == "next_occurrence": objectives[name] = CausalObjective(objective_view) else: objectives[name] = create_objective(objective_view) diff --git a/src/sequifier/templates/train-dpo.yaml b/src/sequifier/templates/train-dpo.yaml index 8d438d0a..7bfce46a 100644 --- a/src/sequifier/templates/train-dpo.yaml +++ b/src/sequifier/templates/train-dpo.yaml @@ -100,21 +100,7 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: {} logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/src/sequifier/templates/train-grpo.yaml b/src/sequifier/templates/train-grpo.yaml index 9a43f6a1..e3bcee12 100644 --- a/src/sequifier/templates/train-grpo.yaml +++ b/src/sequifier/templates/train-grpo.yaml @@ -94,21 +94,7 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: {} logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/src/sequifier/templates/train-iql.yaml b/src/sequifier/templates/train-iql.yaml index 322412cb..8fb488df 100644 --- a/src/sequifier/templates/train-iql.yaml +++ b/src/sequifier/templates/train-iql.yaml @@ -87,23 +87,9 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: resume: policy: if_exists logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/src/sequifier/templates/train-policy-from-supervised.yaml b/src/sequifier/templates/train-policy-from-supervised.yaml index 185b22d1..4e0c930a 100644 --- a/src/sequifier/templates/train-policy-from-supervised.yaml +++ b/src/sequifier/templates/train-policy-from-supervised.yaml @@ -47,24 +47,10 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: resume: policy: if_exists mode: reset_environments logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/src/sequifier/templates/train-policy.yaml b/src/sequifier/templates/train-policy.yaml index dedcb47f..3e1ebe1d 100644 --- a/src/sequifier/templates/train-policy.yaml +++ b/src/sequifier/templates/train-policy.yaml @@ -87,24 +87,11 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} data_loading: episode_cache_size: 8 - compilation: - torch_compile: none recovery: resume: policy: never logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/src/sequifier/templates/train-ppo.yaml b/src/sequifier/templates/train-ppo.yaml index 66e89bcb..96e3a5d4 100644 --- a/src/sequifier/templates/train-ppo.yaml +++ b/src/sequifier/templates/train-ppo.yaml @@ -95,24 +95,10 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} - data_loading: {} - compilation: - torch_compile: none recovery: resume: policy: if_exists mode: exact logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/src/sequifier/templates/train-supervised-to-policy.yaml b/src/sequifier/templates/train-supervised-to-policy.yaml index ea2c9262..ee0ab7c6 100644 --- a/src/sequifier/templates/train-supervised-to-policy.yaml +++ b/src/sequifier/templates/train-supervised-to-policy.yaml @@ -127,24 +127,11 @@ run: enforce_determinism: false execution: device: cpu - distribution: - strategy: local - world_size: 1 - backend: nccl - precision: {} data_loading: episode_cache_size: 8 - compilation: - torch_compile: none recovery: resume: policy: never logging: {} outputs: {} - export: - onnx: false - pt: true - batch_size: 1 - generative_model: true - embedding_model: false - with_dropout: false + export: {} diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 8c8d58c6..b93ee02a 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -84,13 +84,8 @@ def _supervised_config(config, phase, index, incoming, resume): "training_objective", "target_offset", "forward_attention", - "next_occurrence_config", ) } - if training.next_occurrence_config is not None: - model_values["execution"]["next_occurrence_config"] = ( - training.next_occurrence_config.model_dump() - ) for name, interface in authored.model.interfaces.items(): if name in model_values["interfaces"]: model_values["interfaces"][name]["input_contract"] = ( diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 1ebff218..eafa26a6 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -728,7 +728,6 @@ def configure_phase(config, network, datasets, phase_index, device): config.runtime_settings.bert_spec = phase.learning.bert_spec config.runtime_settings.next_occurrence_config = ( phase.learning.next_occurrence_config - or config.runtime_settings.next_occurrence_config ) config.runtime_settings.window_stride = phase.learning.window_stride config.curriculum_training = phase.curriculum is not None @@ -743,9 +742,10 @@ def configure_phase(config, network, datasets, phase_index, device): dataset.class_weights = learning.class_weights dataset.loss_weights = learning.loss_weights dataset.frozen_parameter_ids = masks[name] - dataset.objective = create_objective( - interface_build_view(config, dataset.config.interface) - ) + if name in phase.learning.datasets: + dataset.objective = create_objective( + interface_build_view(config, dataset.config.interface) + ) for part in dataset.parts.values(): part.factory.loaders.clear() part.factory.batch_size = phase.batch_size From b42f6b934da9452202b71ea50ffb338f17a62926 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 12 Nov 2026 17:06:31 +0100 Subject: [PATCH 32/71] small changes --- .../configs/train-iql-from-table.yaml | 3 ++- src/sequifier/artifacts/model_config.py | 2 ++ .../canonical_hyperparameter_search_config.py | 26 ++++++++++++++++--- .../config/input_contract_resolution.py | 9 +++---- .../io/sequifier_dataset_from_file.py | 3 +-- src/sequifier/rl/collection.py | 2 +- src/sequifier/rl/prepared_inputs.py | 25 +++++++++++++++--- 7 files changed, 53 insertions(+), 17 deletions(-) diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index e6ab19ff..acd5c039 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -42,7 +42,8 @@ dataset_training: columns: sequence_id: sequenceId item_position: itemPosition - action: action + actions: + action: action reward: reward terminated: terminated truncated: truncated diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index d9ec9908..101c490e 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -219,6 +219,8 @@ def execution_config(model, interfaces, runtime_settings, *, device="cpu", **opt def extend_prediction_distributions(execution, declared_model): """Allow explicit new scale parameters while preserving an initialized model.""" exclude = {"interfaces": {"__all__": {"prediction_distributions"}}} + if hasattr(declared_model, "architecture_model"): + declared_model = declared_model.architecture_model() if declared_model.model_dump(exclude=exclude) != execution.model.model_dump( exclude=exclude ): diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index a87a62be..50681432 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -24,6 +24,24 @@ from sequifier.config.train_config import SequifierConfig, load_train_config_with_source from sequifier.typechecking import beartype + +def _validate_training_candidate(values: dict[str, Any]) -> SequifierConfig: + from sequifier.config.train_config import validate_training_contract + + candidate = SequifierConfig.model_validate(values) + validate_training_contract(candidate) + is_next = candidate.runtime_settings.training_objective == "next_occurrence" + for phase in candidate.training_plan.phases: + if phase.algorithm != "supervised": + continue + if (phase.learning.next_occurrence_config is not None) != is_next: + raise ValueError( + f"{phase.name}: learning.next_occurrence_config must be set " + "exactly for next_occurrence training" + ) + return candidate + + ConfigPath = tuple[str | int, ...] _MISSING = object() _DISTRIBUTION_KEYS = frozenset({"low", "high", "step", "log", "type"}) @@ -759,8 +777,8 @@ def validate_compiled_search(self) -> None: for description, trial in candidates: try: - SequifierConfig.model_validate(self._materialized_values(trial, 0)) - except ValidationError as error: + _validate_training_candidate(self._materialized_values(trial, 0)) + except (ValidationError, ValueError) as error: raise ValueError( "Search parameters produce an invalid " f"training config for {description}:\n{error}" @@ -812,8 +830,8 @@ def sample_trial(self, trial: Any, run_index: int) -> SequifierConfig: values = self._materialized_values(trial, run_index) try: - return SequifierConfig.model_validate(values) - except ValidationError as error: + return _validate_training_candidate(values) + except (ValidationError, ValueError) as error: parameters = getattr(trial, "params", {}) raise ValueError( "Sampled hyperparameters produce an invalid canonical training " diff --git a/src/sequifier/config/input_contract_resolution.py b/src/sequifier/config/input_contract_resolution.py index 436d28f0..b62eb086 100644 --- a/src/sequifier/config/input_contract_resolution.py +++ b/src/sequifier/config/input_contract_resolution.py @@ -36,14 +36,11 @@ def validate_input_contracts(config, resolved): for dataset in datasets: interface = dataset.interface expected = normalization.statistics + real_inputs = interface.real_columns + interface.static_real_columns if normalization.source == "identity": - expected = { - column: {"mean": 0.0, "std": 1.0} - for column in interface.input_columns + interface.static_columns - if column in interface.real_columns - } + expected = {column: {"mean": 0.0, "std": 1.0} for column in real_inputs} for column, values in expected.items(): - if column not in interface.real_columns: + if column not in real_inputs: raise ValueError( f"{name}: normalization references unknown/non-real input {column!r}" ) diff --git a/src/sequifier/io/sequifier_dataset_from_file.py b/src/sequifier/io/sequifier_dataset_from_file.py index 54d9fed4..8ee341d8 100644 --- a/src/sequifier/io/sequifier_dataset_from_file.py +++ b/src/sequifier/io/sequifier_dataset_from_file.py @@ -14,7 +14,6 @@ resolve_window_sampling_plan, ) from sequifier.io.batch import SequifierBatch -from sequifier.io.config import runtime_settings from sequifier.io.images import ImageStore from sequifier.io.iteration_state import ( read_shared_int, @@ -184,7 +183,7 @@ def set_start_batch(self, start_batch: int): @beartype def __len__(self) -> int: - num_workers = max(1, runtime_settings(self.config).num_workers) + num_workers = max(1, self.config.runtime_settings.num_workers) total_batches = 0 for worker_id in range(num_workers): worker_samples = self.n_samples // num_workers + ( diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index 86378b78..e727df59 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -71,8 +71,8 @@ def decide(self, value=None, *, deterministic=False): actor.eval() if value is not None: value.eval() - features, metadata = actor.model_inputs(self.inputs) distribution = actor.distribution(self.inputs) + features, metadata = distribution.features, distribution.metadata action = distribution.select(deterministic, self.generator) estimate = ( value.prepared_value(features, metadata) diff --git a/src/sequifier/rl/prepared_inputs.py b/src/sequifier/rl/prepared_inputs.py index 626524fe..041a95eb 100644 --- a/src/sequifier/rl/prepared_inputs.py +++ b/src/sequifier/rl/prepared_inputs.py @@ -90,6 +90,7 @@ def __init__(self): self._checksums = {} self._inputs = {} self._partitions = {} + self._catalogs = {} self._preference_manifests = {} @classmethod @@ -198,14 +199,32 @@ def prepare_phase(self, config, phase, policy, inputs, *, fitting=False): } ) if key not in self._partitions: - self._partitions[key] = _PolicyPartitions( - PolicyDataCatalog( + catalog_key = fingerprint( + { + "sources": { + name: { + "source": source.source.model_dump(mode="json"), + "paths": [str(path) for path in source.paths], + "kind": source.kind, + "checksums": { + str(path): value + for path, value in source.checksums.items() + }, + } + for name, source in sources.items() + }, + "policy": _data_policy_contract(policy), + "episode_cache_size": config.episode_cache_size, + } + ) + if catalog_key not in self._catalogs: + self._catalogs[catalog_key] = PolicyDataCatalog( sources, policy, config.episode_cache_size, preference_manifests=self._preference_manifests, ) - ) + self._partitions[key] = _PolicyPartitions(self._catalogs[catalog_key]) partitions = self._partitions[key] partitions.rebind(policy) return prepare(config, phase, policy, partitions.catalog, partitions.prepared) From 59573f3ea3a83855048e0950e62ae0c660431a78 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 12 Nov 2026 19:58:56 +0100 Subject: [PATCH 33/71] small changes --- documentation/configs/ownership.md | 7 +++- src/sequifier/accurate_softmax.py | 9 +++-- src/sequifier/artifacts/model_config.py | 13 ++++-- src/sequifier/config/components.py | 2 +- src/sequifier/config/parameter_selection.py | 44 +++++++++++++++++++-- src/sequifier/model/layers.py | 4 +- src/sequifier/optimizers/plan.py | 12 +++++- src/sequifier/preprocess.py | 7 +++- src/sequifier/rl/learner.py | 11 +++++- src/sequifier/training/coordinator.py | 5 +++ src/sequifier/training/loss.py | 16 +++++--- 11 files changed, 108 insertions(+), 22 deletions(-) diff --git a/documentation/configs/ownership.md b/documentation/configs/ownership.md index b4690658..2e33f077 100644 --- a/documentation/configs/ownership.md +++ b/documentation/configs/ownership.md @@ -138,7 +138,12 @@ Freezing and optimizer routing use the same catalog selectors: `component`, `branch_path`, or a `module` subtree. Conditions in a selector are conjunctive; selector lists form a union. `exclude_modules` and `exclude_semantic_groups` express explicit subtractions, for nested component policies. -Unknown or empty selections fail preflight. +Unknown or empty selections fail preflight. Every `semantic_groups` pattern must +match at least one eligible parameter within the selector's structural scope; +every `exclude_semantic_groups` pattern must match the positively selected scope +before exclusions are applied. Patterns are case-sensitive and are never +corrected or ignored. A valid alternative does not hide an unmatched pattern. +Overlapping and repeated matching patterns remain valid union selections. Each role accepts either `freeze` or `freeze_except`. Empty `freeze` masks nothing; empty `freeze_except` masks everything and fails if an update has no trainable diff --git a/src/sequifier/accurate_softmax.py b/src/sequifier/accurate_softmax.py index 9571dba1..866107d8 100644 --- a/src/sequifier/accurate_softmax.py +++ b/src/sequifier/accurate_softmax.py @@ -18,9 +18,12 @@ def accurate_softmax_terms( zero = scores - scores return zero, zero - selected = scores.gather(-1, selected_index) - shifted = (scores - selected).scatter(-1, selected_index, 0.0) - others = shifted.scatter(-1, selected_index, -torch.inf) + selected_mask = ( + torch.arange(scores.shape[-1], device=scores.device) == selected_index + ) + selected = torch.where(selected_mask, scores, 0.0).sum(dim=-1, keepdim=True) + shifted = torch.where(selected_mask, 0.0, scores - selected) + others = torch.where(selected_mask, -torch.inf, shifted) log_normalizer = F.softplus(torch.logsumexp(others, dim=-1, keepdim=True)) return shifted, log_normalizer diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 101c490e..6fcef2a1 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -221,9 +221,16 @@ def extend_prediction_distributions(execution, declared_model): exclude = {"interfaces": {"__all__": {"prediction_distributions"}}} if hasattr(declared_model, "architecture_model"): declared_model = declared_model.architecture_model() - if declared_model.model_dump(exclude=exclude) != execution.model.model_dump( - exclude=exclude - ): + declared = declared_model.model_dump(exclude=exclude) + existing = execution.model.model_dump(exclude=exclude) + for name, interface in declared["interfaces"].items(): + if name in existing["interfaces"]: + tokens = interface["categorical_decoder_special_tokens"] + for target, value in existing["interfaces"][name][ + "categorical_decoder_special_tokens" + ].items(): + tokens.setdefault(target, value) + if declared != existing: raise ValueError( "declared model architecture differs from initialize_from model" ) diff --git a/src/sequifier/config/components.py b/src/sequifier/config/components.py index 778d1643..c20b4232 100644 --- a/src/sequifier/config/components.py +++ b/src/sequifier/config/components.py @@ -792,7 +792,7 @@ class BackboneFeedForwardConfig(ConfigModel): model_config = ConfigDict(extra="forbid") hidden_dim: int = Field(..., gt=0) - activation: Literal["relu", "gelu", "swiglu"] = "swiglu" + activation: Literal["relu", "gelu", "swiglu", "softplus"] = "swiglu" class BackboneNormalizationConfig(ConfigModel): diff --git a/src/sequifier/config/parameter_selection.py b/src/sequifier/config/parameter_selection.py index a819cb6e..7c37fbba 100644 --- a/src/sequifier/config/parameter_selection.py +++ b/src/sequifier/config/parameter_selection.py @@ -107,6 +107,38 @@ def in_module(name, module): and (self.module is None or any(in_module(n, self.module) for n in names)) ) + def validate_semantic_patterns(self, descriptors): + """Require every literal/glob to match within this selector's scope. + + Validate inclusions before subtraction, so overlapping or repeated patterns + remain valid. Exclusions must match the positively selected scope. Never + normalize, autocorrect, or silently discard an unmatched pattern. + """ + structural = self.model_copy( + update={"semantic_groups": None, "exclude_semantic_groups": []} + ) + scoped = [ + descriptor for descriptor in descriptors if structural.matches(descriptor) + ] + positive = self.model_copy(update={"exclude_semantic_groups": []}) + included = [descriptor for descriptor in scoped if positive.matches(descriptor)] + for field, candidates in ( + ("semantic_groups", scoped), + ("exclude_semantic_groups", included), + ): + for pattern in getattr(self, field) or []: + if not any( + fnmatchcase(group, pattern) + for descriptor in candidates + for group in ( + descriptor.semantic_aliases or (descriptor.semantic_group,) + ) + ): + raise ValueError( + f"{field} pattern {pattern!r} matched no parameters " + "in the selector's eligible scope" + ) + class RoleTrainability(ConfigModel): model_config = ConfigDict(extra="forbid") @@ -144,10 +176,16 @@ def resolve(self, module, *, eligible=None): ) if unknown := set(paths) - known_modules: raise ValueError(f"unknown selector modules: {sorted(unknown)}") + eligible_descriptors = [ + descriptor + for descriptor in catalog.descriptors() + if descriptor.parameter_id in candidates + ] + selector.validate_semantic_patterns(eligible_descriptors) matched = { - d.parameter_id - for d in catalog.descriptors() - if d.parameter_id in candidates and selector.matches(d) + descriptor.parameter_id + for descriptor in eligible_descriptors + if selector.matches(descriptor) } if not matched: raise ValueError( diff --git a/src/sequifier/model/layers.py b/src/sequifier/model/layers.py index 7763fd08..d9d46883 100644 --- a/src/sequifier/model/layers.py +++ b/src/sequifier/model/layers.py @@ -88,7 +88,9 @@ def __init__(self, model_dim, dim_feedforward, activation, dropout): else: self.linear1 = nn.Linear(model_dim, dim_feedforward) self.linear2 = nn.Linear(dim_feedforward, model_dim) - self.act = nn.GELU() if activation == "gelu" else nn.ReLU() + self.act = {"gelu": nn.GELU, "relu": nn.ReLU, "softplus": nn.Softplus}[ + activation + ]() self.dropout = nn.Dropout(dropout) diff --git a/src/sequifier/optimizers/plan.py b/src/sequifier/optimizers/plan.py index a335a258..98a8aa8e 100644 --- a/src/sequifier/optimizers/plan.py +++ b/src/sequifier/optimizers/plan.py @@ -40,8 +40,16 @@ def route_optimizer_parameters( seen: set[int] = set() explicit = plan.groups[:-1] fallback = plan.groups[-1] - - for descriptor in catalog.descriptors(): + eligible_descriptors = [ + descriptor + for descriptor in catalog.descriptors() + if id(catalog.parameter(descriptor.parameter_id)) in supplied + ] + for group in explicit: + if isinstance(group.select, OptimizerSelector): + group.select.validate_semantic_patterns(eligible_descriptors) + + for descriptor in eligible_descriptors: parameter = catalog.parameter(descriptor.parameter_id) identity = id(parameter) if identity not in supplied or identity in seen: diff --git a/src/sequifier/preprocess.py b/src/sequifier/preprocess.py index e7168c79..aad7feda 100644 --- a/src/sequifier/preprocess.py +++ b/src/sequifier/preprocess.py @@ -722,7 +722,12 @@ def preprocess(args: Any, args_config: dict[str, Any]) -> None: import_sequences(config) else: - Preprocessor(**config.dict()) + Preprocessor( + **config.model_dump(exclude={"run"}), + project_root=config.project_root, + seed=config.seed, + num_workers=config.num_workers, + ) logger.info("--- Preprocessing Complete ---") diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 15eea467..3c35d9e3 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -130,9 +130,16 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: actor = _initialize_actor(config) representation = ActorRepresentation.from_actor(actor, config.units) if config.normalization is not None: + expected_normalization = resolve_normalization( + actor.config, config.normalization + ) require_equal( - resolve_normalization(actor.config, config.normalization), - representation.values["statistics"], + expected_normalization, + { + key: value + for key, value in representation.values["statistics"].items() + if key in expected_normalization + }, "run.normalization", ) validate_data(data, actor) diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index b93ee02a..754c731d 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -112,6 +112,11 @@ def _supervised_config(config, phase, index, incoming, resume): resolved.ownership = config.ownership resolved.source_validation_fractions = config.source_validation_fractions resolved.normalization_by_interface = config.normalization_by_interface + if source is None: + for dataset in resolved.dataset_training.values(): + dataset.interface.policy = config.policy_bindings.get( + dataset.model_interface + ) if source is not None: # Preserve policy bindings and their model-owned scale modules on return # to supervised learning. Dataset resolution owns the data/window views. diff --git a/src/sequifier/training/loss.py b/src/sequifier/training/loss.py index bf8d8171..555fd699 100644 --- a/src/sequifier/training/loss.py +++ b/src/sequifier/training/loss.py @@ -196,11 +196,15 @@ def calculate( ) criterion = dataset.criteria[target] if isinstance(criterion, AccurateCrossEntropyLoss): - # Exclude masked rows before evaluating the loss so + # Replace masked rows before evaluating the loss so # their logits cannot enter the backward graph. + safe_logits = torch.where( + flat_mask[:, None], flat_logits, 0.0 + ) + safe_label = torch.where(flat_mask, label, 0) component_sums.append( - criterion( - flat_logits[flat_mask], label[flat_mask] + torch.where( + flat_mask, criterion(safe_logits, safe_label), 0.0 ).sum() ) else: @@ -279,8 +283,10 @@ def calculate( ) sums[target] = -distribution.log_prob(observed).sum() elif isinstance(criterion, AccurateCrossEntropyLoss): - sums[target] = criterion( - logits_for_loss[flat_mask], target_for_loss[flat_mask] + safe_logits = torch.where(flat_mask[:, None], logits_for_loss, 0.0) + safe_target = torch.where(flat_mask, target_for_loss, 0) + sums[target] = torch.where( + flat_mask, criterion(safe_logits, safe_target), 0.0 ).sum() else: raw = criterion(logits_for_loss, target_for_loss) From 6e7f056aaf70b2628c922567073a0862b196efcd Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Fri, 13 Nov 2026 12:32:47 +0100 Subject: [PATCH 34/71] change --- src/sequifier/config/policy_optimization.py | 3 ++ src/sequifier/export/onnx.py | 34 +++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/sequifier/config/policy_optimization.py b/src/sequifier/config/policy_optimization.py index fbe79499..6df2b84a 100644 --- a/src/sequifier/config/policy_optimization.py +++ b/src/sequifier/config/policy_optimization.py @@ -160,6 +160,9 @@ def validate_policy_scheduler(self): class SupervisedOptimizationConfig(RoleOptimizationConfig): """Epoch scheduling includes metric-driven ReduceLROnPlateau.""" + # Supervised training historically leaves gradients unclipped unless requested. + gradient_clip_norm: float | None = Field(default=None, gt=0) + scheduler_step_on: Literal["epoch", "optimizer_step"] = "epoch" scheduler: ComponentSpec = Field( default_factory=lambda: ComponentSpec( diff --git a/src/sequifier/export/onnx.py b/src/sequifier/export/onnx.py index 212e4b97..b88078a4 100644 --- a/src/sequifier/export/onnx.py +++ b/src/sequifier/export/onnx.py @@ -26,6 +26,39 @@ from sequifier.runtime.random_state import RandomStateManager +class _ExportLayerNorm(nn.LayerNorm): + """Lower centered variance instead of ONNX Runtime's cancellation-prone norm.""" + + def forward(self, value: Tensor) -> Tensor: + if not torch.compiler.is_compiling(): + # Publication parity must still compare against native PyTorch. + return super().forward(value) + axes = tuple(range(-len(self.normalized_shape), 0)) + centered = value - value.mean(dim=axes, keepdim=True) + variance = centered.square().mean(dim=axes, keepdim=True) + result = centered * torch.rsqrt(variance + self.eps) + if self.weight is not None: + result = result * self.weight + if self.bias is not None: + result = result + self.bias + return result + + +def _lower_layer_norms(module: nn.Module) -> None: + for name, child in tuple(module.named_children()): + if type(child) is nn.LayerNorm: + replacement = _ExportLayerNorm( + child.normalized_shape, + eps=child.eps, + elementwise_affine=child.elementwise_affine, + bias=child.bias is not None, + ) + replacement.load_state_dict(child.state_dict()) + setattr(module, name, replacement) + else: + _lower_layer_norms(child) + + class _OnnxWrapper(nn.Module): def __init__(self, model, schema, interface_name): super().__init__() @@ -309,6 +342,7 @@ def export( try: # All graph lowering and dtype conversion happen on a disposable copy. model = copy.deepcopy(model).cpu().float() + _lower_layer_norms(model) for module in model.modules(): if isinstance(module, SelfAttention): module._sequifier_export_attention = True From 4bcd151d8b59bce3ecce5d5d7014f54a18922b24 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Fri, 13 Nov 2026 15:13:16 +0100 Subject: [PATCH 35/71] reverse computation changes --- src/sequifier/accurate_softmax.py | 9 +++------ src/sequifier/config/ownership.py | 2 +- src/sequifier/training/engine.py | 3 +-- src/sequifier/training/loss.py | 16 +++++----------- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/sequifier/accurate_softmax.py b/src/sequifier/accurate_softmax.py index 866107d8..9571dba1 100644 --- a/src/sequifier/accurate_softmax.py +++ b/src/sequifier/accurate_softmax.py @@ -18,12 +18,9 @@ def accurate_softmax_terms( zero = scores - scores return zero, zero - selected_mask = ( - torch.arange(scores.shape[-1], device=scores.device) == selected_index - ) - selected = torch.where(selected_mask, scores, 0.0).sum(dim=-1, keepdim=True) - shifted = torch.where(selected_mask, 0.0, scores - selected) - others = torch.where(selected_mask, -torch.inf, shifted) + selected = scores.gather(-1, selected_index) + shifted = (scores - selected).scatter(-1, selected_index, 0.0) + others = shifted.scatter(-1, selected_index, -torch.inf) log_normalizer = F.softplus(torch.logsumexp(others, dim=-1, keepdim=True)) return shifted, log_normalizer diff --git a/src/sequifier/config/ownership.py b/src/sequifier/config/ownership.py index 628abc41..24dc8082 100644 --- a/src/sequifier/config/ownership.py +++ b/src/sequifier/config/ownership.py @@ -37,7 +37,7 @@ class PrecisionSettings(Settings): class CompilationSettings(Settings): - torch_compile: Literal["outer", "inner", "none"] = "none" + torch_compile: Literal["outer", "inner", "none"] = "outer" class DataLoadingSettings(Settings): diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index cdb60297..ecd1cbe1 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -388,7 +388,7 @@ def flush_metrics( and phase.evaluation.before_phase and phase_index not in state.initial_evaluated_phases ): - initial = self._evaluate( + self._evaluate( run, self._evaluation_context( run, @@ -400,7 +400,6 @@ def flush_metrics( batches_total=1, ), ) - self._update_monitor(run, initial, phase, count_patience=False) state.initial_evaluated_phases.append(phase_index) for dataset in run.datasets.values(): for part in dataset.parts.values(): diff --git a/src/sequifier/training/loss.py b/src/sequifier/training/loss.py index 555fd699..bf8d8171 100644 --- a/src/sequifier/training/loss.py +++ b/src/sequifier/training/loss.py @@ -196,15 +196,11 @@ def calculate( ) criterion = dataset.criteria[target] if isinstance(criterion, AccurateCrossEntropyLoss): - # Replace masked rows before evaluating the loss so + # Exclude masked rows before evaluating the loss so # their logits cannot enter the backward graph. - safe_logits = torch.where( - flat_mask[:, None], flat_logits, 0.0 - ) - safe_label = torch.where(flat_mask, label, 0) component_sums.append( - torch.where( - flat_mask, criterion(safe_logits, safe_label), 0.0 + criterion( + flat_logits[flat_mask], label[flat_mask] ).sum() ) else: @@ -283,10 +279,8 @@ def calculate( ) sums[target] = -distribution.log_prob(observed).sum() elif isinstance(criterion, AccurateCrossEntropyLoss): - safe_logits = torch.where(flat_mask[:, None], logits_for_loss, 0.0) - safe_target = torch.where(flat_mask, target_for_loss, 0) - sums[target] = torch.where( - flat_mask, criterion(safe_logits, safe_target), 0.0 + sums[target] = criterion( + logits_for_loss[flat_mask], target_for_loss[flat_mask] ).sum() else: raw = criterion(logits_for_loss, target_for_loss) From a72ba2f3c8bc65b80c3ab7ba18903932e42e4103 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Fri, 13 Nov 2026 16:40:28 +0100 Subject: [PATCH 36/71] continue best model across phases --- documentation/configs/train.md | 17 +++- src/sequifier/config/phase_controls.py | 4 +- src/sequifier/config/policy_plan.py | 10 +- src/sequifier/config/train_config.py | 1 + src/sequifier/evaluation/compatibility.py | 108 ++++++++++++++++++++++ src/sequifier/training/engine.py | 8 +- src/sequifier/training/runtime.py | 5 +- 7 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 src/sequifier/evaluation/compatibility.py diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 8d0a4733..77e02038 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -722,11 +722,22 @@ uses `evaluation.cadence: {epochs: N}` for supervised phases and `checkpoints.cadence` with the same explicit units. Final progress always evaluates when enabled and checkpoints. -Evaluation owns `before_phase`, `on_snapshot` and `batch_size`. Model selection +Evaluation owns `before_phase`, `on_snapshot` and `batch_size`. For supervised +phases, an omitted or null evaluation `batch_size` inherits the phase training +batch size. An explicit positive value overrides it. Model selection uses `model_selection.handoff: best | last`; stopping uses `stopping: {patience: N, min_delta: 0, on_trigger: end_phase | end_plan}`. -Patience counts scheduled finite observations. Initial/snapshot evaluations can -select best weights without consuming patience. Best handoff, stopping and plateau +Patience counts scheduled finite observations. In supervised training, initial +evaluation is diagnostic and cannot select best weights; snapshot evaluations can +select best weights without consuming patience. Supervised phases preserve best +weights across phase boundaries when their resolved evaluation contracts match. +The contract includes validation sources, model interfaces, loss definitions, +window views, effective evaluation batch size and precision. Optimizer settings, +freezing and training duration do not affect compatibility. BERT phases have +different evaluation-mask seeds and therefore start separate histories. Set +`model_selection.scope: phase` to explicitly start a new history; the default is +`compatible_phases`. Stopping patience remains phase-local. +Best handoff, stopping and plateau scheduling require an evaluation monitor. See [ownership](ownership.md#phase-settings). Frozen normalization is an interface input contract or inherited artifact fact. diff --git a/src/sequifier/config/phase_controls.py b/src/sequifier/config/phase_controls.py index a6fa1e26..805983e3 100644 --- a/src/sequifier/config/phase_controls.py +++ b/src/sequifier/config/phase_controls.py @@ -24,7 +24,6 @@ class EvaluationTriggers(ConfigModel): cadence: Cadence | None = None before_phase: bool = False on_snapshot: bool = False - batch_size: int = Field(default=1, gt=0) class PhaseCheckpoints(ConfigModel): @@ -37,6 +36,7 @@ class PhaseCheckpoints(ConfigModel): class ModelSelection(ConfigModel): model_config = ConfigDict(extra="forbid") handoff: Literal["best", "last"] = "last" + scope: Literal["compatible_phases", "phase"] = "compatible_phases" class StoppingConfig(ConfigModel): @@ -86,6 +86,8 @@ def checkpoint_interval_iterations(self): @model_validator(mode="after") def validate_lifecycle(self): + if self.algorithm != "supervised" and self.model_selection.scope == "phase": + raise ValueError("model_selection.scope is a supervised phase setting") clock = "epochs" if self.algorithm == "supervised" else "iterations" for label, cadence in ( ("checkpoints", self.checkpoints.cadence), diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 955a42bf..2bab3753 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -32,13 +32,17 @@ class DPOEvaluationMonitor(ConfigModel): direction: Literal["min"] = "min" -class OfflineEvaluationConfig(EvaluationTriggers): +class PolicyEvaluationTriggers(EvaluationTriggers): + batch_size: int = Field(default=1, gt=0) + + +class OfflineEvaluationConfig(PolicyEvaluationTriggers): model_config = ConfigDict(extra="forbid") sources: list[TrainingSourceSpecModel] | None = Field(default=None, min_length=1) monitor: OfflineEvaluationMonitor = Field(default_factory=OfflineEvaluationMonitor) -class DPOEvaluationConfig(EvaluationTriggers): +class DPOEvaluationConfig(PolicyEvaluationTriggers): @model_validator(mode="after") def serial_evaluation(self): if self.batch_size != 1: @@ -176,7 +180,7 @@ class OnlineEvaluationMonitor(ConfigModel): direction: Literal["min", "max"] = "max" -class OnlineEvaluationConfig(EvaluationEpisodesConfig, EvaluationTriggers): +class OnlineEvaluationConfig(EvaluationEpisodesConfig, PolicyEvaluationTriggers): @model_validator(mode="after") def serial_evaluation(self): if self.batch_size != 1: diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 6932b21c..9467aebe 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -416,6 +416,7 @@ class EvaluationMonitorSpecModel(ConfigModel): class EvaluationSpecModel(EvaluationTriggers): model_config = ConfigDict(extra="forbid") + batch_size: int | None = Field(default=None, gt=0) sources: list[TrainingSourceSpecModel] = Field(..., min_length=1) monitor: Optional[EvaluationMonitorSpecModel] = None diff --git a/src/sequifier/evaluation/compatibility.py b/src/sequifier/evaluation/compatibility.py new file mode 100644 index 00000000..f36687aa --- /dev/null +++ b/src/sequifier/evaluation/compatibility.py @@ -0,0 +1,108 @@ +"""Resolved scoring contracts for supervised best-model selection.""" + +from pathlib import Path + +from sequifier.fingerprints import fingerprint + + +def evaluation_batch_size(phase): + evaluation = phase.evaluation + if evaluation is not None and evaluation.batch_size is not None: + return evaluation.batch_size + return phase.batch_size + + +def evaluation_fingerprint(config, phase_index): + """Exclude learning mechanics while retaining everything used for scoring. + + Contracts are compared within one run, whose model architecture and execution + precision are fixed. Dataset interfaces and validation bindings are included + explicitly so different routes or held-out sources cannot share a history. + """ + phase = config.training_plan[phase_index] + if phase.algorithm != "supervised" or phase.evaluation_monitor is None: + return None + sources = [] + for source in phase.evaluation_sources: + dataset = config.dataset_training[source.dataset] + learning = phase.learning.datasets.get(source.dataset) + if learning is None: + return None + parts = (source.part,) if source.part is not None else tuple(dataset.parts) + sources.append( + { + "source": source.source, + "interface": dataset.model_interface, + "interface_contract": dataset.interface.model_dump(mode="json"), + "parts": [ + { + "name": name, + "validation_path": str( + Path(dataset.parts[name].validation_data_path).resolve() + ) + if dataset.parts[name].validation_data_path is not None + else None, + "read_format": dataset.parts[name].read_format, + "storage_form": dataset.parts[name].storage_form, + "file_order": dataset.parts[name].file_order, + "metadata": dataset.parts[name].metadata.model_dump( + mode="json" + ), + "image_paths": dataset.parts[name].image_paths, + "preconditioning_image_paths": dataset.parts[ + name + ].preconditioning_image_paths, + } + for name in parts + ], + "criterion": learning.criterion, + "class_weights": learning.class_weights, + "loss_weights": { + target: (learning.loss_weights or {}).get(target, 1.0) + for target in dataset.interface.target_columns + }, + } + ) + runtime = config.runtime_settings + return fingerprint( + { + "model": config.model.model_dump(mode="json"), + "monitor": phase.evaluation_monitor.model_dump(mode="json"), + "sources": sources, + "objective": runtime.training_objective, + "context_length": runtime.context_length, + "target_offset": runtime.target_offset, + "window_stride": phase.learning.window_stride, + "bert_spec": phase.learning.bert_spec.model_dump(mode="json") + if phase.learning.bert_spec is not None + else None, + # EvaluationService seeds corruption by phase, epoch and batch index. + "mask_seed_phase": phase_index + if runtime.training_objective == "bert" + else None, + "next_occurrence_config": phase.learning.next_occurrence_config.model_dump( + mode="json" + ) + if phase.learning.next_occurrence_config is not None + else None, + "batch_size": evaluation_batch_size(phase), + "device": config.device, + "layer_type_dtypes": runtime.layer_type_dtypes, + "layer_autocast": runtime.layer_autocast, + "float32_matmul_precision": runtime.float32_matmul_precision, + "distributed": runtime.distributed, + "data_parallelism": runtime.data_parallelism, + "world_size": runtime.world_size, + "num_workers": runtime.num_workers, + } + ) + + +def compatible_supervised_phases(config, previous_index, current_index): + current = config.training_plan[current_index] + if current.model_selection.scope == "phase": + return False + previous = evaluation_fingerprint(config, previous_index) + return previous is not None and previous == evaluation_fingerprint( + config, current_index + ) diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index ecd1cbe1..80276e0e 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -16,6 +16,7 @@ from sequifier.artifacts.manifests import write_manifest from sequifier.artifacts.run_checkpoint import LoaderState from sequifier.artifacts.state_dict import canonicalize_state_dict +from sequifier.evaluation.compatibility import compatible_supervised_phases from sequifier.evaluation.service import EvaluationContext, EvaluationResult from sequifier.export.service import ExportOptions from sequifier.integration.contexts import ( @@ -362,8 +363,11 @@ def flush_metrics( f"{previous.name}: best handoff has no finite evaluation" ) run.distributed.restore_model_state(run.network, selected) - state.best_validation_loss = float("inf") - state.best_model_state_dict = None + if not compatible_supervised_phases( + config, phase_index - 1, phase_index + ): + state.best_validation_loss = float("inf") + state.best_model_state_dict = None state.epochs_without_improvement = 0 state.stopping_state = {} last_snapshot = time.monotonic() diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index eafa26a6..7139fd35 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -81,12 +81,13 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: return self.loaders[split] phase = self.config.training_plan[self.config.active_phase_index] - evaluation = phase.evaluation + from sequifier.evaluation.compatibility import evaluation_batch_size + dataset_config: Any = replace( dataset_part_view(self.config, self.dataset_name, self.part_name), batch_size=self.batch_size if split == "training" - else (evaluation.batch_size if evaluation is not None else self.batch_size), + else evaluation_batch_size(phase), ) if split != "training" or not _training_plan_uses_part( self.config, self.dataset_name, self.part_name From 60d6441ffd07458d2a6c8a867f7ca2aeca364abe Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Fri, 13 Nov 2026 19:03:16 +0100 Subject: [PATCH 37/71] reinstate prior behaviour --- documentation/configs/train.md | 4 +-- src/sequifier/export/onnx.py | 34 --------------------- src/sequifier/helpers.py | 2 +- src/sequifier/runtime/builder.py | 12 +++++--- src/sequifier/training/engine.py | 8 +---- src/sequifier/training/runtime.py | 11 ++++++- src/sequifier/training/scheduler_budgets.py | 6 +--- 7 files changed, 23 insertions(+), 54 deletions(-) diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 77e02038..34a955ac 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -728,8 +728,8 @@ batch size. An explicit positive value overrides it. Model selection uses `model_selection.handoff: best | last`; stopping uses `stopping: {patience: N, min_delta: 0, on_trigger: end_phase | end_plan}`. Patience counts scheduled finite observations. In supervised training, initial -evaluation is diagnostic and cannot select best weights; snapshot evaluations can -select best weights without consuming patience. Supervised phases preserve best +and snapshot evaluations are diagnostic and cannot select best weights in the +temporary legacy-compatibility implementation. Supervised phases preserve best weights across phase boundaries when their resolved evaluation contracts match. The contract includes validation sources, model interfaces, loss definitions, window views, effective evaluation batch size and precision. Optimizer settings, diff --git a/src/sequifier/export/onnx.py b/src/sequifier/export/onnx.py index b88078a4..212e4b97 100644 --- a/src/sequifier/export/onnx.py +++ b/src/sequifier/export/onnx.py @@ -26,39 +26,6 @@ from sequifier.runtime.random_state import RandomStateManager -class _ExportLayerNorm(nn.LayerNorm): - """Lower centered variance instead of ONNX Runtime's cancellation-prone norm.""" - - def forward(self, value: Tensor) -> Tensor: - if not torch.compiler.is_compiling(): - # Publication parity must still compare against native PyTorch. - return super().forward(value) - axes = tuple(range(-len(self.normalized_shape), 0)) - centered = value - value.mean(dim=axes, keepdim=True) - variance = centered.square().mean(dim=axes, keepdim=True) - result = centered * torch.rsqrt(variance + self.eps) - if self.weight is not None: - result = result * self.weight - if self.bias is not None: - result = result + self.bias - return result - - -def _lower_layer_norms(module: nn.Module) -> None: - for name, child in tuple(module.named_children()): - if type(child) is nn.LayerNorm: - replacement = _ExportLayerNorm( - child.normalized_shape, - eps=child.eps, - elementwise_affine=child.elementwise_affine, - bias=child.bias is not None, - ) - replacement.load_state_dict(child.state_dict()) - setattr(module, name, replacement) - else: - _lower_layer_norms(child) - - class _OnnxWrapper(nn.Module): def __init__(self, model, schema, interface_name): super().__init__() @@ -342,7 +309,6 @@ def export( try: # All graph lowering and dtype conversion happen on a disposable copy. model = copy.deepcopy(model).cpu().float() - _lower_layer_norms(model) for module in model.modules(): if isinstance(module, SelfAttention): module._sequifier_export_attention = True diff --git a/src/sequifier/helpers.py b/src/sequifier/helpers.py index e1b0b5ab..11cf2e3d 100644 --- a/src/sequifier/helpers.py +++ b/src/sequifier/helpers.py @@ -948,8 +948,8 @@ def configure_determinism(seed: int, strict: bool = False) -> None: torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True - torch.use_deterministic_algorithms(strict) if strict: + torch.use_deterministic_algorithms(True, warn_only=True) os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index d775595f..51bced36 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -219,6 +219,7 @@ def build( config, network, state.phase_index ) previous_masks = None + permanent_masks = [] for phase_index, phase in enumerate(config.training_plan): masks = PhaseTrainabilityPlan.resolve( config, network, phase_index @@ -228,6 +229,13 @@ def build( "optimizer continuation requires identical resolved parameter masks" ) previous_masks = masks + permanent_masks.extend(set(selected) for selected in masks.values()) + # Compatibility: avoid building backward graphs for parameters that + # every dataset in every phase leaves frozen. + if permanent_masks: + catalog = ParameterCatalog(network) + for parameter_id in set.intersection(*permanent_masks): + catalog.parameter(parameter_id).requires_grad_(False) compile_before_ddp = ( execution.distributed @@ -282,10 +290,6 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: ) validate_supervised_scheduler_budgets(config, datasets, state.phase_index) - for phase_index in range(len(config.training_plan)): - if phase_index != state.phase_index: - optimization_factory(phase_index) - optimization = optimization_factory(state.phase_index) if loaded is not None: restorer.restore_optimization(loaded, optimization, network, strategy) diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 80276e0e..84263feb 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -646,7 +646,7 @@ def flush_metrics( and phase.evaluation is not None and phase.evaluation.on_snapshot ): - snapshot_evaluation = self._evaluate( + self._evaluate( run, self._evaluation_context( run, @@ -658,12 +658,6 @@ def flush_metrics( batches_total=current_batches_total, ), ) - self._update_monitor( - run, - snapshot_evaluation, - phase, - count_patience=False, - ) run.callable_network.train() state.last_snapshot_batch_step = state.global_batch_step run.checkpoints.save( diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 7139fd35..3a2b0d09 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -722,9 +722,17 @@ def iter_epoch(self, epoch: int) -> Iterator[RuntimeBatch]: def configure_phase(config, network, datasets, phase_index, device): """Apply phase learning intent without replacing loader recovery identities.""" from sequifier.config.train_config import interface_build_view + from sequifier.evaluation.compatibility import evaluation_batch_size from sequifier.objectives import create_objective phase = config.training_plan[phase_index] + previous = config.training_plan[config.active_phase_index] + rebuild_loaders = ( + previous.batch_size != phase.batch_size + or evaluation_batch_size(previous) != evaluation_batch_size(phase) + or previous.learning.window_stride != phase.learning.window_stride + or previous.curriculum != phase.curriculum + ) config.active_phase_index = phase_index config.runtime_settings.bert_spec = phase.learning.bert_spec config.runtime_settings.next_occurrence_config = ( @@ -748,7 +756,8 @@ def configure_phase(config, network, datasets, phase_index, device): interface_build_view(config, dataset.config.interface) ) for part in dataset.parts.values(): - part.factory.loaders.clear() + if rebuild_loaders: + part.factory.loaders.clear() part.factory.batch_size = phase.batch_size diff --git a/src/sequifier/training/scheduler_budgets.py b/src/sequifier/training/scheduler_budgets.py index 303535c0..fedaa82a 100644 --- a/src/sequifier/training/scheduler_budgets.py +++ b/src/sequifier/training/scheduler_budgets.py @@ -84,6 +84,7 @@ def validate_supervised_scheduler_budgets(config, datasets, active_phase_index): if ( role.scheduler.name != "OneCycleLR" or role.scheduler_step_on != "optimizer_step" + or role.scheduler.arguments.get("total_steps") is not None ): continue start = index @@ -101,11 +102,6 @@ def validate_supervised_scheduler_budgets(config, datasets, active_phase_index): config, datasets, selected ) budget = sum(phase_budgets[selected] for selected in range(start, end)) - explicit = role.scheduler.arguments.get("total_steps") - if explicit is not None and explicit != budget: - raise ValueError( - f"{phase.name}: OneCycleLR.total_steps must equal optimizer-update budget {budget}" - ) if budget <= 0: raise ValueError( f"{phase.name}: OneCycleLR requires a positive optimizer-update budget" From e9f42068cf6905bfd3c9f5cf09f04ece9f9808ce Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Fri, 13 Nov 2026 19:35:43 +0100 Subject: [PATCH 38/71] update docs/tests compatible --- documentation/configs/train.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 34a955ac..08a2d90d 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -728,9 +728,11 @@ batch size. An explicit positive value overrides it. Model selection uses `model_selection.handoff: best | last`; stopping uses `stopping: {patience: N, min_delta: 0, on_trigger: end_phase | end_plan}`. Patience counts scheduled finite observations. In supervised training, initial -and snapshot evaluations are diagnostic and cannot select best weights in the -temporary legacy-compatibility implementation. Supervised phases preserve best -weights across phase boundaries when their resolved evaluation contracts match. +and snapshot evaluations are diagnostic: they do not select best weights or +consume stopping patience. The initial evaluation is recorded as complete in +checkpoints, so resuming does not repeat it. Best weights are selected from +scheduled epoch-end evaluations, even if the initial loss was better. Supervised +phases preserve best weights across phase boundaries when their resolved evaluation contracts match. The contract includes validation sources, model interfaces, loss definitions, window views, effective evaluation batch size and precision. Optimizer settings, freezing and training duration do not affect compatibility. BERT phases have @@ -841,12 +843,15 @@ training; the shared optimizer shape does not imply shared scheduler capabilitie For epoch OneCycleLR, the resolved `total_steps` is the phase's epoch count, or the sum of epochs in its contiguous continuation chain. An explicit value must -match. Batch OneCycleLR derives `total_steps`. An explicit value is an assertion. Preflight counts loader -batches using the same sequential/interleaved source scheduler, phase seeds and -accumulation rules as training, including partial updates flushed at dataset -boundaries. The budget sums over a contiguous continuation chain. This calculation -does not consume samples or advance loader RNG. Early stopping and explicitly -skipped updates may consume less than the authored maximum. +match. For batch OneCycleLR, an explicit `total_steps` is used as configured +without checking it against the number of optimizer updates. If omitted, preflight +derives the value by counting loader batches using the same sequential/interleaved +source scheduler, phase seeds and accumulation rules as training, including partial +updates flushed at dataset boundaries. The derived budget sums over a contiguous +continuation chain. This calculation does not consume samples or advance loader +RNG. Early stopping and explicitly skipped updates may consume fewer steps. If an +explicit budget is shorter than the updates performed, the scheduler stops +advancing after its configured number of steps. Policy OneCycleLR derives the maximum optimizer-update budget below. An explicit `total_steps` must match it: From 22a8a79306cd39d776b85d07faf52657f0e76149 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 16 Nov 2026 11:28:29 +0100 Subject: [PATCH 39/71] improve grpo efficiency --- src/sequifier/io/policy_materialization.py | 4 + src/sequifier/model/images.py | 6 +- src/sequifier/rl/collection.py | 6 +- src/sequifier/rl/environment.py | 38 +++- src/sequifier/rl/grpo.py | 24 +++ src/sequifier/rl/history.py | 68 ++++++- tests/test_rl_image_throughput.py | 215 +++++++++++++++++++++ 7 files changed, 348 insertions(+), 13 deletions(-) create mode 100644 tests/test_rl_image_throughput.py diff --git a/src/sequifier/io/policy_materialization.py b/src/sequifier/io/policy_materialization.py index cd22a62e..f8b6d4f6 100644 --- a/src/sequifier/io/policy_materialization.py +++ b/src/sequifier/io/policy_materialization.py @@ -8,6 +8,10 @@ def encode_value(value, column, config, *, force_raw=False): + # Numeric/image payloads have no categorical codec. Avoid recursively + # walking every pixel in every frame of each reconstructed history. + if not config.id_maps.get(column): + return value if isinstance(value, (list, tuple)): return [ encode_value(item, column, config, force_raw=force_raw) for item in value diff --git a/src/sequifier/model/images.py b/src/sequifier/model/images.py index 16ce9352..12d257c5 100644 --- a/src/sequifier/model/images.py +++ b/src/sequifier/model/images.py @@ -106,7 +106,11 @@ def _encode_valid(self, flat: Tensor, valid: Tensor) -> Tensor: def forward(self, images: Tensor, valid: Tensor | None = None) -> Tensor: shape = images.shape[:-3] flat = images.reshape(-1, *images.shape[-3:]) - if self.training and valid is not None and not torch.onnx.is_in_onnx_export(): + if ( + (self.training or self.frozen) + and valid is not None + and not torch.onnx.is_in_onnx_export() + ): return self._encode_valid(flat, valid).reshape(*shape, -1) return self._encode(flat).reshape(*shape, -1) diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index e727df59..a20b5831 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -49,7 +49,7 @@ def refresh(self): self.actor.model_inputs(self.inputs) def reset(self, result, episode_id, desired_return=None): - result = validate_reset(result, self.units) + result = validate_reset(result, self.units, compact_images=True) if self.actor.config.return_conditioned and desired_return is None: desired_return = self.actor.config.initial_desired_return if desired_return is None: @@ -91,7 +91,9 @@ def decide(self, value=None, *, deterministic=False): ) def step(self, decision): - result = validate_step(self.adapter.step(decision.domain_action), self.units) + result = validate_step( + self.adapter.step(decision.domain_action), self.units, compact_images=True + ) self.last_result = result self.episode_return += float(result.reward) self.episode_length += 1 diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py index c9ff0040..269798e3 100644 --- a/src/sequifier/rl/environment.py +++ b/src/sequifier/rl/environment.py @@ -5,7 +5,14 @@ from importlib import import_module from typing import Any, Literal, Mapping, Protocol -from sequifier.io.values import DomainValue, numeric_scalar, portable_fields +import numpy as np + +from sequifier.io.values import ( + DomainValue, + numeric_scalar, + portable_fields, + portable_value, +) @dataclass(frozen=True) @@ -138,19 +145,36 @@ def validate_start_token(token): ) from exc -def validate_reset(result, units): +def _observation_fields(values, compact_images): + if not compact_images: + return portable_fields(values) + return { + key: ( + value.copy() + if isinstance(value, np.ndarray) + and ( + (key.startswith("__image:") and value.dtype == np.uint8) + or (key.startswith("__image_mask:") and value.dtype == np.bool_) + ) + else portable_value(value) + ) + for key, value in values.items() + } + + +def validate_reset(result, units, *, compact_images=False): if not isinstance(result, OnlineResetResult) or dict(result.units) != units: raise ValueError( "online reset requires an initial observation with the frozen units" ) return OnlineResetResult( - portable_fields(result.observation), + _observation_fields(result.observation, compact_images), portable_fields(result.static_inputs), dict(result.units), ) -def validate_step(result, units): +def validate_step(result, units, *, compact_images=False): if not isinstance(result, OnlineStepResult): raise ValueError( "online step must return OnlineStepResult with an immediate scalar reward" @@ -169,7 +193,11 @@ def validate_step(result, units): if result.success is not None and not isinstance(result.success, bool): raise ValueError("adapter success must be boolean when defined") return OnlineStepResult( - portable_fields(result.observation) if result.observation is not None else None, + ( + _observation_fields(result.observation, compact_images) + if result.observation is not None + else None + ), float(result.reward), result.terminated, result.truncated, diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 9558b109..3ce5ac14 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -3,8 +3,10 @@ import statistics from copy import deepcopy from pathlib import Path +from time import perf_counter import torch +from loguru import logger from sequifier.config.policy_plan import GRPOPhaseConfig from sequifier.io.atomic import write_json_atomic @@ -83,6 +85,7 @@ def evaluate(): evaluate() while groups < phase.total_groups and not session.stopping.triggered: count = min(phase.groups_per_iteration, phase.total_groups - groups) + collection_start = perf_counter() buffer, executed = collect_groups( context, adapter, @@ -92,7 +95,12 @@ def evaluate(): env_steps, restart_stream, ) + collection_seconds = perf_counter() - collection_start + update_start = perf_counter() record = update_grpo(session, buffer) + if actor.device.type == "cuda": + torch.cuda.synchronize(actor.device) + update_seconds = perf_counter() - update_start env_steps += executed groups += count session.transaction += 1 @@ -109,8 +117,10 @@ def evaluate(): "trajectories": [t.as_dict() for t in buffer.trajectories], }, ) + evaluation_start = perf_counter() if session.should_evaluate(final): evaluate() + evaluation_seconds = perf_counter() - evaluation_start scores = [t.score for t in buffer.trajectories] record.update( { @@ -130,9 +140,23 @@ def evaluate(): deepcopy(group[0].start_token) for group in buffer.groups ], "buffer_bytes": buffer.estimated_bytes, + "collection_seconds": collection_seconds, + "collection_env_steps_per_second": executed / collection_seconds, + "update_seconds": update_seconds, + "evaluation_seconds": evaluation_seconds, } ) session.metrics.append(record) + logger.info( + "GRPO iteration {}: {} steps, collection {:.2f}s ({:.1f} steps/s), " + "update {:.2f}s, evaluation {:.2f}s", + session.transaction, + executed, + collection_seconds, + executed / collection_seconds, + update_seconds, + evaluation_seconds, + ) session.checkpoint( final, groups=groups, diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py index d68dc207..f62d0cc0 100644 --- a/src/sequifier/rl/history.py +++ b/src/sequifier/rl/history.py @@ -4,8 +4,13 @@ from collections import deque from copy import deepcopy from dataclasses import dataclass +from functools import cached_property + +import numpy as np +import torch from sequifier.io.policy_window import PolicyWindowView +from sequifier.io.values import portable_fields @dataclass(frozen=True) @@ -17,7 +22,7 @@ class PolicyFrame: step_id: int desired_return: float | None = None - def state_dict(self): + def _values(self): return { "observation": dict(self.observation), "previous_action": self.previous_action, @@ -27,6 +32,14 @@ def state_dict(self): "desired_return": self.desired_return, } + def state_dict(self): + # Materialize portable lists only at checkpoint/archive boundaries. + return portable_fields(self._values()) + + @cached_property + def storage_bytes(self): + return 1024 + 4 * _json_payload_bytes(self._values()) + @classmethod def from_state_dict(cls, values): return cls( @@ -39,11 +52,47 @@ def from_state_dict(cls, values): ) +def _json_payload_bytes(value): + """Size of default json.dumps(...).encode('utf-8') without expanding pixels.""" + if isinstance(value, np.ndarray) and value.ndim and value.size: + if value.dtype == np.uint8: + digits = ( + value.size + + np.count_nonzero(value >= 10) + + np.count_nonzero(value >= 100) + ) + elif value.dtype == np.bool_: + digits = 5 * value.size - np.count_nonzero(value) + else: + return _json_payload_bytes(value.tolist()) + # Each list contributes brackets; separators contribute two bytes. + parents = sum(int(np.prod(value.shape[:axis])) for axis in range(1, value.ndim)) + return int(digits + 2 * value.size + 2 * parents) + if isinstance(value, np.ndarray): + return _json_payload_bytes(value.tolist()) + if isinstance(value, dict): + return ( + 2 + + max(0, len(value) - 1) * 2 + + sum( + len(json.dumps({key: None}, allow_nan=False).encode("utf-8")) + - 6 + + _json_payload_bytes(item) + for key, item in value.items() + ) + ) + if isinstance(value, (list, tuple)): + return ( + 2 + + max(0, len(value) - 1) * 2 + + sum(_json_payload_bytes(item) for item in value) + ) + return len(json.dumps(value, allow_nan=False).encode("utf-8")) + + def frame_storage_bytes(frame): """Conservative host-object allowance plus the complete domain payload.""" - return 1024 + 4 * len( - json.dumps(frame.state_dict(), allow_nan=False).encode("utf-8") - ) + return frame.storage_bytes def history_inputs(spec, frames): @@ -94,8 +143,17 @@ def _frame( raise ValueError( "environment payload does not match the model input binding" ) + owned = {} + for key, value in observation.items(): + if key.startswith("__image:") or key.startswith("__image_mask:"): + dtype = torch.uint8 if key.startswith("__image:") else torch.bool + # Preserve the materializer's dtype conversion and take one + # owned compact copy, instead of retaining nested pixel lists. + owned[key] = torch.as_tensor(value, dtype=dtype).cpu().numpy().copy() + else: + owned[key] = deepcopy(value) frame = PolicyFrame( - tuple(deepcopy(observation).items()), + tuple(owned.items()), deepcopy(previous_action), tuple(deepcopy(static).items()), episode_id, diff --git a/tests/test_rl_image_throughput.py b/tests/test_rl_image_throughput.py new file mode 100644 index 00000000..bc29a96d --- /dev/null +++ b/tests/test_rl_image_throughput.py @@ -0,0 +1,215 @@ +"""Verify compact policy histories and padded-image skipping without training.""" + +import json + +import numpy as np +import pytest +import torch +from torch import nn + +from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.io.values import portable_fields +from sequifier.model.execution_schema import ExecutionSchema, InputDescriptor +from sequifier.model.images import ImageEncoder +from sequifier.rl.collection import PolicyCursor, PolicyDecision +from sequifier.rl.environment import ( + OnlineResetResult, + OnlineStepResult, + validate_reset, + validate_step, +) +from sequifier.rl.history import ( + PolicyFrame, + PolicyHistory, + _json_payload_bytes, + frame_storage_bytes, + history_inputs, +) + + +@pytest.fixture(scope="module") +def spec(): + inputs = [ + InputDescriptor(f"feature_{index}", column, "feature", "float32", ("batch", 32)) + for index, column in enumerate( + ("observation_state_x", "observation_state_y", "action_x", "action_y") + ) + ] + inputs += [ + InputDescriptor( + "image_0", "__image:camera", "image", "uint8", ("batch", 32, 1, 3, 224, 224) + ), + InputDescriptor( + "image_mask_0", "__image_mask:camera", "metadata", "bool", ("batch", 32, 1) + ), + InputDescriptor( + "attention_valid_mask", + "attention_valid_mask", + "metadata", + "bool", + ("batch", 32), + ), + ] + return ResolvedPolicyBinding( + actions={ + name: {"type": "real", "scale": {"fixed": 10.0}} + for name in ("action_x", "action_y") + }, + previous_action_inputs={"action_x": "action_x", "action_y": "action_y"}, + initial_inputs={"action_x": 256.0, "action_y": 256.0}, + context_length=32, + interface_name="actor", + observation_columns=[ + "observation_state_x", + "observation_state_y", + "__image:camera", + "__image_mask:camera", + ], + input_schema=ExecutionSchema(tuple(inputs), (), {}, {}).to_dict(), + ) + + +def observation(): + rng = np.random.default_rng(17) + return { + "observation_state_x": 100.0, + "observation_state_y": 200.0, + "__image:camera": rng.integers(0, 256, (1, 3, 224, 224), dtype=np.uint8), + "__image_mask:camera": np.array([True]), + } + + +@pytest.mark.parametrize("dtype", [np.uint8, np.bool_]) +@pytest.mark.parametrize("shape", [(), (0,), (2, 0), (1,), (2, 3), (1, 3, 7, 9)]) +def test_compact_json_size_matches_portable_json(dtype, shape): + rng = np.random.default_rng(12) + array = rng.integers(0, 256, shape, dtype=np.uint8).astype(dtype) + value = {"é": array, "nested": [None, 1.25, "漢字", (False, 1)], 17: {None: True}} + expected = len(json.dumps(portable_fields(value), allow_nan=False).encode("utf-8")) + assert _json_payload_bytes(value) == expected + + +def test_validators_can_own_compact_images_without_changing_portable_default(): + raw = observation() + reset = OnlineResetResult(raw) + step = OnlineStepResult(raw, 1.0) + for validator, result in ((validate_reset, reset), (validate_step, step)): + standard = validator(result, {}) + compact = validator(result, {}, compact_images=True) + assert standard.observation is not None + assert compact.observation is not None + assert isinstance(standard.observation["__image:camera"], list) + assert isinstance(compact.observation["__image:camera"], np.ndarray) + assert portable_fields(compact.observation) == standard.observation + assert not np.shares_memory( + raw["__image:camera"], compact.observation["__image:camera"] + ) + + +def test_history_preserves_tensors_checkpoints_and_accounting(spec): + raw = observation() + history = PolicyHistory(spec) + history.reset(raw, {}, "episode") + owned = dict(history.current.observation)["__image:camera"] + assert isinstance(owned, np.ndarray) + assert not np.shares_memory(owned, raw["__image:camera"]) + frame = history.current + portable = frame.state_dict() + legacy = PolicyFrame.from_state_dict(portable) + assert isinstance(portable["observation"]["__image:camera"], list) + baseline = history_inputs(spec, [legacy]) + compact = history.inputs() + for field in ("inputs", "metadata"): + left, right = getattr(baseline, field), getattr(compact, field) + assert all(torch.equal(left[key], right[key]) for key in left) + expected = 1024 + 4 * len(json.dumps(portable, allow_nan=False).encode("utf-8")) + assert frame_storage_bytes(frame) == expected + # The second accounting pass must use the cached cost, not touch pixels. + assert frame.storage_bytes == expected + raw["__image:camera"].fill(0) + assert np.any(owned) + saved = json.loads(json.dumps(history.state_dict())) + restored = PolicyHistory(spec) + restored.load_state_dict(saved) + assert restored.state_dict()["frames"] == saved["frames"] + assert isinstance(dict(restored.current.observation)["__image:camera"], np.ndarray) + + +def test_compact_history_retains_lagged_actions(spec): + history = PolicyHistory(spec) + history.reset(observation(), {}, "episode") + history.advance({"action_x": 123.0, "action_y": 234.0}, observation()) + inputs = history.inputs() + assert inputs.inputs["action_x"][0, -1] == 123.0 + assert inputs.inputs["action_y"][0, -1] == 234.0 + assert inputs.attention_valid_mask.sum() == 2 + + +def test_collection_cursor_uses_compact_images(spec): + class Actor: + config = spec + device = torch.device("cpu") + + def model_inputs(self, inputs): + return inputs.inputs, inputs.metadata + + class Adapter: + def step(self, actions): + return OnlineStepResult(observation(), 0.5, truncated=True) + + cursor = PolicyCursor(Actor(), Adapter(), {}, 17) + cursor.reset(OnlineResetResult(observation()), "episode") + actions = {"action_x": 123.0, "action_y": 234.0} + result = cursor.step(PolicyDecision(actions, actions, 0.0, 0.0, 0.0)) + assert cursor.ended and result.truncated + assert result.observation is not None + assert isinstance(result.observation["__image:camera"], np.ndarray) + assert isinstance( + dict(cursor.history.current.observation)["__image:camera"], np.ndarray + ) + assert cursor.inputs.inputs["action_x"][0, -1] == 123.0 + + +class CountingBackbone(nn.Module): + def __init__(self): + super().__init__() + self.weight = nn.Parameter(torch.ones(3), requires_grad=False) + self.seen = 0 + + def forward(self, values): + self.seen += values.shape[0] + return values.mean(dim=(-1, -2)) * self.weight + + +def test_frozen_encoder_skips_padding_and_preserves_projection_gradients(): + # Exercise the real encoder methods using a tiny stand-in backbone; no + # pretrained weights, network access, model optimizer, or training run. + encoder = ImageEncoder.__new__(ImageEncoder) + nn.Module.__init__(encoder) + encoder.frozen = True + encoder.backbone = CountingBackbone() + encoder.projection = nn.Linear(3, 2) + encoder.output_dim = 2 + encoder.register_buffer("mean", torch.zeros(1, 3, 1, 1)) + encoder.register_buffer("std", torch.ones(1, 3, 1, 1)) + encoder.eval() + images = ( + torch.arange(6 * 3 * 8 * 8) + .remainder(256) + .to(torch.uint8) + .reshape(2, 3, 1, 3, 8, 8) + ) + valid = torch.tensor([[[False], [False], [True]], [[False], [True], [True]]]) + dense = encoder._encode(images.reshape(-1, 3, 8, 8)).reshape(2, 3, 1, 2) + (dense * valid[..., None]).sum().backward() + gradients = { + name: p.grad.clone() for name, p in encoder.projection.named_parameters() + } + encoder.zero_grad() + encoder.backbone.seen = 0 + sparse = encoder(images, valid) + (sparse * valid[..., None]).sum().backward() + assert encoder.backbone.seen == int(valid.sum()) + torch.testing.assert_close(sparse[valid], dense[valid]) + for name, parameter in encoder.projection.named_parameters(): + torch.testing.assert_close(parameter.grad, gradients[name]) From 2a2b95d15f41021d263f15bc207df87c4fb0d0c4 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 16 Nov 2026 12:44:07 +0100 Subject: [PATCH 40/71] grpo microbatch --- documentation/configs/train.md | 6 ++++++ documentation/policy-learning.md | 7 +++++++ src/sequifier/config/components.py | 1 + src/sequifier/model/images.py | 24 ++++++++++++++++++++++-- src/sequifier/rl/collection.py | 1 - 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 08a2d90d..258dd64b 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -1030,6 +1030,12 @@ the V1 ImageNet weight transforms. ResNet-50 V2 requires `resize_size: 232` on its source; model resolution checks this. `frame_pooling` is `mean` (default) or `attention`; `view_merge` is `concat` (default) or `mean`. +`encoder.microbatch_size` (default `16`) limits the number of frames in each +backbone call independently of policy scoring batches. Valid frames are gathered +one microbatch at a time. This bounds transient backbone memory for frozen +encoders; trainable encoders still retain activations for backward. ONNX export +uses the unsplit encoder path. + Image embeddings may also occupy cells in an existing structured layout. The cell names are part of `feature_layout`; `image_cells` maps those names to image view names. The usual structured processing blocks then combine image and diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index b2a7b97e..7b0432c9 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -150,6 +150,13 @@ old-policy drift separately. The sampled reference penalty is not exact current- are discarded without retry; errors report issued actions. Both online algorithms default to `max_buffer_bytes: 536870912` and reject stale/reused buffers. +Image encoders accept `microbatch_size` (default `16`) independently of GRPO's +`scoring_chunk_size`. Smaller image microbatches reduce peak frozen-backbone +activation memory without changing policy optimizer-step boundaries. GRPO +collection runs sequentially in the training process. Iteration metrics report +collection seconds, collected environment steps per second, update seconds and +evaluation seconds. + Evaluation uses independent seeds/adapters, restores training modes and RNG, and requires `environment`, `seeds`, `max_episode_steps`, `timeout_seconds` and the main-thread POSIX timer. `action_selection` defaults to deterministic. PPO's default diff --git a/src/sequifier/config/components.py b/src/sequifier/config/components.py index c20b4232..1a863a48 100644 --- a/src/sequifier/config/components.py +++ b/src/sequifier/config/components.py @@ -535,6 +535,7 @@ class ImageEncoderConfig(ConfigModel): groups: int = Field(default=1, gt=0) width_per_group: int = Field(default=64, gt=0) replace_stride_with_dilation: tuple[bool, bool, bool] = (False, False, False) + microbatch_size: int = Field(default=16, gt=0) @model_validator(mode="after") def validate_encoder(self): diff --git a/src/sequifier/model/images.py b/src/sequifier/model/images.py index 12d257c5..aae4860c 100644 --- a/src/sequifier/model/images.py +++ b/src/sequifier/model/images.py @@ -27,6 +27,7 @@ def __init__(self, config: Any, output_dim: int, load_pretrained: bool = True): ) from error self.frozen = config.type == "frozen" + self.microbatch_size = config.microbatch_size if self.frozen: weights_type = models.get_model_weights(config.model) try: @@ -82,7 +83,7 @@ def train(self, mode: bool = True): self.backbone.eval() return self - def _encode(self, images: Tensor) -> Tensor: + def _encode_microbatch(self, images: Tensor) -> Tensor: images = images.float() / 255.0 normalized = (images - self.mean) / self.std normalized = normalized.to(dtype=next(self.backbone.parameters()).dtype) @@ -93,13 +94,32 @@ def _encode(self, images: Tensor) -> Tensor: encoded = self.backbone(normalized) return self.projection(encoded.to(dtype=self.projection.weight.dtype)) + def _encode(self, images: Tensor) -> Tensor: + if torch.onnx.is_in_onnx_export(): + return self._encode_microbatch(images) + if images.shape[0] == 0: + return self.projection.weight.new_zeros(0, self.output_dim) + # Bound backbone call sizes independently of policy scoring batches. + return torch.cat( + [ + self._encode_microbatch(chunk) + for chunk in images.split(self.microbatch_size) + ] + ) + @torch.compiler.disable def _encode_valid(self, flat: Tensor, valid: Tensor) -> Tensor: """Skip padded frames while preserving their positions in the output.""" indices = valid.reshape(-1).nonzero().flatten() if indices.numel() == 0: return self.projection.weight.new_zeros(flat.shape[0], self.output_dim) - selected = self._encode(flat.index_select(0, indices)) + # Gather one microbatch at a time, avoiding a second full image tensor. + selected = torch.cat( + [ + self._encode_microbatch(flat.index_select(0, chunk)) + for chunk in indices.split(self.microbatch_size) + ] + ) output = selected.new_zeros(flat.shape[0], self.output_dim) return output.index_copy(0, indices, selected) diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index a20b5831..a00243a9 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -46,7 +46,6 @@ def __init__(self, actor, adapter, units, seed, *, strict_vocabulary=True): def refresh(self): self.inputs = self.history.inputs().to(self.actor.device) - self.actor.model_inputs(self.inputs) def reset(self, result, episode_id, desired_return=None): result = validate_reset(result, self.units, compact_images=True) From 926ca52add30065874cb7a06586e96cec8e02679 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 16 Nov 2026 13:16:22 +0100 Subject: [PATCH 41/71] grpo efficiency --- documentation/configs/train.md | 24 ++ documentation/policy-learning.md | 38 +++ src/sequifier/config/policy_plan.py | 29 +- src/sequifier/io/image_feature_cache.py | 28 ++ src/sequifier/model/frozen_image_cache.py | 175 +++++++++++ src/sequifier/model/images.py | 77 ++++- .../model/prediction_distribution.py | 45 ++- src/sequifier/rl/collection.py | 25 +- src/sequifier/rl/distributions.py | 33 ++- src/sequifier/rl/grpo.py | 43 ++- src/sequifier/rl/grpo_collector.py | 30 +- src/sequifier/rl/history.py | 28 +- src/sequifier/rl/joint_distribution.py | 31 +- src/sequifier/rl/parallel_grpo_collector.py | 274 ++++++++++++++++++ src/sequifier/rl/preparation.py | 6 + src/sequifier/rl/simulator_workers.py | 156 ++++++++++ 16 files changed, 1010 insertions(+), 32 deletions(-) create mode 100644 src/sequifier/io/image_feature_cache.py create mode 100644 src/sequifier/model/frozen_image_cache.py create mode 100644 src/sequifier/rl/parallel_grpo_collector.py create mode 100644 src/sequifier/rl/simulator_workers.py diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 258dd64b..2a3b92fc 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -1036,6 +1036,30 @@ one microbatch at a time. This bounds transient backbone memory for frozen encoders; trainable encoders still retain activations for backward. ONNX export uses the unsplit encoder path. +`training_plan.phases[].image_feature_cache_bytes` defaults to `0` (disabled). +A positive value opts frozen image encoders into GRPO phase-scoped feature caching +on the model device. This is a runtime option, not a portable-model setting. +The limit counts retained feature tensor storage per distinct cache, in addition +to rollout buffers, model activations, allocator reservations and Python metadata. +The trainable projection is always recomputed with autograd enabled. Trainable +backbones always bypass caching. Outside GRPO's cache scope, encoding follows +the ordinary microbatched path. + +Owned policy frames receive process-local identities only while caching is active; +no pixels are hashed and identities are never written into model artifacts or +checkpoints. Entries are discarded when encoder tensor versions, normalization, +device or dtype change. Inference tensors without version counters bypass caching. +External code that mutates frozen weights using `.data` must call +`encoder.clear_feature_cache()`; ordinary weight loading/in-place updates are +tracked automatically. Entries are released after each iteration and on phase +exit; consumed GRPO trajectories do not keep unused features resident. Actor and +reference share entries only when their frozen encoder states, devices, dtypes, +normalization and budgets match exactly; either owner detaches on invalidation. + +Caching avoids repeated CNN computation, but currently still materializes and +transfers raw image windows. Its benefit is workload-dependent. Keep the default +disabled path for workloads without repeated frames or without VRAM headroom. + Image embeddings may also occupy cells in an existing structured layout. The cell names are part of `feature_layout`; `image_cells` maps those names to image view names. The usual structured processing blocks then combine image and diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 7b0432c9..df5109ca 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -157,6 +157,44 @@ collection runs sequentially in the training process. Iteration metrics report collection seconds, collected environment steps per second, update seconds and evaluation seconds. +GRPO defaults to `simulator_workers: 1`, preserving the in-process sequential +collector. Setting a larger value explicitly enables persistent spawned simulator +processes and batched actor/reference encoding. Active workers are capped at +`groups_per_iteration * group_size`; `rollout_inference_batch_size` (default `32`) +caps each inference call. `worker_threads` (default `1`) bounds PyTorch intra-op +threads per simulator, and `worker_timeout_seconds` (default `120`) sets response +deadlines from dispatch. Worker factories/configurations must be importable and +picklable; custom Python entry points require a `__main__` guard. + +An adapter must explicitly declare `supports_parallel_group_start = True` before +workers can be enabled. This promises that a portable group token restores the +same scenario in an independent adapter instance and that `create_group_start` +depends on its seed/configuration, not the live trajectory. Existing adapters +remain valid with one worker. Construction/worker errors abort instead of silently +falling back or retrying issued actions. Pending groups are never checkpointed. +Pools close on all exit paths; unresponsive processes are terminated after a +bounded cleanup period. + +Every sample retains its independent action and environment seed streams. The +batched path preserves entropy-estimator RNG draws consumed by the sequential +path, even though GRPO does not use entropy. All samples use a single actor +version and all complete groups are sealed before optimization. Group ordering, +trajectory weighting, optimizer boundaries and old-score checks are unchanged. +Floating-point results may vary with inference batch shape; cross-device or +cross-configuration bitwise equivalence is not promised. + +Frozen image feature caching is independently opt-in through +`image_feature_cache_bytes`; see the training reference. Caches are transient, +memory-bounded and scoped to a phase. Metrics include actual simulator count and, +when caching is active, cumulative cache hits/misses and retained feature bytes. +The default disabled options are omitted from serialized contracts to preserve +existing plan/artifact identities. Enabled options are part of the resume plan. + +These options do not guarantee a speedup for every workload or machine. Compare +collection steps/second, update time and peak memory against the default paths. +Increasing groups per iteration changes update frequency; compare equivalent +learning budgets. Additional workers and caches require CPU/GPU memory headroom. + Evaluation uses independent seeds/adapters, restores training modes and RNG, and requires `environment`, `seeds`, `max_episode_steps`, `timeout_seconds` and the main-thread POSIX timer. `action_selection` defaults to deterministic. PPO's default diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 2bab3753..b609cebd 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -2,7 +2,13 @@ from typing import Annotated, ClassVar, Literal -from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic import ( + ConfigDict, + Field, + field_validator, + model_serializer, + model_validator, +) from sequifier.config.identifiers import identifier from sequifier.config.naming import ConfigModel @@ -271,6 +277,12 @@ class GRPOEvaluationConfig(OnlineEvaluationConfig): class GRPOPhaseConfig(BatchedPhaseConfig): scoring_chunk_size: int = Field(default=64, gt=0) + # One preserves the original in-process collector, with no worker startup. + simulator_workers: int = Field(default=1, gt=0) + rollout_inference_batch_size: int = Field(default=32, gt=0) + worker_timeout_seconds: float = Field(default=120.0, gt=0, allow_inf_nan=False) + worker_threads: int = Field(default=1, gt=0) + image_feature_cache_bytes: int = Field(default=0, ge=0) algorithm: Literal["grpo"] environment: OnlineEnvironmentConfig evaluation: GRPOEvaluationConfig @@ -289,6 +301,21 @@ class GRPOPhaseConfig(BatchedPhaseConfig): reward_scale: float = Field(default=1.0, gt=0) archive_directory: str | None = None + @model_serializer(mode="wrap") + def serialize_execution_options(self, handler): + values = handler(self) + # Disabled/default options must not change legacy checkpoint plan hashes. + for name in ( + "simulator_workers", + "rollout_inference_batch_size", + "worker_timeout_seconds", + "worker_threads", + "image_feature_cache_bytes", + ): + if getattr(self, name) == type(self).model_fields[name].default: + values.pop(name, None) + return values + def optimizer_step_budget(self, training_size=None): full, remainder = divmod(self.total_groups, self.groups_per_iteration) size = self.batch_size * self.accumulation_batches diff --git a/src/sequifier/io/image_feature_cache.py b/src/sequifier/io/image_feature_cache.py new file mode 100644 index 00000000..5cecc6d9 --- /dev/null +++ b/src/sequifier/io/image_feature_cache.py @@ -0,0 +1,28 @@ +"""Scoped, transient identities for owned policy frames; never checkpoint state.""" + +from contextlib import contextmanager +from contextvars import ContextVar +from itertools import count + +IMAGE_CACHE_PREFIX = "__image_cache_id:" +_enabled = ContextVar("sequifier_image_feature_cache", default=False) +_identities = count(1) + + +def image_feature_cache_enabled(): + return _enabled.get() + + +def new_image_identity(): + # Frames own their image data. A monotonic process-local identity avoids + # hashing pixels or conflating visually identical but independently owned rows. + return next(_identities) + + +@contextmanager +def image_identity_scope(enabled): + token = _enabled.set(enabled) + try: + yield + finally: + _enabled.reset(token) diff --git a/src/sequifier/model/frozen_image_cache.py b/src/sequifier/model/frozen_image_cache.py new file mode 100644 index 00000000..584c82a0 --- /dev/null +++ b/src/sequifier/model/frozen_image_cache.py @@ -0,0 +1,175 @@ +"""Bounded, phase-scoped frozen-backbone features, outside model state dicts.""" + +from collections import OrderedDict +from contextlib import contextmanager +from dataclasses import dataclass, field + +import torch + +from sequifier.io.image_feature_cache import ( + image_feature_cache_enabled, + image_identity_scope, +) + + +@dataclass +class FrozenImageCache: + budget: int + entries: OrderedDict = field(default_factory=OrderedDict) + retained_bytes: int = 0 + hits: int = 0 + misses: int = 0 + + def __deepcopy__(self, memo): + return type(self)(self.budget) + + def clear(self): + self.entries.clear() + self.retained_bytes = 0 + + def get(self, key): + feature = self.entries.get(key) + if feature is not None: + self.entries.move_to_end(key) + self.hits += 1 + else: + self.misses += 1 + return feature + + def put(self, key, feature): + size = feature.numel() * feature.element_size() + if size > self.budget: + return + previous = self.entries.pop(key, None) + if previous is not None: + self.retained_bytes -= previous.numel() * previous.element_size() + while self.retained_bytes + size > self.budget: + _, evicted = self.entries.popitem(last=False) + self.retained_bytes -= evicted.numel() * evicted.element_size() + # Own a row, not a view retaining an entire microbatch's GPU storage. + with torch.inference_mode(False), torch.no_grad(): + self.entries[key] = feature.detach().clone() + self.retained_bytes += size + + +def encoder_signature(encoder): + """Track ordinary in-place updates, weight loading and device/dtype moves. + + Inference tensors lack version counters, so caching is bypassed for them. + Updating frozen weights via `.data` requires explicit clear_feature_cache(). + """ + if any(module.training for module in encoder.backbone.modules()) or any( + p.requires_grad for p in encoder.backbone.parameters() + ): + return None + tensors = [ + *encoder.backbone.parameters(), + *encoder.backbone.buffers(), + encoder.mean, + encoder.std, + ] + try: + tensor_state = tuple( + (id(value), value.data_ptr(), value._version, value.device, value.dtype) + for value in tensors + ) + device_type = encoder.mean.device.type + autocast = torch.is_autocast_enabled(device_type) + return ( + tensor_state, + autocast, + torch.get_autocast_dtype(device_type) if autocast else None, + torch.get_float32_matmul_precision(), + torch.backends.cudnn.allow_tf32, + ) + except RuntimeError: + return None + + +def _same_backbone(left, right): + a = {**left.backbone.state_dict(), "__mean": left.mean, "__std": left.std} + b = {**right.backbone.state_dict(), "__mean": right.mean, "__std": right.std} + return a.keys() == b.keys() and all( + a[key].shape == b[key].shape + and a[key].dtype == b[key].dtype + and a[key].device == b[key].device + and torch.equal(a[key], b[key]) + for key in a + ) + + +def _active_caches(models): + if not image_feature_cache_enabled(): + return {} + from sequifier.model.images import ImageEncoder + + return { + id(encoder._feature_cache): encoder._feature_cache + for model in models + for encoder in model.modules() + if isinstance(encoder, ImageEncoder) and encoder._feature_cache is not None + } + + +def image_cache_statistics(*models): + caches = _active_caches(models) + if not caches: + return {} + return { + "image_cache_hits": sum(cache.hits for cache in caches.values()), + "image_cache_misses": sum(cache.misses for cache in caches.values()), + "image_cache_retained_bytes": sum( + cache.retained_bytes for cache in caches.values() + ), + } + + +def release_image_cache_entries(*models): + # GRPO never reuses trajectories after their update. Do not retain features + # from previous iterations merely because the phase's LRU budget has space. + for cache in _active_caches(models).values(): + cache.clear() + + +@contextmanager +def frozen_image_cache_scope(actor, reference, budget): + """Install optional GRPO caches, releasing all entries on success or failure. + + Sharing requires matching module names and exact frozen backbone state. + Each owner checks its own tensor versions before reuse and detaches from a + shared cache if its weights, device, dtype or normalization have changed. + """ + from sequifier.model.images import ImageEncoder + + if not budget: + with image_identity_scope(False): + yield + return + saved, active = [], {} + try: + for model in (actor, reference): + for name, encoder in model.named_modules(): + if not isinstance(encoder, ImageEncoder): + continue + if not encoder.frozen: + continue + signature = encoder_signature(encoder) + if signature is None: + continue + saved.append( + (encoder, encoder._feature_cache, encoder._cache_signature) + ) + other = active.get(name) + encoder._feature_cache = ( + other._feature_cache + if other is not None and _same_backbone(encoder, other) + else FrozenImageCache(budget) + ) + encoder._cache_signature = signature + active[name] = encoder + with image_identity_scope(bool(saved)): + yield + finally: + for encoder, cache, signature in reversed(saved): + encoder._feature_cache = cache + encoder._cache_signature = signature diff --git a/src/sequifier/model/images.py b/src/sequifier/model/images.py index aae4860c..fe25c8eb 100644 --- a/src/sequifier/model/images.py +++ b/src/sequifier/model/images.py @@ -6,7 +6,9 @@ import torch from torch import Tensor, nn +from sequifier.io.image_feature_cache import IMAGE_CACHE_PREFIX from sequifier.model.dtypes import cast_floating_to_module_dtype +from sequifier.model.frozen_image_cache import FrozenImageCache, encoder_signature from sequifier.model.ingestions import BaseFeatureIngestion, _AxisPoolBlock @@ -28,6 +30,8 @@ def __init__(self, config: Any, output_dim: int, load_pretrained: bool = True): self.frozen = config.type == "frozen" self.microbatch_size = config.microbatch_size + self._feature_cache: FrozenImageCache | None = None + self._cache_signature = None if self.frozen: weights_type = models.get_model_weights(config.model) try: @@ -83,7 +87,13 @@ def train(self, mode: bool = True): self.backbone.eval() return self - def _encode_microbatch(self, images: Tensor) -> Tensor: + def clear_feature_cache(self): + """Invalidate transient features after an external frozen-weight change.""" + if self._feature_cache is not None: + self._feature_cache = FrozenImageCache(self._feature_cache.budget) + self._cache_signature = None + + def _backbone_microbatch(self, images: Tensor) -> Tensor: images = images.float() / 255.0 normalized = (images - self.mean) / self.std normalized = normalized.to(dtype=next(self.backbone.parameters()).dtype) @@ -92,7 +102,12 @@ def _encode_microbatch(self, images: Tensor) -> Tensor: encoded = self.backbone(normalized) else: encoded = self.backbone(normalized) - return self.projection(encoded.to(dtype=self.projection.weight.dtype)) + return encoded + + def _encode_microbatch(self, images: Tensor) -> Tensor: + return self.projection( + self._backbone_microbatch(images).to(dtype=self.projection.weight.dtype) + ) def _encode(self, images: Tensor) -> Tensor: if torch.onnx.is_in_onnx_export(): @@ -108,11 +123,19 @@ def _encode(self, images: Tensor) -> Tensor: ) @torch.compiler.disable - def _encode_valid(self, flat: Tensor, valid: Tensor) -> Tensor: + def _encode_valid(self, flat: Tensor, valid: Tensor, cache_ids=None) -> Tensor: """Skip padded frames while preserving their positions in the output.""" indices = valid.reshape(-1).nonzero().flatten() if indices.numel() == 0: return self.projection.weight.new_zeros(flat.shape[0], self.output_dim) + if self._feature_cache is not None and cache_ids is not None: + signature = encoder_signature(self) + if signature is not None: + if signature != self._cache_signature: + self.clear_feature_cache() + self._cache_signature = signature + return self._encode_cached(flat, indices, cache_ids) + self.clear_feature_cache() # Gather one microbatch at a time, avoiding a second full image tensor. selected = torch.cat( [ @@ -123,7 +146,47 @@ def _encode_valid(self, flat: Tensor, valid: Tensor) -> Tensor: output = selected.new_zeros(flat.shape[0], self.output_dim) return output.index_copy(0, indices, selected) - def forward(self, images: Tensor, valid: Tensor | None = None) -> Tensor: + def _encode_cached(self, flat, indices, cache_ids): + cache = self._feature_cache + assert cache is not None + if cache_ids.numel() != flat.shape[0]: + raise ValueError("image cache identities must match the frame axes") + identities = cache_ids.reshape(-1).index_select(0, indices).cpu().tolist() + positions = indices.cpu().tolist() + if any(identity <= 0 for identity in identities): + raise ValueError("valid images require positive transient identities") + shape = tuple(flat.shape[1:]) + keys = [(shape, identity) for identity in identities] + features, missing = {}, {} + for position, key in zip(positions, keys): + if key in features or key in missing: + continue + feature = cache.get(key) + if feature is None: + missing[key] = position + else: + features[key] = feature + items = list(missing.items()) + for start in range(0, len(items), self.microbatch_size): + chunk = items[start : start + self.microbatch_size] + rows = torch.tensor([position for _, position in chunk], device=flat.device) + with torch.inference_mode(False), torch.no_grad(): + encoded = self._backbone_microbatch(flat.index_select(0, rows)) + for (key, _), feature in zip(chunk, encoded): + features[key] = feature + cache.put(key, feature) + # Only the frozen backbone is cached. This projection remains in autograd. + selected = self.projection( + torch.stack([features[key] for key in keys]).to( + dtype=self.projection.weight.dtype + ) + ) + output = selected.new_zeros(flat.shape[0], self.output_dim) + return output.index_copy(0, indices, selected) + + def forward( + self, images: Tensor, valid: Tensor | None = None, cache_ids=None + ) -> Tensor: shape = images.shape[:-3] flat = images.reshape(-1, *images.shape[-3:]) if ( @@ -131,7 +194,7 @@ def forward(self, images: Tensor, valid: Tensor | None = None) -> Tensor: and valid is not None and not torch.onnx.is_in_onnx_export() ): - return self._encode_valid(flat, valid).reshape(*shape, -1) + return self._encode_valid(flat, valid, cache_ids).reshape(*shape, -1) return self._encode(flat).reshape(*shape, -1) @@ -162,7 +225,9 @@ def forward( for index, view in enumerate(self.views): images = inputs[f"__image:{view}"] valid = metadata[f"__image_mask:{view}"].bool() - encoded = self.encoder(images, valid) + encoded = self.encoder( + images, valid, metadata.get(IMAGE_CACHE_PREFIX + view) + ) frame_score = self.frame_score if frame_score is None: weights = valid.to(encoded.dtype) diff --git a/src/sequifier/model/prediction_distribution.py b/src/sequifier/model/prediction_distribution.py index e3f09c3e..6e19092a 100644 --- a/src/sequifier/model/prediction_distribution.py +++ b/src/sequifier/model/prediction_distribution.py @@ -83,20 +83,45 @@ def mode(self): @torch.no_grad() def sample(self, generator=None): - device = generator.device if generator is not None else self.location.device - noise = torch.randn( - self.location.shape, - device=device, - dtype=self.dtype, - generator=generator, - ).to(self.location.device) + """Sample a tensor, optionally with an independent generator per batch row. + + Only random draws are split by environment. Transforms and mixture + selection remain batched, and the scalar-generator path is unchanged. + """ + if isinstance(generator, (list, tuple)): + if self.location.ndim == 0 or len(generator) != self.location.shape[0]: + raise ValueError("one action RNG is required per leading batch row") + if any(not isinstance(rng, torch.Generator) for rng in generator): + raise TypeError("batched action RNGs must be torch.Generator instances") + + def draw(random, **options): + if isinstance(generator, (list, tuple)): + if not generator: + return torch.empty( + self.location.shape, device=self.location.device, **options + ) + return torch.cat( + [ + random( + self.location[index : index + 1].shape, + device=rng.device, + generator=rng, + **options, + ).to(self.location.device) + for index, rng in enumerate(generator) + ] + ) + device = generator.device if generator is not None else self.location.device + return random( + self.location.shape, device=device, generator=generator, **options + ).to(self.location.device) + + noise = draw(torch.randn, dtype=self.dtype) self.sampled_latent = self.location + self.scale * noise value = self._transform(self.sampled_latent) self.sampled_component = torch.full_like(value, -1, dtype=torch.long) if self.field.atoms: - choice = torch.rand( - self.location.shape, device=device, generator=generator - ).to(value.device) + choice = draw(torch.rand) cumulative = 0.0 for index, (atom, probability) in enumerate( sorted(self.field.atoms.items()) diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index a00243a9..d7c8bc01 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -36,8 +36,18 @@ class PolicyCursor: account for issued actions before handing control to the environment. """ - def __init__(self, actor, adapter, units, seed, *, strict_vocabulary=True): + def __init__( + self, + actor, + adapter, + units, + seed, + *, + strict_vocabulary=True, + defer_device_transfer=False, + ): self.actor, self.adapter, self.units = actor, adapter, units + self.defer_device_transfer = defer_device_transfer self.history = PolicyHistory(actor.config, strict_vocabulary=strict_vocabulary) self.generator = action_generator(actor.device, seed) self.desired_return = None @@ -45,7 +55,10 @@ def __init__(self, actor, adapter, units, seed, *, strict_vocabulary=True): self.last_result = None def refresh(self): - self.inputs = self.history.inputs().to(self.actor.device) + inputs = self.history.inputs() + self.inputs = ( + inputs if self.defer_device_transfer else inputs.to(self.actor.device) + ) def reset(self, result, episode_id, desired_return=None): result = validate_reset(result, self.units, compact_images=True) @@ -90,9 +103,11 @@ def decide(self, value=None, *, deterministic=False): ) def step(self, decision): - result = validate_step( - self.adapter.step(decision.domain_action), self.units, compact_images=True - ) + return self.accept_step(decision, self.adapter.step(decision.domain_action)) + + def accept_step(self, decision, result): + """Validate and commit exactly one simulator response.""" + result = validate_step(result, self.units, compact_images=True) self.last_result = result self.episode_return += float(result.reward) self.episode_length += 1 diff --git a/src/sequifier/rl/distributions.py b/src/sequifier/rl/distributions.py index 1fbad8a3..23df7e77 100644 --- a/src/sequifier/rl/distributions.py +++ b/src/sequifier/rl/distributions.py @@ -26,8 +26,39 @@ def mode(self) -> Tensor: def select(self, deterministic: bool, generator: torch.Generator) -> Tensor: return self.mode() if deterministic else self.sample(generator) - def sample(self, generator: torch.Generator | None = None) -> Tensor: + def sample( + self, + generator: torch.Generator + | list[torch.Generator] + | tuple[torch.Generator, ...] + | None = None, + ) -> Tensor: + """Keep probability computation batched, with optional per-row RNGs.""" probabilities = self.distribution.probs + if isinstance(generator, (list, tuple)): + if len(generator) != probabilities.shape[0]: + raise ValueError("one action RNG is required per leading batch row") + if any(not isinstance(rng, torch.Generator) for rng in generator): + raise TypeError("batched action RNGs must be torch.Generator instances") + if not generator: + return torch.empty( + probabilities.shape[:-1], + device=probabilities.device, + dtype=torch.int64, + ) + sampled = torch.cat( + [ + torch.multinomial( + probabilities[index : index + 1] + .to(rng.device) + .reshape(-1, probabilities.shape[-1]), + 1, + generator=rng, + ).to(probabilities.device) + for index, rng in enumerate(generator) + ] + ) + return sampled.reshape(probabilities.shape[:-1]) # MPS has no device-specific Generator; sample on its generator's device. sampling_device = ( generator.device if generator is not None else probabilities.device diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 3ce5ac14..1b38cb0e 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -1,6 +1,7 @@ """Actor-only GRPO runtime; checkpoints contain no live or reusable group.""" import statistics +from contextlib import ExitStack from copy import deepcopy from pathlib import Path from time import perf_counter @@ -10,6 +11,11 @@ from sequifier.config.policy_plan import GRPOPhaseConfig from sequifier.io.atomic import write_json_atomic +from sequifier.model.frozen_image_cache import ( + frozen_image_cache_scope, + image_cache_statistics, + release_image_cache_entries, +) from sequifier.rl.environment import ( adapter_identity, load_online_adapter, @@ -49,6 +55,8 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): raise ValueError("GRPO environment clock exceeds authored bounds") actor.grpo_policy_version = session.transaction adapter = load_online_adapter(phase.environment) + resources = ExitStack() + workers = None try: require_group_start(adapter, exact=context.run.resume.mode == "exact") identity = adapter_identity(adapter) @@ -66,6 +74,31 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): ) session.restore_random() + # Register cleanup before collection or evaluation can fail. Both + # optimizations are disabled for configurations that omit their opt-ins. + resources.enter_context( + frozen_image_cache_scope( + actor, context.reference, phase.image_feature_cache_bytes + ) + ) + if phase.simulator_workers > 1 and groups < phase.total_groups: + from sequifier.rl.simulator_workers import ( + SimulatorWorkers, + require_parallel_group_start, + ) + + require_parallel_group_start(adapter) + workers = SimulatorWorkers( + phase.environment, + min( + phase.simulator_workers, + phase.groups_per_iteration * phase.group_size, + ), + phase.worker_timeout_seconds, + phase.worker_threads, + identity, + ) + resources.callback(workers.close) def evaluate(): result = evaluate_policy(actor, phase.evaluation, forbidden_adapter=adapter) @@ -94,6 +127,7 @@ def evaluate(): session.transaction, env_steps, restart_stream, + workers=workers, ) collection_seconds = perf_counter() - collection_start update_start = perf_counter() @@ -140,12 +174,16 @@ def evaluate(): deepcopy(group[0].start_token) for group in buffer.groups ], "buffer_bytes": buffer.estimated_bytes, + "simulator_workers": len(workers) if workers is not None else 1, + "rollout_inference_batch_size": phase.rollout_inference_batch_size, "collection_seconds": collection_seconds, "collection_env_steps_per_second": executed / collection_seconds, "update_seconds": update_seconds, "evaluation_seconds": evaluation_seconds, } ) + record.update(image_cache_statistics(actor, context.reference)) + release_image_cache_entries(actor, context.reference) session.metrics.append(record) logger.info( "GRPO iteration {}: {} steps, collection {:.2f}s ({:.1f} steps/s), " @@ -167,7 +205,10 @@ def evaluate(): restart_stream=restart_stream, ) finally: - adapter.close() + try: + resources.close() + finally: + adapter.close() return session.finish( { "optimizer_steps": session.steps["actor"], diff --git a/src/sequifier/rl/grpo_collector.py b/src/sequifier/rl/grpo_collector.py index b007ef4c..b99dae50 100644 --- a/src/sequifier/rl/grpo_collector.py +++ b/src/sequifier/rl/grpo_collector.py @@ -7,6 +7,7 @@ from sequifier.fingerprints import fingerprint from sequifier.io.actions import action_batch +from sequifier.io.image_feature_cache import IMAGE_CACHE_PREFIX from sequifier.rl.collection import PolicyCursor, stream_seed from sequifier.rl.environment import require_group_start, validate_start_token from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer @@ -23,8 +24,29 @@ def __init__(self, message, executed_actions): @torch.no_grad() def collect_groups( - context, adapter, start_group, count, generation, env_steps, restart_stream=0 + context, + adapter, + start_group, + count, + generation, + env_steps, + restart_stream=0, + *, + workers=None, ): + if workers is not None: + from sequifier.rl.parallel_grpo_collector import collect_parallel_groups + + return collect_parallel_groups( + context, + adapter, + workers, + start_group, + count, + generation, + env_steps, + restart_stream, + ) actor, reference, phase = context.actor.eval(), context.reference, context.phase require_group_start(adapter) bound = count * phase.group_size * phase.max_steps @@ -70,7 +92,11 @@ def collect_groups( key = fingerprint( { "features": {k: v.tolist() for k, v in inputs.inputs.items()}, - "metadata": {k: v.tolist() for k, v in inputs.metadata.items()}, + "metadata": { + k: v.tolist() + for k, v in inputs.metadata.items() + if not k.startswith(IMAGE_CACHE_PREFIX) + }, "units": dict(cursor.units), } ) diff --git a/src/sequifier/rl/history.py b/src/sequifier/rl/history.py index f62d0cc0..b002b0f4 100644 --- a/src/sequifier/rl/history.py +++ b/src/sequifier/rl/history.py @@ -9,6 +9,11 @@ import numpy as np import torch +from sequifier.io.image_feature_cache import ( + IMAGE_CACHE_PREFIX, + image_feature_cache_enabled, + new_image_identity, +) from sequifier.io.policy_window import PolicyWindowView from sequifier.io.values import portable_fields @@ -22,6 +27,17 @@ class PolicyFrame: step_id: int desired_return: float | None = None + @cached_property + def image_identities(self): + # Transient IDs are deliberately excluded from state_dict and archives. + return { + key.removeprefix("__image:"): torch.tensor( + [new_image_identity() for _ in value], dtype=torch.int64 + ) + for key, value in self.observation + if key.startswith("__image:") + } + def _values(self): return { "observation": dict(self.observation), @@ -97,12 +113,22 @@ def frame_storage_bytes(frame): def history_inputs(spec, frames): frames = list(frames) - return PolicyWindowView(spec).inputs( + inputs = PolicyWindowView(spec).inputs( [dict(frame.observation) for frame in frames], [frame.previous_action for frame in frames], [frame.desired_return for frame in frames] if spec.return_conditioned else None, dict(frames[-1].static_inputs), ) + if image_feature_cache_enabled(): + for view in frames[-1].image_identities: + identities = torch.stack([frame.image_identities[view] for frame in frames]) + padding = identities.new_zeros( + spec.context_length - len(frames), *identities.shape[1:] + ) + inputs.metadata[IMAGE_CACHE_PREFIX + view] = torch.cat( + [padding, identities] + )[None] + return inputs class PolicyHistory: diff --git a/src/sequifier/rl/joint_distribution.py b/src/sequifier/rl/joint_distribution.py index 2527a47a..aa72430e 100644 --- a/src/sequifier/rl/joint_distribution.py +++ b/src/sequifier/rl/joint_distribution.py @@ -18,14 +18,35 @@ class JointActionDistribution: """The model owns all weights; this object owns one encoded input window.""" - def __init__(self, actor, inputs): + def __init__(self, actor, inputs, *, _encoded=None): self.actor = actor - self.features, self.metadata = actor.model_inputs(inputs) - self.representation = actor.network.encode( - self.features, self.metadata, interface_name=actor.interface_name - ) + if _encoded is None: + self.features, self.metadata = actor.model_inputs(inputs) + self.representation = actor.network.encode( + self.features, self.metadata, interface_name=actor.interface_name + ) + else: + self.features, self.metadata, self.representation = _encoded self.batch, self.time = self.representation.shape[:2] + def row(self, index): + """View one encoded batch row without re-encoding or sharing action RNG. + + Only the representation and immutable input views are shared. Selection, + autoregressive decoding, likelihood and entropy retain their normal path. + """ + if not 0 <= index < self.batch: + raise IndexError("joint-distribution row is outside the batch") + return type(self)( + self.actor, + None, + _encoded=( + {key: value[index : index + 1] for key, value in self.features.items()}, + {key: value[index : index + 1] for key, value in self.metadata.items()}, + self.representation[index : index + 1], + ), + ) + def _teachers(self, actions): interface = self.actor.interface teachers = {} diff --git a/src/sequifier/rl/parallel_grpo_collector.py b/src/sequifier/rl/parallel_grpo_collector.py new file mode 100644 index 00000000..77d32f56 --- /dev/null +++ b/src/sequifier/rl/parallel_grpo_collector.py @@ -0,0 +1,274 @@ +"""Explicitly enabled synchronous GRPO collection across independent simulators.""" + +import math +from collections import deque +from dataclasses import dataclass, field, replace + +import torch + +from sequifier.fingerprints import fingerprint +from sequifier.io.actions import action_record, decode_action +from sequifier.io.batch import collate_model_inputs +from sequifier.io.image_feature_cache import IMAGE_CACHE_PREFIX +from sequifier.rl.collection import PolicyCursor, PolicyDecision, stream_seed +from sequifier.rl.environment import validate_start_token +from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer +from sequifier.rl.grpo_collector import GroupCollectionError +from sequifier.rl.history import frame_storage_bytes +from sequifier.rl.simulator_workers import require_parallel_group_start + + +@dataclass +class _Trajectory: + group: int + sample: int + token: object + cursor: PolicyCursor + frames: list = field(default_factory=list) + actions: list = field(default_factory=list) + rewards: list = field(default_factory=list) + old_scores: list = field(default_factory=list) + reference_scores: list = field(default_factory=list) + + def seal(self, generation, result): + score = float(torch.tensor(self.rewards, dtype=torch.float32).sum()) + if not math.isfinite(score): + raise ValueError("non-finite FP32 GRPO trajectory score") + reason = ( + "terminated" + if result.terminated + else "truncated" + if result.truncated + else "max_steps" + ) + return GRPOTrajectory( + group_index=self.group, + sample_index=self.sample, + start_token=self.token, + episode_id=self.cursor.history.current.episode_id, + frames=tuple(self.frames), + context_length=self.cursor.actor.config.context_length, + actions=tuple(self.actions), + rewards=tuple(self.rewards), + old_log_probabilities=tuple(self.old_scores), + reference_log_probabilities=tuple(self.reference_scores), + cutoff_reason=reason, + score=score, + advantage=0.0, + generation=generation, + ) + + +@torch.no_grad() +def _decide(actor, reference, cursors): + inputs = collate_model_inputs([cursor.inputs for cursor in cursors]).to( + actor.device + ) + distribution = actor.distribution(inputs) + generators = [cursor.generator for cursor in cursors] + # Decode and validate distributions across the full environment batch. + # Only random draws use individual generators; no per-row decoder calls. + actions = distribution.select(False, generators) + old_scores = distribution.log_prob(actions) + if not distribution.entropy_is_exact: + # The sequential collector samples another action to estimate entropy. + # Preserve those per-environment RNG draws without calculating the + # unused entropy likelihood or surrogate. Exact entropy consumes no RNG. + distribution.sample(generators) + scores = ( + torch.stack([old_scores, reference.evaluate_actions(inputs, actions)], dim=1) + .cpu() + .tolist() + ) + host_actions = {name: value.cpu() for name, value in actions.items()} + decisions = [] + for index, (old, ref) in enumerate(scores): + if not math.isfinite(old) or not math.isfinite(ref): + raise ValueError("non-finite GRPO policy/reference action score") + action = action_record(host_actions, index) + decisions.append( + ( + PolicyDecision( + decode_action(action, actor.config), action, old, 0.0, 0.0 + ), + ref, + ) + ) + return decisions + + +def _initial_history_identity(cursor): + # reset() has exactly one valid temporal position; all earlier positions + # are canonical zero padding. Compare its encoded values and full shapes + # without expanding context_length copies of padded camera pixels. + inputs = cursor.inputs + static = set(cursor.actor.config.static_columns) + return fingerprint( + { + "features": { + key: { + "shape": list(value.shape), + "values": (value if key in static else value[:, -1:]).tolist(), + } + for key, value in inputs.inputs.items() + }, + "metadata": { + key: value.tolist() + for key, value in inputs.metadata.items() + if not key.startswith(IMAGE_CACHE_PREFIX) + }, + "units": dict(cursor.units), + } + ) + + +@torch.no_grad() +def collect_parallel_groups( + context, + adapter, + workers, + start_group, + count, + generation, + env_steps, + restart_stream, +): + actor, reference, phase = ( + context.actor.eval(), + context.reference.eval(), + context.phase, + ) + require_parallel_group_start(adapter) + bound = count * phase.group_size * phase.max_steps + if phase.max_env_steps is not None and env_steps + bound > phase.max_env_steps: + raise ValueError("complete GRPO groups exceed max_env_steps") + if ( + bound * (2048 + 128 * len(actor.interface.input_columns)) + > phase.max_buffer_bytes + ): + raise ValueError("complete GRPO groups exceed the minimum buffer estimate") + seed = stream_seed( + context.run.seed, f"grpo-phase-{context.index}-restart-{restart_stream}", 0 + ) + used, executed = 0, 0 + pending, active, completed, initial = deque(), {}, {}, {} + + def account(size): + nonlocal used + used += size + if used > phase.max_buffer_bytes: + raise ValueError("GRPO collection exceeded max_buffer_bytes") + + try: + from sequifier.rl.history import _json_payload_bytes + + for group in range(start_group, start_group + count): + token = validate_start_token( + adapter.create_group_start(stream_seed(seed, "group-start", group)) + ) + account(_json_payload_bytes(token) * phase.group_size + 1024) + completed[group] = [] + pending.extend((group, sample, token) for sample in range(phase.group_size)) + while pending or active: + resetting = [] + for slot in range(len(workers)): + if slot in active or not pending: + continue + group, sample, token = pending.popleft() + cursor = PolicyCursor( + actor, + adapter, + context.run.units, + stream_seed(seed, f"group-{group}-actions", sample), + defer_device_transfer=True, + ) + active[slot] = _Trajectory(group, sample, token, cursor) + workers.send( + slot, + "reset_group", + validate_start_token(token), + stream_seed(seed, f"group-{group}-environment", sample), + ) + resetting.append(slot) + for slot in resetting: + state = active[slot] + state.cursor.reset( + workers.receive(slot), + f"group-{state.group}-sample-{state.sample}", + ) + frame = state.cursor.history.current + identity = _initial_history_identity(state.cursor) + if state.group in initial and initial[state.group] != identity: + raise ValueError( + "GRPO group members have different initial histories" + ) + initial[state.group] = identity + # Include all live history before any environment step is issued. + account(frame_storage_bytes(frame) + 1024) + account( + sum( + value.numel() * value.element_size() + for value in ( + *state.cursor.inputs.inputs.values(), + *state.cursor.inputs.metadata.values(), + ) + ) + ) + decisions = {} + slots = sorted(active) + for start in range(0, len(slots), phase.rollout_inference_batch_size): + chunk = slots[start : start + phase.rollout_inference_batch_size] + scored = _decide(actor, reference, [active[s].cursor for s in chunk]) + decisions.update(zip(chunk, scored)) + for decision, _ in decisions.values(): + account(4 * _json_payload_bytes(decision.action)) + for slot, (decision, _) in decisions.items(): + # Count attempted actions conservatively before sending: a failed + # send must never hide a potentially executed simulator action. + executed += 1 + workers.send(slot, "step", decision.domain_action) + for slot, (decision, ref) in decisions.items(): + state = active[slot] + state.frames.append(state.cursor.history.current) + result = state.cursor.accept_step(decision, workers.receive(slot)) + reward = float(result.reward) * phase.reward_scale + if not math.isfinite(reward) or not torch.isfinite( + torch.tensor(reward, dtype=torch.float32) + ): + raise ValueError("GRPO reward exceeds finite FP32 range") + state.actions.append(decision.action) + state.rewards.append(reward) + state.old_scores.append(decision.log_probability) + state.reference_scores.append(ref) + # Account the successor retained by a live cursor, even if the + # episode ends or the collector's finite horizon is reached. + if result.observation is not None: + account(frame_storage_bytes(state.cursor.history.current) + 1024) + if ( + result.terminated + or result.truncated + or len(state.actions) == phase.max_steps + ): + completed[state.group].append(state.seal(generation, result)) + del active[slot] + groups = [] + for trajectories in completed.values(): + if len(trajectories) != phase.group_size: + raise ValueError("GRPO cannot seal an incomplete group") + trajectories.sort(key=lambda value: value.sample_index) + scores = torch.tensor([t.score for t in trajectories], dtype=torch.float32) + mean, spread = scores.mean(), scores.std(unbiased=False) + advantages = (scores - mean) / (spread + phase.advantage_epsilon) + if not torch.isfinite(advantages).all(): + raise ValueError("non-finite GRPO group advantages") + groups.append( + tuple( + replace(t, advantage=float(a)) + for t, a in zip(trajectories, advantages) + ) + ) + return SealedGroupBuffer( + groups, generation, phase.group_size, context.reference_identity, used + ), executed + except Exception as exc: + raise GroupCollectionError(str(exc), executed) from exc diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 893003a0..7261f454 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -148,6 +148,12 @@ def preflight_environments(config): require_group_start( adapter, exact=config.resume.mode == "exact" ) + if phase.simulator_workers > 1: + from sequifier.rl.simulator_workers import ( + require_parallel_group_start, + ) + + require_parallel_group_start(adapter) elif config.resume.mode == "exact": require_exact_snapshot(adapter) identities[f"{index}.{role}"] = adapter_identity(adapter) diff --git a/src/sequifier/rl/simulator_workers.py b/src/sequifier/rl/simulator_workers.py new file mode 100644 index 00000000..c86a29c8 --- /dev/null +++ b/src/sequifier/rl/simulator_workers.py @@ -0,0 +1,156 @@ +"""Opt-in spawned simulators with explicit requests, deadlines and bounded cleanup.""" + +import multiprocessing +import time +import traceback + +from sequifier.rl.environment import ( + adapter_identity, + load_online_adapter, + require_group_start, +) + + +def require_parallel_group_start(adapter): + require_group_start(adapter) + if getattr(adapter, "supports_parallel_group_start", False) is not True: + raise ValueError( + "parallel GRPO requires adapter.supports_parallel_group_start = True; " + "its portable group-start tokens must work in independent adapter instances" + ) + + +def _serve(connection, spec, threads): + adapter = None + request = 0 + try: + import torch + + torch.set_num_threads(threads) + adapter = load_online_adapter(spec) + require_parallel_group_start(adapter) + connection.send((request, True, adapter_identity(adapter))) + while True: + request, method, args = connection.recv() + if method == "close": + break + if method not in {"reset_group", "step"}: + raise ValueError(f"unsupported simulator operation: {method}") + connection.send((request, True, getattr(adapter, method)(*args))) + except (EOFError, BrokenPipeError): + pass + except BaseException: + try: + connection.send((request, False, traceback.format_exc()[-32768:])) + except (OSError, EOFError): + pass + finally: + try: + if adapter is not None: + adapter.close() + finally: + connection.close() + + +class SimulatorWorkers: + """One in-flight request per worker; never retries an issued environment action.""" + + def __init__(self, spec, count, timeout, threads, expected_identity): + self.timeout = timeout + self.connections, self.processes = [], [] + self.pending, self.sequence = {}, 0 + self.closed = False + try: + context = multiprocessing.get_context("spawn") + for index in range(count): + parent, child = context.Pipe() + process = context.Process( + target=_serve, + args=(child, spec, threads), + name=f"sequifier-grpo-simulator-{index}", + ) + try: + process.start() + except BaseException: + parent.close() + child.close() + raise + child.close() + self.connections.append(parent) + self.processes.append(process) + self.pending[index] = (0, "startup", time.monotonic() + timeout) + for index in range(count): + if self.receive(index) != expected_identity: + raise ValueError( + f"simulator {index} adapter identity differs from parent" + ) + except BaseException: + self.close() + raise + + def __len__(self): + return len(self.connections) + + def send(self, index, method, *args): + if self.closed or index in self.pending: + raise RuntimeError( + "simulator is closed or already has an in-flight request" + ) + if method not in {"reset_group", "step"}: + raise ValueError(f"unsupported simulator operation: {method}") + self.sequence += 1 + self.pending[index] = (self.sequence, method, time.monotonic() + self.timeout) + try: + self.connections[index].send((self.sequence, method, args)) + except (OSError, EOFError) as exc: + raise RuntimeError( + f"simulator {index} failed while sending {method}" + ) from exc + + def receive(self, index): + if self.closed or index not in self.pending: + raise RuntimeError("simulator has no pending response") + request, method, deadline = self.pending[index] + connection = self.connections[index] + try: + if not connection.poll(max(0.0, deadline - time.monotonic())): + raise TimeoutError(f"simulator {index} timed out during {method}") + received, ok, result = connection.recv() + except (OSError, EOFError) as exc: + raise RuntimeError(f"simulator {index} exited during {method}") from exc + if received != request: + raise RuntimeError(f"simulator {index} returned an out-of-order response") + del self.pending[index] + if not ok: + raise RuntimeError(f"simulator {index} failed during {method}:\n{result}") + return result + + def close(self): + if self.closed: + return + self.closed = True + # Do not queue close behind a blocked, incomplete response. Such workers + # are terminated below; no pending collection is ever checkpointed. + for index, connection in enumerate(self.connections): + if index not in self.pending: + try: + connection.send((-1, "close", ())) + except (OSError, EOFError): + pass + deadline = time.monotonic() + 2.0 + for process in self.processes: + process.join(timeout=max(0.0, deadline - time.monotonic())) + for process in self.processes: + if process.is_alive(): + process.terminate() + deadline = time.monotonic() + 2.0 + for process in self.processes: + process.join(timeout=max(0.0, deadline - time.monotonic())) + for process in self.processes: + if process.is_alive(): + process.kill() + for process in self.processes: + process.join(timeout=0.1) + for connection in self.connections: + connection.close() + self.pending.clear() From df588e8b7c6a739e9aec7132937209411d65bd8b Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Mon, 16 Nov 2026 14:34:06 +0100 Subject: [PATCH 42/71] grpo efficiency --- documentation/policy-learning.md | 12 +++++++----- src/sequifier/rl/collection.py | 4 ++-- src/sequifier/rl/grpo_collector.py | 2 +- src/sequifier/rl/parallel_grpo_collector.py | 5 ----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index df5109ca..ddfaf8c8 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -175,11 +175,13 @@ falling back or retrying issued actions. Pending groups are never checkpointed. Pools close on all exit paths; unresponsive processes are terminated after a bounded cleanup period. -Every sample retains its independent action and environment seed streams. The -batched path preserves entropy-estimator RNG draws consumed by the sequential -path, even though GRPO does not use entropy. All samples use a single actor -version and all complete groups are sealed before optimization. Group ordering, -trajectory weighting, optimizer boundaries and old-score checks are unchanged. +Every sample retains its independent action and environment seed streams. GRPO +training does not calculate entropy or sample unused entropy-estimator actions, +in either collector. Seeded trajectories therefore differ from older versions +that consumed those extra draws; PPO entropy computation is unchanged. All samples +use a single actor version and all complete groups are sealed before optimization. +Group ordering, trajectory weighting, optimizer boundaries and old-score checks +are unchanged. Floating-point results may vary with inference batch shape; cross-device or cross-configuration bitwise equivalence is not promised. diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index d7c8bc01..5ceae8d4 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -77,7 +77,7 @@ def reset(self, result, episode_id, desired_return=None): self.refresh() @torch.no_grad() - def decide(self, value=None, *, deterministic=False): + def decide(self, value=None, *, deterministic=False, compute_entropy=True): """Score action and history-only V from exactly the same causal input.""" actor = self.actor actor.eval() @@ -98,7 +98,7 @@ def decide(self, value=None, *, deterministic=False): decode_action(index, actor.config), index, float(distribution.log_prob(action)[0]), - float(distribution.entropy(self.generator)[0]), + float(distribution.entropy(self.generator)[0]) if compute_entropy else 0.0, float(estimate[0]), ) diff --git a/src/sequifier/rl/grpo_collector.py b/src/sequifier/rl/grpo_collector.py index b99dae50..7b51ef53 100644 --- a/src/sequifier/rl/grpo_collector.py +++ b/src/sequifier/rl/grpo_collector.py @@ -113,7 +113,7 @@ def collect_groups( if used > phase.max_buffer_bytes: raise ValueError("GRPO group exceeded max_buffer_bytes") inputs = cursor.inputs - decision = cursor.decide() + decision = cursor.decide(compute_entropy=False) used += 4 * len( json.dumps(decision.action, allow_nan=False).encode() ) diff --git a/src/sequifier/rl/parallel_grpo_collector.py b/src/sequifier/rl/parallel_grpo_collector.py index 77d32f56..8bda75f7 100644 --- a/src/sequifier/rl/parallel_grpo_collector.py +++ b/src/sequifier/rl/parallel_grpo_collector.py @@ -70,11 +70,6 @@ def _decide(actor, reference, cursors): # Only random draws use individual generators; no per-row decoder calls. actions = distribution.select(False, generators) old_scores = distribution.log_prob(actions) - if not distribution.entropy_is_exact: - # The sequential collector samples another action to estimate entropy. - # Preserve those per-environment RNG draws without calculating the - # unused entropy likelihood or surrogate. Exact entropy consumes no RNG. - distribution.sample(generators) scores = ( torch.stack([old_scores, reference.evaluate_actions(inputs, actions)], dim=1) .cpu() From 6fc6f52641c6b8983bb41b54fb294dc6f2994917 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 17 Nov 2026 10:40:08 +0100 Subject: [PATCH 43/71] Fix ineffective config options --- documentation/configs/ownership.md | 15 +- .../example_inputs/configs/infer-policy.yaml | 2 +- .../example_inputs/configs/infer-ppo-toy.yaml | 2 +- .../configs/rollout-ppo-toy.yaml | 2 +- documentation/supervised-policy-learning.md | 2 +- documentation/training/resource-controls.md | 70 ++++ release-notes/unreleased-policy-training.md | 2 +- src/sequifier/config/train_config.py | 2 +- src/sequifier/config/training_resolution.py | 5 +- src/sequifier/evaluation/service.py | 10 +- src/sequifier/helpers.py | 7 +- src/sequifier/io/folder_dataset.py | 135 ++++---- src/sequifier/io/memory_budget.py | 308 ++++++++++++++++++ src/sequifier/runtime/builder.py | 37 ++- src/sequifier/runtime/precision.py | 63 ++++ src/sequifier/templates/infer-policy.yaml | 2 +- .../train-policy-from-supervised.yaml | 2 +- src/sequifier/training/engine.py | 59 ++-- src/sequifier/training/optimization.py | 13 +- src/sequifier/training/runtime.py | 9 + 20 files changed, 622 insertions(+), 125 deletions(-) create mode 100644 documentation/training/resource-controls.md create mode 100644 src/sequifier/io/memory_budget.py create mode 100644 src/sequifier/runtime/precision.py diff --git a/documentation/configs/ownership.md b/documentation/configs/ownership.md index 2e33f077..70c94dc9 100644 --- a/documentation/configs/ownership.md +++ b/documentation/configs/ownership.md @@ -62,6 +62,15 @@ Data loading may configure `episode_cache_size`, but not `num_workers`, `onnx`, `pt`, `generative_model`, `embedding_model`, `with_dropout`, and `batch_size`; the PT actor export is selected automatically. +Native supervised `data_loading.max_ram_gb` budgets prepared-data loading per rank, +including workers, all cached loaders, chunk working space and prefetched batches. +Oversized plans fail before materialization; model/optimizer memory is additional. +`precision.layer_autocast` applies mixed precision to local/DDP training, +validation and warm-up as well as the FSDP policy. Matrix precision is applied +inside each training process. `enforce_determinism` uses errors rather than +warning-only deterministic algorithms. See [resource controls](../training/resource-controls.md) +for budget accounting, supported devices and the compiled option inventory. + ## Dataset and model contracts Prepared parts own `read_format`; sequence parts own `format`. Raw datasets own @@ -120,8 +129,10 @@ supervised phase. Sampling stride is `learning.window_stride`. All algorithms share these owners. Supervised cadence uses `{epochs: N}`; policy cadence uses `{iterations: N}`. A policy iteration is a complete algorithm transaction, possibly containing multiple optimizer steps. Final progress always -evaluates when enabled and checkpoints. Initial evaluation and snapshot evaluation -can select best weights but do not consume stopping patience. Patience counts +evaluates when enabled and checkpoints. Supervised initial and snapshot evaluations +are diagnostic: they do not select best weights or consume stopping patience. +Policy initial and snapshot evaluations can select best weights without consuming +stopping patience. Patience counts scheduled finite evaluation observations; `on_trigger` explicitly ends the phase or the entire plan. Stopping state is persisted so resume cannot count twice. diff --git a/documentation/example_inputs/configs/infer-policy.yaml b/documentation/example_inputs/configs/infer-policy.yaml index fb5c102f..acadcb10 100644 --- a/documentation/example_inputs/configs/infer-policy.yaml +++ b/documentation/example_inputs/configs/infer-policy.yaml @@ -5,7 +5,7 @@ desired_return: 10.0 action_selection: deterministic task: policy model_paths: -- models/policy/model.pt +- models/policy.pt run: project_root: . execution: diff --git a/documentation/example_inputs/configs/infer-ppo-toy.yaml b/documentation/example_inputs/configs/infer-ppo-toy.yaml index 42683d30..78d19a96 100644 --- a/documentation/example_inputs/configs/infer-ppo-toy.yaml +++ b/documentation/example_inputs/configs/infer-ppo-toy.yaml @@ -5,7 +5,7 @@ output_path: outputs/ppo-actions.csv action_selection: deterministic task: policy model_paths: -- models/online-ppo/model.pt +- models/online-ppo.pt run: project_root: . execution: diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index 3216fb08..283e3da0 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -14,7 +14,7 @@ rollout: action_selection: deterministic task: policy model_paths: -- models/online-ppo/model.pt +- models/online-ppo.pt run: project_root: . execution: diff --git a/documentation/supervised-policy-learning.md b/documentation/supervised-policy-learning.md index b01a84e3..a06fcb84 100644 --- a/documentation/supervised-policy-learning.md +++ b/documentation/supervised-policy-learning.md @@ -6,7 +6,7 @@ not automatically add inputs or replace a pretrained decoder. Start from an ordinary supervised PT artifact or supported run checkpoint: ```yaml -initialize_from: models/supervised/model.pt +initialize_from: models/supervised.pt policy_bindings: default: actions: diff --git a/documentation/training/resource-controls.md b/documentation/training/resource-controls.md new file mode 100644 index 00000000..483a7381 --- /dev/null +++ b/documentation/training/resource-controls.md @@ -0,0 +1,70 @@ +# Training resource controls + +The I12 source audit found these missing effects before this change: + +| Configuration | Missing behavior | +| --- | --- | +| `run.execution.data_loading.max_ram_gb` | No runtime consumer or data-loading budget. | +| `run.execution.precision.layer_autocast` | Ignored by local and DDP forward/validation; only FSDP applied mixed precision. | +| `run.execution.precision.float32_matmul_precision` | Set in the launcher, but lost in spawned training processes. | +| `run.reproducibility.enforce_determinism` | Requested deterministic algorithms in warning-only mode; false did not undo a previous true setting in the same process. | + +Device, distribution, FSDP offload, compilation, layer parameter dtypes, worker +counts, concatenation limits and episode-cache sizes already have consumers on +their applicable routes. Policy-containing plans reject unsupported resource +controls. `load_full_data_to_ram` selects eager/lazy **folder** loading; single +prepared files are eager. + +## Data-loading RAM + +`max_ram_gb` is a positive finite budget in decimal gigabytes for prepared-data +loading in each training rank, including its loader workers. It covers all cached +training/validation loaders together, decoded data, view/order indices, static +tables, decoded-frame caches, chunk/conversion working space and prefetched +batches. It is not an operating-system limit on total process RSS: model, +optimizer, activation, interpreter and third-party allocator memory require +additional headroom. Multiple ranks each have their own budget. + +Preflight uses decoded sizes rather than compressed file sizes and reserves +conversion/concatenation and worker/prefetch headroom. The largest applicable +phase batching/view requirements are considered before loading data. Actual +resident dataset storage is checked against the plan after construction, and +streamed shards are checked against their working allowance before gathering. An +over-budget plan raises `MemoryError` with required/available sizes and remedies; +it does not silently change an authored eager/lazy choice. Use lazy folders, +smaller shards, fewer workers, smaller batches or a larger budget as appropriate. + +Lazy folders gather at most a microbatch at a time, including when view expansion +produces many logical examples from one stored chunk. Chunk loading remains +bounded by the shard size, so an individual shard must fit its working allowance. + +## Autocast and matrix precision + +`layer_autocast: true` applies Torch autocast to local/DDP training forward and +loss calculation, validation and warm-up. Backward runs outside autocast. +Parameter storage dtypes remain governed by `layer_type_dtypes`. A configured +linear FP16/BF16 dtype also selects that autocast dtype; otherwise the device's +Torch default is used (CPU BF16, CUDA FP16). MPS uses BF16 and requires macOS 14 +or newer; native MPS FP16 autocast is rejected because the native gradient-scaler +protocol supports CPU/CUDA. Unsupported device/dtype requests fail explicitly. +FSDP retains its BF16 mixed-precision policy by default. + +FP16 autocast with FP32 optimizer parameters enables gradient scaling. Explicit +FP16 parameters retain the existing unscaled update protocol. The scaler is +persisted/restored with native recovery. FSDP uses its sharded mixed-precision +policy rather than a second enclosing autocast context. + +`float32_matmul_precision` is applied inside each native training process before +model construction/compilation, including spawned ranks and direct builder use. +It is independent of autocast and parameter storage precision. + +## Determinism + +`enforce_determinism: true` enables deterministic algorithms with errors for +unsupported nondeterministic operations and configures CUDA workspace policy +before seeding. False disables Torch's deterministic-algorithm requirement, +including after an earlier deterministic run in the same process. Seeds and +intentional stochastic action selection remain independent controls. + +No tests have been written or run for this change, as requested. Runtime/device +and memory-estimation validation remains pending. diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index 4c7f9793..d36d350a 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -25,7 +25,7 @@ phase configures optimizer roles under `optimization`; gradient validation and clipping precede all optimizer steps in a transaction. - Actor-only portable artifacts remain usable for inference, rollout and cross-run - initialization. Default output is `models//model.pt`; phase reports and + initialization. Default output is `models/.pt`; phase reports and selected actors use indexed directories. - Templates, examples, public exports and guides use the canonical schema. The pre-existing supervised training runtime remains independent. diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 9467aebe..0fe4a850 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -134,7 +134,7 @@ class TrainingRuntimeSettings(ConfigModel): enforce_determinism: bool = False distributed: bool = False load_full_data_to_ram: bool = True - max_ram_gb: int | float = 16 + max_ram_gb: float = Field(default=16, gt=0, allow_inf_nan=False) world_size: int = Field(default=1, gt=0) num_workers: int = Field(default=0, ge=0) backend: str = "nccl" diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 2ce02fdc..bd20e58a 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -540,10 +540,7 @@ def resolve_training_run(authored: SequifierConfig): resolved.model_path = str( ( root - / ( - config.run.outputs.model_path - or f"models/{config.run.model_name}/model.pt" - ) + / (config.run.outputs.model_path or f"models/{config.run.model_name}.pt") ).resolve() ) resolved.report_path = str( diff --git a/src/sequifier/evaluation/service.py b/src/sequifier/evaluation/service.py index 99b38915..a2d71d1c 100644 --- a/src/sequifier/evaluation/service.py +++ b/src/sequifier/evaluation/service.py @@ -4,6 +4,7 @@ import json import time +from contextlib import nullcontext from dataclasses import asdict, dataclass, field, is_dataclass from typing import Any, Sequence @@ -12,6 +13,7 @@ from sequifier.helpers import construct_index_maps from sequifier.model.network import ModelOutput +from sequifier.runtime.precision import PrecisionPolicy from sequifier.training.loss import LossService, PreparedBatch from sequifier.training.runtime import DatasetRuntimeRegistry, build_source_runtime @@ -66,8 +68,11 @@ class BaselineAccounting: class EvaluationService: - def __init__(self, loss_service: LossService) -> None: + def __init__( + self, loss_service: LossService, *, precision: PrecisionPolicy | None = None + ) -> None: self.loss_service = loss_service + self.precision = precision self._baselines: dict[tuple[str, str], BaselineAccounting] = {} @staticmethod @@ -154,7 +159,8 @@ def evaluate( was_training = network.training network.eval() try: - with torch.no_grad(): + autocast = self.precision.autocast() if self.precision else nullcontext() + with torch.no_grad(), autocast: for source_config in sources: started = time.perf_counter() source = build_source_runtime(source_config, datasets) diff --git a/src/sequifier/helpers.py b/src/sequifier/helpers.py index 11cf2e3d..06021ed9 100644 --- a/src/sequifier/helpers.py +++ b/src/sequifier/helpers.py @@ -941,6 +941,9 @@ def configure_logger( @beartype def configure_determinism(seed: int, strict: bool = False) -> None: """Enforces deterministic execution for reproducibility.""" + if strict: + # Set before CUDA RNG initialization can create a cuBLAS context. + os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) @@ -948,9 +951,7 @@ def configure_determinism(seed: int, strict: bool = False) -> None: torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True - if strict: - torch.use_deterministic_algorithms(True, warn_only=True) - os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" + torch.use_deterministic_algorithms(strict, warn_only=False) @beartype diff --git a/src/sequifier/io/folder_dataset.py b/src/sequifier/io/folder_dataset.py index 41f75403..29c5f97b 100644 --- a/src/sequifier/io/folder_dataset.py +++ b/src/sequifier/io/folder_dataset.py @@ -44,6 +44,9 @@ class FolderDataset(IterableDataset): """Initialize the metadata and state shared by every folder loader.""" + total_batches: int + target_samples: int + def __init__(self, data_path: str, config: Any, shuffle: bool, format_hint: str): super().__init__() # DataLoader spawn workers do not inherit the process group. @@ -117,6 +120,9 @@ def __len__(self) -> int: class EagerFolderDataset(FolderDataset): """Shared sample sharding for fully materialized folders.""" + n_samples: int + file_sample_orders: list[tuple[int, SampleOrderPlan]] + @beartype def _get_target_samples(self) -> int: world_size = self.world_size @@ -192,6 +198,10 @@ class LazyFolderChunk: class LazyFolderDataset(FolderDataset): """Schedule, order, and batch streamed folder chunks.""" + batch_files_info: list[dict[str, Any]] + chunk_format: str + ram_chunk_limit: int | None = None + @beartype def _get_target_samples(self) -> int: world_size = self.world_size @@ -290,6 +300,17 @@ def _iter_batches(self, worker_info: Any) -> Iterator[SequifierBatch]: file_path = os.path.join(self.data_dir, self.batch_files_info[f_id]["path"]) chunk = self._read_chunk(file_path) + if self.ram_chunk_limit is not None: + from sequifier.io.memory_budget import resident_bytes + + actual = resident_bytes(chunk) + if actual > self.ram_chunk_limit: + raise MemoryError( + f"Decoded shard {file_path!r} uses {actual / 1e9:.3f} GB, " + f"exceeding its planned {self.ram_chunk_limit / 1e9:.3f} GB " + "working allowance under max_ram_gb. Use smaller shards " + "or increase the data-loading RAM budget." + ) if len(chunk.sample_index) != file_samples: raise RuntimeError( f"Expanded sample count mismatch for {file_path}: " @@ -306,69 +327,67 @@ def _iter_batches(self, worker_info: Any) -> Iterator[SequifierBatch]: worker_file_start_idx = max(0, worker_start_sample - file_start) worker_file_end_idx = min(file_samples, worker_end_sample - file_start) worker_indices = indices[worker_file_start_idx:worker_file_end_idx] - logical_positions = torch.arange( - file_start + worker_file_start_idx, - file_start + worker_file_end_idx, - dtype=torch.int64, - ) - # Repeated file cycles are padding even when their source rows are real. - sample_is_real = logical_positions < rank_real_samples if len(worker_indices) == 0: + del chunk continue - - new_batch = build_window_batch( - chunk.sequences, - self.config.input_columns, - self.config.target_columns, - chunk.sample_index, - worker_indices, - sample_is_real, - depth_valid_masks=chunk.depth_valid_masks, - start_item_positions=chunk.start_item_positions, - split_start_item_positions=chunk.split_start_item_positions, - split_end_item_positions=chunk.split_end_item_positions, - sequence_ids=chunk.sequence_ids, - static_table=self.static_table, - image_store=self.image_store, - ) - new_seq = new_batch.inputs - new_tgt = new_batch.targets - new_meta = new_batch.metadata - if buffer_len == 0: - seq_buffer = new_seq - tgt_buffer = new_tgt - meta_buffer = new_meta - else: - seq_buffer = { - k: torch.cat([seq_buffer[k], new_seq[k]], dim=0) for k in seq_buffer - } - tgt_buffer = { - k: torch.cat([tgt_buffer[k], new_tgt[k]], dim=0) for k in tgt_buffer - } - if set(meta_buffer) != set(new_meta): - raise RuntimeError( - "Inconsistent leftPadLength metadata across " - f"{self.chunk_format} chunks." - ) - meta_buffer = { - k: torch.cat([meta_buffer[k], new_meta[k]], dim=0) - for k in meta_buffer - } - buffer_len += len(worker_indices) - - while buffer_len >= self.batch_size: + offset = 0 + while offset < len(worker_indices): if yielded_samples >= worker_target_samples: break - yield SequifierBatch( - inputs={k: v[: self.batch_size] for k, v in seq_buffer.items()}, - targets={k: v[: self.batch_size] for k, v in tgt_buffer.items()}, - metadata={k: v[: self.batch_size] for k, v in meta_buffer.items()}, + # Expanding an entire shard here can dwarf the stored payload, + # particularly with images, depth or multiple views per row. + count = min(self.batch_size - buffer_len, len(worker_indices) - offset) + positions = torch.arange( + file_start + worker_file_start_idx + offset, + file_start + worker_file_start_idx + offset + count, + dtype=torch.int64, + ) + new_batch = build_window_batch( + chunk.sequences, + self.config.input_columns, + self.config.target_columns, + chunk.sample_index, + worker_indices[offset : offset + count], + positions < rank_real_samples, + depth_valid_masks=chunk.depth_valid_masks, + start_item_positions=chunk.start_item_positions, + split_start_item_positions=chunk.split_start_item_positions, + split_end_item_positions=chunk.split_end_item_positions, + sequence_ids=chunk.sequence_ids, + static_table=self.static_table, + image_store=self.image_store, ) - yielded_samples += self.batch_size - seq_buffer = {k: v[self.batch_size :] for k, v in seq_buffer.items()} - tgt_buffer = {k: v[self.batch_size :] for k, v in tgt_buffer.items()} - meta_buffer = {k: v[self.batch_size :] for k, v in meta_buffer.items()} - buffer_len -= self.batch_size + if buffer_len == 0: + seq_buffer = new_batch.inputs + tgt_buffer = new_batch.targets + meta_buffer = new_batch.metadata + else: + if set(meta_buffer) != set(new_batch.metadata): + raise RuntimeError( + f"Inconsistent metadata across {self.chunk_format} chunks." + ) + seq_buffer = { + k: torch.cat([seq_buffer[k], new_batch.inputs[k]], dim=0) + for k in seq_buffer + } + tgt_buffer = { + k: torch.cat([tgt_buffer[k], new_batch.targets[k]], dim=0) + for k in tgt_buffer + } + meta_buffer = { + k: torch.cat([meta_buffer[k], new_batch.metadata[k]], dim=0) + for k in meta_buffer + } + offset += count + buffer_len += count + if buffer_len == self.batch_size: + yield SequifierBatch(seq_buffer, tgt_buffer, meta_buffer) + yielded_samples += buffer_len + seq_buffer, tgt_buffer, meta_buffer = {}, {}, {} + buffer_len = 0 + del new_batch + # Do not retain the previous chunk while decoding its successor. + del chunk if buffer_len > 0 and yielded_samples < worker_target_samples: final_yield_size = min(buffer_len, worker_target_samples - yielded_samples) diff --git a/src/sequifier/io/memory_budget.py b/src/sequifier/io/memory_budget.py new file mode 100644 index 00000000..d58a291b --- /dev/null +++ b/src/sequifier/io/memory_budget.py @@ -0,0 +1,308 @@ +"""Preflight and resident-storage accounting for prepared-data loading. + +The budget belongs to one training rank and its loader workers. It does not cap +model/optimizer memory or the Python process's address space. Estimates reserve +decode/conversion and prefetch headroom before any dataset is materialized. +""" + +from __future__ import annotations + +import csv +import json +import math +import sys +import zipfile +from dataclasses import dataclass, replace +from decimal import Decimal +from pathlib import Path +from typing import Any + +import torch +from loguru import logger + +from sequifier.helpers import PANDAS_TO_TORCH_TYPES, normalize_path + + +def resident_bytes(value: Any, seen: set[Any] | None = None) -> int: + """Count CPU backing stores once, including aliased views and Python indices.""" + if seen is None: + seen = set() + if id(value) in seen: + return 0 + seen.add(id(value)) + if isinstance(value, torch.Tensor): + if value.device.type != "cpu": + return 0 + storage = value.untyped_storage() + identity = ("storage", storage._cdata) + if identity in seen: + return 0 + seen.add(identity) + return sys.getsizeof(value) + storage.nbytes() + size = sys.getsizeof(value) + if isinstance(value, dict): + return size + sum( + resident_bytes(key, seen) + resident_bytes(item, seen) + for key, item in value.items() + if key != "config" + ) + if isinstance(value, (tuple, list, set, frozenset)): + return size + sum(resident_bytes(item, seen) for item in value) + if hasattr(value, "__dict__") and not isinstance(value, type): + return size + resident_bytes(vars(value), seen) + return size + + +def _parquet_size(path: Path, string_width: int) -> tuple[int, int]: + import pyarrow as pa + import pyarrow.parquet as pq + + metadata = pq.ParquetFile(path).metadata + schema = pq.read_schema(path) + row_width = 0 + for field in schema: + dtype = field.type + if pa.types.is_dictionary(dtype): + dtype = dtype.value_type + if pa.types.is_boolean(dtype): + row_width += 1 + elif pa.types.is_integer(dtype) or pa.types.is_floating(dtype): + row_width += dtype.bit_width // 8 + else: + # Prepared tables contain numeric values plus feature-name strings. + row_width += string_width + 8 + row_width += 1 # Validity and alignment headroom. + decoded = metadata.num_rows * row_width + pages = sum( + metadata.row_group(i).total_byte_size for i in range(metadata.num_row_groups) + ) + return max(decoded, pages), metadata.num_rows + + +def _source_size( + path: Path, read_format: str, string_width: int +) -> tuple[int, int | None]: + if read_format == "parquet": + return _parquet_size(path, string_width) + if read_format == "pt": + if not zipfile.is_zipfile(path): + raise ValueError( + f"RAM-budget preflight requires a current ZIP-based PT payload: {path}" + ) + with zipfile.ZipFile(path) as archive: + return sum(item.file_size for item in archive.infolist()), None + if read_format == "csv": + # Streaming inspection counts quoted records correctly without loading + # feature values into a DataFrame just to estimate its memory demand. + rows, decoded = 0, 0 + with path.open(newline="", encoding="utf-8-sig") as stream: + reader = csv.reader(stream) + next(reader, None) + for row in reader: + rows += 1 + decoded += sum(max(8, len(item.encode("utf-8")) + 8) for item in row) + return decoded, rows + raise ValueError(f"Cannot plan RAM for prepared format {read_format!r}") + + +def _column_bytes(config: Any, column: str) -> int: + dtype = PANDAS_TO_TORCH_TYPES[config.column_data_types[column]] + width = torch.empty((), dtype=dtype).element_size() + layout = config.depth_layouts.layout_for_column(column) + return width * (layout.max_depth_length if layout is not None else 1) + + +def _batch_bytes(config: Any) -> int: + context = config.window_view.context_length + values = context * sum( + _column_bytes(config, column) + for column in config.input_columns + config.target_columns + ) + values += context * ( + 32 + sum(layout.max_depth_length for _, layout in config.depth_layouts.items()) + ) + values += sum(_column_bytes(config, column) for column in config.static_columns) + for source in config.image_sources.values(): + values += context * source.frame_capacity * (3 * source.crop_size**2 + 1) + prefix = config.preconditioning_image + if prefix is not None: + values += len(prefix.views) * 3 * prefix.crop_size**2 + return config.batch_size * (values + 64) + + +@dataclass(frozen=True) +class LoaderMemoryEstimate: + resident: int + working: int + chunk_limit: int + + @property + def total(self) -> int: + return self.resident + self.working + + +def _estimate_loader(path: str, config: Any, eager: bool) -> LoaderMemoryEstimate: + source = Path(normalize_path(path, config.project_root)) + width = config.storage_layout.window_length + infos = None + metadata_bytes = 0 + if source.is_dir(): + with (source / "metadata.json").open() as stream: + metadata = json.load(stream) + metadata_bytes = resident_bytes(metadata) + width = metadata["window_length"] + infos = metadata["batch_files"] + files = ( + [(source / info["path"], int(info["samples"])) for info in infos] + if infos is not None + else [(source, None)] + ) + columns = set(config.input_columns + config.target_columns) + stored_row = width * sum(_column_bytes(config, column) for column in columns) + 48 + stored_row += width * sum( + layout.max_depth_length for _, layout in config.depth_layouts.items() + ) + stride = config.window_stride + views = ( + 1 + if stride is None + else max( + 1, + ( + width + - config.storage_layout.max_target_offset + - config.window_view.context_length + ) + // stride + + 1, + ) + ) + string_width = ( + max((len(name.encode("utf-8")) for name in config.column_data_types), default=0) + + 64 + ) + residents, decode_peak = [], 0 + for file, samples in files: + decoded, rows = _source_size(file, config.read_format, string_width) + if samples is None: + samples = ( + math.ceil(rows / max(1, len(columns))) + if rows is not None + else math.ceil(decoded / max(1, stored_row)) + ) + # Window indices, curriculum order/tie groups and epoch shuffles include + # Python lists as well as tensor indices. Reserve their expanded size. + indices = samples * (64 + 160 * views) + resident = samples * stored_row + indices + residents.append(resident) + decode_peak = max(decode_peak, 3 * decoded + 2 * resident) + static = 0 + if config.static_features_path is not None and config.static_columns: + static_path = Path( + normalize_path(config.static_features_path, config.project_root) + ) + static = 3 * _parquet_size(static_path, string_width)[0] + image_sources = list(config.image_sources.values()) + if config.preconditioning_image is not None: + image_sources.append(config.preconditioning_image) + cache = 128 * max((3 * item.crop_size**2 for item in image_sources), default=0) + workers = config.runtime_settings.num_workers + copies = max(1, workers) + batch = _batch_bytes(config) + # PyTorch's worker queue has four prefetched batches per worker. Pinning + # can overlap the queued copy; reserve both it and the consumer's batch. + prefetch = 2 * (4 * workers + 1) * batch + base = metadata_bytes + static + if eager: + resident = (sum(residents) + base) * (workers + 1) + working = decode_peak + copies * cache + prefetch + else: + resident = base * (workers + 1) + working = copies * (decode_peak + cache + 2 * batch) + prefetch + return LoaderMemoryEstimate(resident, working, decode_peak) + + +@dataclass +class DataLoadingBudget: + limit: int + estimates: dict[str, LoaderMemoryEstimate] + + def check(self) -> None: + required = sum(estimate.total for estimate in self.estimates.values()) + if required > self.limit: + raise MemoryError( + "Prepared-data loading exceeds run.execution.data_loading.max_ram_gb: " + f"planned {required / 1e9:.3f} GB, budget {self.limit / 1e9:.3f} GB " + "per training rank including loader workers. Use lazy folders " + "(load_full_data_to_ram: false), smaller shards/batches, fewer " + "workers, or increase max_ram_gb. Model/optimizer memory is additional." + ) + + def validate_dataset(self, key: str, dataset: Any, workers: int) -> None: + estimate = self.estimates[key] + measured = resident_bytes(dataset) * (workers + 1) + self.estimates[key] = replace( + estimate, resident=max(estimate.resident, measured) + ) + self.check() + # Plain integers survive DataLoader spawn without serializing the + # run-wide budget or retaining references to other loaders. + dataset.ram_chunk_limit = estimate.chunk_limit + + +def plan_data_loading(config: Any) -> DataLoadingBudget: + """Budget every cached source and the largest phase working requirements.""" + from sequifier.config.train_config import dataset_part_view + from sequifier.evaluation.compatibility import evaluation_batch_size + + settings = config.runtime_settings + if not math.isfinite(settings.max_ram_gb) or settings.max_ram_gb <= 0: + raise ValueError("max_ram_gb must be finite and positive") + budget = DataLoadingBudget(int(Decimal(str(settings.max_ram_gb)) * 10**9), {}) + phases = config.training_plan + batch = max( + [phase.batch_size for phase in phases] + + [evaluation_batch_size(phase) for phase in phases], + default=1, + ) + strides = [ + phase.learning.window_stride + for phase in phases + if phase.learning.window_stride is not None + ] + stride = min(strides) if strides else None + for name, dataset in config.dataset_training.items(): + for part_name, part in dataset.parts.items(): + selected = replace( + dataset_part_view(config, name, part_name), + batch_size=batch, + window_stride=stride, + ) + for split, path in ( + ("training", part.training_data_path), + ("validation", part.validation_data_path), + ): + if path is None: + continue + normalized = normalize_path(path, config.project_root) + splits = [ + normalize_path(item, config.project_root) + for item in part.metadata.split_paths + ] + layout = ( + part.metadata.layout_for_split(splits.index(normalized)) + if normalized in splits + else selected.storage_layout + ) + view = replace(selected, storage_layout=layout) + eager = part.storage_form == "file" or settings.load_full_data_to_ram + budget.estimates[f"{name}.{part_name}.{split}"] = _estimate_loader( + path, view, eager + ) + budget.check() + logger.info( + "Prepared-data RAM plan: {:.3f} GB / {:.3f} GB per rank (including workers)", + sum(item.total for item in budget.estimates.values()) / 1e9, + budget.limit / 1e9, + ) + return budget diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 51bced36..fd38d7d3 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -6,7 +6,7 @@ from collections.abc import Callable from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, cast import torch from loguru import logger @@ -16,14 +16,15 @@ from sequifier.config.optimizer_config import OptimizerPlan from sequifier.evaluation.service import EvaluationService from sequifier.export.service import ExportService -from sequifier.helpers import get_torch_dtype from sequifier.integration.callbacks import IntegrationManager +from sequifier.io.memory_budget import plan_data_loading from sequifier.model.factory import build_transformer_network, compile_unique_layers from sequifier.model.parameter_catalog import ( ParameterCatalog, semantic_optimizer_groups, ) from sequifier.runtime.context import ExecutionEnvironment, RunContext +from sequifier.runtime.precision import PrecisionPolicy from sequifier.runtime.random_state import RandomStateManager from sequifier.training.checkpoint_service import CheckpointRestorer, CheckpointService from sequifier.training.distributed_strategy import ( @@ -67,6 +68,7 @@ class TrainingRun: loss: LossService loader_state: LoaderStateService context: RunContext + precision: PrecisionPolicy def start_phase(self, phase_index: int) -> None: """Enter the phase with its resolved reset or continuation policy.""" @@ -97,22 +99,18 @@ def __init__(self, *, semantic_optimizer_grouping: bool = False) -> None: self.semantic_optimizer_grouping = semantic_optimizer_grouping def _strategy( - self, config: Any, execution: ExecutionEnvironment + self, config: Any, execution: ExecutionEnvironment, precision: PrecisionPolicy ) -> DistributedStrategy: if not execution.distributed: return LocalStrategy(device=execution.device) if config.runtime_settings.data_parallelism == "fsdp": - dtype = None - if config.runtime_settings.layer_autocast: - layer_types = config.runtime_settings.layer_type_dtypes or {} - dtype = get_torch_dtype(layer_types.get("linear", "bfloat16")) return FullyShardedStrategy( rank=execution.rank, local_rank=execution.local_rank, world_size=execution.world_size, device=execution.device, cpu_offload=bool(config.runtime_settings.fsdp_cpu_offload), - mixed_precision_dtype=dtype, + mixed_precision_dtype=precision.autocast_dtype, ) return DistributedDataParallelStrategy( rank=execution.rank, @@ -144,10 +142,10 @@ def _warm_up(self, run: TrainingRun) -> None: run.config.runtime_settings.context_length, ) features, metadata = schema.bind( - schema.example_inputs(batch_size, run.distributed.device) + schema.example_inputs(batch_size, str(run.distributed.device)) ) schema.validate(features, metadata) - with torch.no_grad(): + with torch.no_grad(), run.precision.autocast(): run.callable_network( features, metadata, interface_name=dataset.interface_name ) @@ -171,7 +169,11 @@ def build( *, initial_artifact=None, ) -> TrainingRun: - strategy = self._strategy(config, execution) + precision = PrecisionPolicy.from_settings( + config.runtime_settings, execution.device + ) + memory_budget = plan_data_loading(config) + strategy = self._strategy(config, execution, precision) random_manager = RandomStateManager(execution.device) loader_state = LoaderStateService() restorer = CheckpointRestorer() @@ -242,7 +244,11 @@ def build( and config.runtime_settings.data_parallelism == "ddp" and config.runtime_settings.torch_compile == "outer" ) - network_to_prepare = torch.compile(network) if compile_before_ddp else network + network_to_prepare = ( + cast(torch.nn.Module, torch.compile(network)) + if compile_before_ddp + else network + ) prepared = strategy.prepare_network(network_to_prepare) revisions = strategy.gather_objects(state.backbone_parent_revision_id) if any(revision != revisions[0] for revision in revisions[1:]): @@ -260,6 +266,9 @@ def build( runtime_metadata=built.runtime_metadata, freezing_plan=freezing_plan, ) + for dataset in datasets.values(): + for part in dataset.parts.values(): + part.factory.memory_budget = memory_budget configure_phase(config, network, datasets, state.phase_index, execution.device) for dataset in datasets.values(): for part in dataset.parts.values(): @@ -287,6 +296,7 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: parameter_catalog=catalog, accumulation_batches=phase.accumulation_batches, layer_type_dtypes=config.runtime_settings.layer_type_dtypes, + autocast_dtype=precision.autocast_dtype, ) validate_supervised_scheduler_budgets(config, datasets, state.phase_index) @@ -328,7 +338,7 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: distributed=strategy, random=random_manager, integrations=integrations, - evaluation=EvaluationService(loss_service), + evaluation=EvaluationService(loss_service, precision=precision), exporting=ExportService(config, execution.rank), checkpoints=CheckpointService( RunCheckpointStore(config, config.model_name), @@ -340,6 +350,7 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: loss=loss_service, loader_state=loader_state, context=context, + precision=precision, ) self._construct_loaders(run) if loaded is not None: diff --git a/src/sequifier/runtime/precision.py b/src/sequifier/runtime/precision.py new file mode 100644 index 00000000..65cac391 --- /dev/null +++ b/src/sequifier/runtime/precision.py @@ -0,0 +1,63 @@ +"""A process-local precision policy shared by forward, validation and warm-up.""" + +from contextlib import nullcontext +from dataclasses import dataclass +from typing import Any + +import torch + + +@dataclass(frozen=True) +class PrecisionPolicy: + device_type: str + autocast_dtype: torch.dtype | None = None + sharded: bool = False + + @classmethod + def from_settings(cls, settings: Any, device: torch.device) -> "PrecisionPolicy": + torch.set_float32_matmul_precision(settings.float32_matmul_precision) + sharded = settings.distributed and settings.data_parallelism == "fsdp" + if not settings.layer_autocast: + return cls(device.type, sharded=sharded) + if not torch.amp.autocast_mode.is_autocast_available(device.type): + raise ValueError(f"layer_autocast is unavailable on {device.type}") + configured = (settings.layer_type_dtypes or {}).get("linear") + dtype = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + }.get(configured or "") + if dtype is None: + dtype = ( + torch.bfloat16 + if sharded or device.type == "mps" + else torch.get_autocast_dtype(device.type) + ) + if dtype not in {torch.float16, torch.bfloat16}: + raise ValueError(f"layer_autocast requires FP16 or BF16, found {dtype}") + if device.type == "cuda": + if not torch.cuda.is_available(): + raise ValueError( + "CUDA layer_autocast requires an available CUDA device" + ) + with torch.cuda.device(device): + if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported(): + raise ValueError( + "BF16 layer_autocast is unsupported by this CUDA device" + ) + if device.type == "mps" and ( + not torch.backends.mps.is_available() + or not torch.backends.mps.is_macos_or_newer(14, 0) + ): + raise ValueError("BF16 layer_autocast requires MPS on macOS 14 or newer") + if dtype == torch.float16 and device.type not in {"cpu", "cuda"}: + # Native recovery owns a GradScaler; do not silently run unscaled AMP. + raise ValueError( + f"FP16 layer_autocast on {device.type} has no supported native " + "gradient scaler; select BF16 linear precision where supported" + ) + return cls(device.type, dtype, sharded) + + def autocast(self): + if self.autocast_dtype is None or self.sharded: + return nullcontext() + return torch.autocast(self.device_type, dtype=self.autocast_dtype) diff --git a/src/sequifier/templates/infer-policy.yaml b/src/sequifier/templates/infer-policy.yaml index 2593e25b..443ce1e3 100644 --- a/src/sequifier/templates/infer-policy.yaml +++ b/src/sequifier/templates/infer-policy.yaml @@ -5,7 +5,7 @@ desired_return: 10.0 action_selection: deterministic task: policy model_paths: -- models/policy/model.pt +- models/policy.pt run: project_root: . execution: diff --git a/src/sequifier/templates/train-policy-from-supervised.yaml b/src/sequifier/templates/train-policy-from-supervised.yaml index 4e0c930a..afd853bc 100644 --- a/src/sequifier/templates/train-policy-from-supervised.yaml +++ b/src/sequifier/templates/train-policy-from-supervised.yaml @@ -1,4 +1,4 @@ -initialize_from: models/supervised/model.pt +initialize_from: models/supervised.pt dataset_training: demonstrations: parts: diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 84263feb..57647d96 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -506,38 +506,39 @@ def flush_metrics( else nullcontext() ) with gradient_sync_context: - trace = run.integrations.forward_trace(prepared_event) - output = run.callable_network( - prepared.features, - prepared.metadata, - teacher_targets=prepared.loss_targets, - teacher_valid_mask=prepared.loss_valid_mask, - interface_name=dataset.interface_name, - trace=trace, - ) - if run.integrations.enabled: - run.integrations.emit( - ForwardCompleted( - access=access, - identity=identity, - outputs=output.logits, - captures={} - if trace is None - else dict(trace.captures), - ) + with run.precision.autocast(): + trace = run.integrations.forward_trace(prepared_event) + output = run.callable_network( + prepared.features, + prepared.metadata, + teacher_targets=prepared.loss_targets, + teacher_valid_mask=prepared.loss_valid_mask, + interface_name=dataset.interface_name, + trace=trace, ) - loss = run.loss.calculate( - output, prepared, dataset, run.network - ) - if run.integrations.enabled: - run.integrations.emit( - LossComputed( - access=access, - identity=identity, - loss=loss.backward_loss.detach(), - backward_loss=loss.backward_loss, + if run.integrations.enabled: + run.integrations.emit( + ForwardCompleted( + access=access, + identity=identity, + outputs=output.logits, + captures={} + if trace is None + else dict(trace.captures), + ) ) + loss = run.loss.calculate( + output, prepared, dataset, run.network ) + if run.integrations.enabled: + run.integrations.emit( + LossComputed( + access=access, + identity=identity, + loss=loss.backward_loss.detach(), + backward_loss=loss.backward_loss, + ) + ) run.optimization.accumulate( loss.backward_loss, identity, diff --git a/src/sequifier/training/optimization.py b/src/sequifier/training/optimization.py index 5b1de80c..a43b0bc7 100644 --- a/src/sequifier/training/optimization.py +++ b/src/sequifier/training/optimization.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any, Iterable -from torch import float16, nn +from torch import dtype, float16, nn from torch.amp.grad_scaler import GradScaler from torch.optim import Optimizer from torch.optim.lr_scheduler import ReduceLROnPlateau @@ -40,9 +40,8 @@ def create_optimizer(spec, parameters, learning_rate, *, parameter_catalog=None) return CompositeOptimizer( route_optimizer_parameters(spec, parameter_catalog, parameters) ) - return get_optimizer_class(spec.name)( - parameters, lr=learning_rate, **spec.arguments - ) + optimizer_class: Any = get_optimizer_class(spec.name) + return optimizer_class(parameters, lr=learning_rate, **spec.arguments) @dataclass(frozen=True) @@ -99,6 +98,7 @@ def create( parameter_catalog: ParameterCatalog | None = None, accumulation_batches: int = 1, layer_type_dtypes: dict[str, str] | None = None, + autocast_dtype: dtype | None = None, ) -> "OptimizationRuntime": optimizer = create_optimizer( training.optimizer, @@ -120,8 +120,9 @@ def create( for group in optimizer.param_groups for parameter in group["params"] ) - use_scaler = not has_fp16_parameters and bool( - layer_type_dtypes and "float16" in layer_type_dtypes.values() + use_scaler = not has_fp16_parameters and ( + autocast_dtype == float16 + or bool(layer_type_dtypes and "float16" in layer_type_dtypes.values()) ) return cls( optimizer=optimizer, diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 3a2b0d09..1ee79f1d 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -21,6 +21,7 @@ ) from sequifier.helpers import normalize_path from sequifier.io.batch import SequifierBatch +from sequifier.io.memory_budget import DataLoadingBudget, plan_data_loading from sequifier.io.sequifier_dataset_from_file import SequifierDatasetFromFile from sequifier.io.sequifier_dataset_from_folder_parquet import ( SequifierDatasetFromFolderParquet, @@ -65,6 +66,7 @@ class PartLoaderFactory: dataset_name: str part_name: str batch_size: int = 1 + memory_budget: DataLoadingBudget | None = None loaders: dict[Literal["training", "validation"], DataLoader] = field( default_factory=dict, init=False, @@ -79,6 +81,8 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: global_spec = self.config.runtime_settings if split in self.loaders: return self.loaders[split] + if self.memory_budget is None: + self.memory_budget = plan_data_loading(self.config) phase = self.config.training_plan[self.config.active_phase_index] from sequifier.evaluation.compatibility import evaluation_batch_size @@ -143,6 +147,11 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: raise ValueError( f"Folder parts do not support read_format={dataset_config.read_format!r}" ) + self.memory_budget.validate_dataset( + f"{self.dataset_name}.{self.part_name}.{split}", + dataset, + global_spec.num_workers, + ) generator = self.generators.get(split) if generator is None: generator = torch.Generator().manual_seed( From 0d5dd8a393ed6627b65f81dc734a4ab271e53c51 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 17 Nov 2026 16:35:09 +0100 Subject: [PATCH 44/71] unify interfaces --- docs/source/guides/artifacts.rst | 2 + docs/source/index.rst | 1 + .../configs/hyperparameter-search.md | 5 +- documentation/configs/infer.md | 12 +- documentation/configs/ownership.md | 7 +- documentation/configs/train.md | 12 +- documentation/policy-learning.md | 6 +- documentation/training/artifacts.md | 122 ++++++++++++++ .../unreleased-training-artifact-contracts.md | 20 +++ .../artifacts/checkpoint_envelope.py | 43 +++++ src/sequifier/artifacts/model_artifact.py | 19 ++- src/sequifier/artifacts/phase.py | 2 +- src/sequifier/artifacts/reports.py | 57 +++++++ src/sequifier/artifacts/run_checkpoint.py | 33 +++- .../canonical_hyperparameter_search_config.py | 25 +++ src/sequifier/config/inference_common.py | 3 + src/sequifier/config/ownership.py | 1 + src/sequifier/config/policy_infer_config.py | 14 +- src/sequifier/config/train_config.py | 14 +- src/sequifier/config/training_resolution.py | 4 +- src/sequifier/hyperparameter_search.py | 67 +++++--- src/sequifier/infer.py | 11 ++ src/sequifier/io/output_paths.py | 144 +++++++++++++++++ src/sequifier/rl/checkpoint.py | 5 +- src/sequifier/rl/dpo.py | 1 + src/sequifier/rl/grpo.py | 25 +-- src/sequifier/rl/inference.py | 78 ++++++--- src/sequifier/rl/iql.py | 4 + src/sequifier/rl/learner.py | 46 +++--- src/sequifier/rl/metrics.py | 90 +++++++++++ src/sequifier/rl/policy.py | 1 + src/sequifier/rl/ppo.py | 11 +- src/sequifier/rl/ppo_buffer.py | 9 +- src/sequifier/rl/preparation.py | 3 + src/sequifier/rl/run_checkpoint.py | 73 +++++++-- src/sequifier/rl/run_config.py | 2 + src/sequifier/rl/run_state.py | 32 +++- src/sequifier/rl/training.py | 1 + src/sequifier/runtime/builder.py | 16 ++ src/sequifier/train.py | 13 ++ src/sequifier/training/checkpoint_service.py | 2 +- src/sequifier/training/coordinator.py | 151 +++++++++++++----- src/sequifier/training/engine.py | 122 ++++++++++++-- src/sequifier/training/metrics.py | 122 +++++++++++++- src/sequifier/training/metrics_service.py | 5 + src/sequifier/training/state.py | 3 + src/sequifier/trajectory/archives.py | 38 +++++ src/sequifier/visualize_training.py | 81 ++++++++-- 48 files changed, 1366 insertions(+), 192 deletions(-) create mode 100644 docs/source/guides/artifacts.rst create mode 100644 documentation/training/artifacts.md create mode 100644 release-notes/unreleased-training-artifact-contracts.md create mode 100644 src/sequifier/artifacts/checkpoint_envelope.py create mode 100644 src/sequifier/artifacts/reports.py create mode 100644 src/sequifier/rl/metrics.py create mode 100644 src/sequifier/trajectory/archives.py diff --git a/docs/source/guides/artifacts.rst b/docs/source/guides/artifacts.rst new file mode 100644 index 00000000..5a759ab3 --- /dev/null +++ b/docs/source/guides/artifacts.rst @@ -0,0 +1,2 @@ +.. include:: ../../../documentation/training/artifacts.md + :parser: myst_parser.sphinx_ diff --git a/docs/source/index.rst b/docs/source/index.rst index abef44b4..dbd03814 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,6 +14,7 @@ Welcome to Sequifier's documentation! Visualizing training Distributed and multi-node training Training runtime architecture + Training artifacts .. toctree:: :maxdepth: 1 diff --git a/documentation/configs/hyperparameter-search.md b/documentation/configs/hyperparameter-search.md index fda886b3..85f00b1d 100644 --- a/documentation/configs/hyperparameter-search.md +++ b/documentation/configs/hyperparameter-search.md @@ -3,7 +3,10 @@ `sequifier hyperparameter-search` searches over complete canonical training configurations with Optuna. It supports Bayesian optimization, random sampling, finite grid search, cooperative pruning, and custom single- or multi-objective -evaluation. +evaluation. Supervised plans are supported; policy and mixed plans are not yet +supported. The final phase monitor supplies the default objective; configure the +matching `min`/`max` search direction. See +[Training artifacts](../training/artifacts.md). ```console sequifier hyperparameter-search --config-path configs/hyperparameter-search.yaml diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index 6cf53fb9..74ea90a3 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -45,7 +45,8 @@ sequifier infer --config-path configs/infer.yaml > `run.execution.device: cuda` alone does not enable GPU inference. See the > [ONNX Runtime CUDA compatibility guide](https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html). -ONNX is the default training export and the deployment-oriented inference path. +PT is the default training export. Supervised runs may explicitly request +ONNX for deployment-oriented inference. Exports embed their execution contract. Select a training route when providing model configuration through a training config: @@ -67,9 +68,14 @@ metadata. Add `part` when the dataset has several parts. Current ONNX exports also embed this contract and metadata, so the route is optional. Older exports need a training route or the full contract and metadata supplied explicitly. +All routes accept `output_path` for one selected model. It overrides the first +prediction or embedding file; later chunks append `-part-NNN` to its stem. +Prediction identity and probability-file layouts are described in +[Training artifacts](../training/artifacts.md#inference-and-collection-outputs). + ## ONNX or PT? -| | ONNX (default) | PT | +| | ONNX | PT (default) | | --- | --- | --- | | Best fit | Portable, deployment-oriented inference. | Python/PyTorch workflows and easier configuration. | | Runtime | ONNX Runtime on CPU or CUDA (with a CUDA-enabled ONNX Runtime installation). | PyTorch on CPU, CUDA, or MPS. | @@ -297,7 +303,7 @@ selection, read/write formats, device, seed, `enforce_determinism` and batch siz | `desired_return_column` | `desired_return`; per-row budget when no fixed budget is supplied | | `previous_action_columns` | {}; recorded lagged-action column mappings for fragments | | `sequence_id_column`, `item_position_column` | `sequenceId`, `itemPosition`; raw history ordering | -| `output_probabilities` | false; include joint log probability and conditional categorical probabilities | +| `output_probabilities` | false; include joint log probability and write separate target-specific categorical probability files | | `rollout` | null; execute in a version-3 online environment instead of writing prediction rows | Optional assertions `metadata_config_path`, `input_columns`, `static_columns`, diff --git a/documentation/configs/ownership.md b/documentation/configs/ownership.md index 70c94dc9..bf4bc21d 100644 --- a/documentation/configs/ownership.md +++ b/documentation/configs/ownership.md @@ -49,14 +49,17 @@ run: Distribution is one coherent choice: `local`, `ddp` or `fsdp`. FSDP additionally requires an explicit `cpu_offload`. Current policy execution is local, without -custom precision, compilation, concatenation limits, batch logging, or worker/lazy-loader/RAM-budget settings. +custom precision, compilation, concatenation limits, or worker/lazy-loader/RAM-budget settings. +Policy training logs use `run.logging.log_interval_iterations`; supervised +training logs use `run.logging.log_interval_batches`. Mixed plans may set both. Policy-containing plans export the portable PT actor; graph, embedding, dropout and batched exports are rejected. Runs default to PT export. Supervised runs can request graph exports explicitly. Defaults do not request those capabilities. Random run seeds resolve once; derived phase streams and resolved phase declarations are recorded in native recovery/provenance. For policy-containing plans, omit `run.execution.distribution`, `precision`, -`compilation`, and `max_concat_tensors`, as well as `run.logging.log_interval_batches`. +`compilation`, and `max_concat_tensors`. Policy-only plans also omit +`run.logging.log_interval_batches`. Data loading may configure `episode_cache_size`, but not `num_workers`, `load_full_data_to_ram`, or `max_ram_gb`. Omit the fixed export controls `onnx`, `pt`, `generative_model`, `embedding_model`, `with_dropout`, and diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 2a3b92fc..99456e84 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -925,7 +925,12 @@ selection must use their canonical YAML locations. ## Artifacts -ONNX is exported by default; PT inference bundles are opt-in. ONNX favors a +See [Training artifacts](../training/artifacts.md) for the shared output paths, +checkpoint envelopes, reports, logging and collection schemas across algorithms. + +PT is the default training export; supervised ONNX exports are opt-in. All +training routes also publish the canonical final PT model configured by +`run.outputs.model_path`. ONNX favors a portable deployment runtime, while PT embeds its execution contract and retains PyTorch behavior. See the "ONNX or PT?" section of the inference guide for the trade-offs. @@ -937,8 +942,9 @@ run-wide. Generated filenames do not use a `sequifier-` prefix. The PT inference bundle contains `artifact_type`, `format_version`, `model_state_dict`, an execution-only `model_config`, and export metadata -(trace-site names and provenance). Optimizers, paths, parts, training plans, -evaluation policy, and dataset bindings remain outside that bundle. +(trace-site names and provenance). Optimizer state, dataset parts, and evaluation runtime state remain outside +the execution contract. Optional provenance may include resolved training plans +and source paths. `export_with_dropout` affects ONNX export only: enabling it exports the ONNX graph in training mode and disables constant folding so dropout remains active. diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index ddfaf8c8..1bceecb1 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -207,10 +207,12 @@ Online intervals count completed collection/update iterations. ## Checkpoints and inference -Policy run checkpoints use format 6 and read format 5. They retain representation, +Policy runs use the shared version-4 run-checkpoint envelope; their native +recovery payload uses policy state version 7. See +[Training artifacts](training/artifacts.md). They retain representation, plan/source/partition/adapter identities, selected phase results and checksums, RNG and live phase networks/optimization/sampler/selection state. Keep committed -phase actors alongside the checkpoint; reports are regenerated from committed +phase model artifacts alongside the checkpoint; reports are regenerated from committed results. Source files and output-path conflicts are rechecked at phase boundaries. Atomic publication precedes checkpoint advancement, so completed phases never rerun. diff --git a/documentation/training/artifacts.md b/documentation/training/artifacts.md new file mode 100644 index 00000000..6a54a3e0 --- /dev/null +++ b/documentation/training/artifacts.md @@ -0,0 +1,122 @@ +# Training artifacts + +Supervised learning, policy imitation, IQL, PPO, DPO, GRPO and mixed plans use the +same public output, checkpoint, report and scalar metric contracts. These are +new major-version contracts. Previous native checkpoint formats remain readable; +earlier metric schemas are not migrated. + +## Configuration and layout + +All paths are relative to `run.project_root` unless absolute. + +| Setting | Default | +| --- | --- | +| `run.outputs.model_path` | `models/.pt` | +| `run.outputs.report_path` | `outputs/training//report.json` | +| `run.outputs.phase_directory` | `outputs/training//phases` | +| `run.recovery.resume.checkpoint_path` | `checkpoints/runs//-latest.pt` | + +Every runner publishes a final portable PT model, independently of the optional +best/last, embedding and ONNX exports controlled by `run.export`. Supervised +training retains its existing `-best-` and `-last-` +exports. The canonical final model follows the final executed phase's +`model_selection.handoff`. + +Every completed phase publishes `/-/model.pt` +and `report.json`. Mixed-plan native recovery is stored in that phase's `run.pt`; +native child artifacts are below `native/`. Policy snapshots use +`checkpoints/transaction-NNNNNNNN.pt` below the native phase directory. Supervised +snapshots retain their model/epoch filenames below the same phase `checkpoints/` +directory; the rolling checkpoint remains at its configured run-level path. + +Phase trees, logging directories and collection archive directories are reserved +output namespaces. Model, report, checkpoint and manifest files must be distinct +and cannot overlap reserved namespaces or source data. Validation runs before +publication, including generated phase destinations and symlink escapes. + +## Portable models and recovery + +Portable models use `artifact_type: sequifier_model`, `format_version: 3`. +Their execution contract embeds interfaces, codecs, normalization, policy +bindings and units. Policy execution reads units from the selected interface; +provenance is not a second source of execution configuration. + +All newly written run checkpoints have the same envelope: + +```text +artifact_type: sequifier_run_checkpoint +format_version: 4 +model: +recovery_type: supervised | policy | learning +recovery: +``` + +The recovery payload retains optimizer, scheduler, sampler, RNG, selection and +algorithm-specific state. Exact recovery validates the runner type as well as +its own state version and contracts. A checkpoint cannot resume another runner. +Inference, initialization and analysis use the common embedded `model`, without +having to parse recovery state or supply training YAML. Loading a checkpoint +that precedes the first executable model raises an explicit error. A mixed-plan +envelope embeds the last committed incoming phase model; its native phase +checkpoint contains in-phase progress. + +## Reports and discovery + +All final reports use `artifact_type: sequifier_run_manifest`, +`format_version: 3`, at the configured report path and at `manifest.json` beside +the rolling checkpoint. Common fields identify the logical run, current session, +status, completion reason, final model, checkpoint, exports and completed phases. +Additional fields contain runner-specific provenance. + +Each phase report uses `artifact_type: sequifier_phase_report`, +`format_version: 1`. Its common fields include phase index, name, algorithm, +model and checkpoint paths, monitor/direction, handoff, selected metric and +transaction, selected weight hash, clocks, metrics, provenance and stop scope. +Unavailable measurements are null. Algorithm-specific metrics keep their names +and units; a reward monitor is not represented as a loss. + +## Logging and scalar metrics + +Every native runner configures the model-scoped logger under `logs/`. +Structured scalar observations use the supervised writer's `*-training.csv`, +`*-training-full.csv`, `*-validation.csv` and `*-validation-full.csv` tables. +All tables carry `schema_version: 3`. Full tables include phase, algorithm and +iteration identity, run/session IDs, UTC timestamps and progress. Named scalar +observations use `metric`, `target`, `value` and `is_primary`; policy observations +use `target: __total__`. Nested diagnostic records remain in phase reports. + +`run.logging.log_interval_batches` controls supervised training observations; +`run.logging.log_interval_iterations` controls policy training observations and +defaults to 10. Mixed plans may configure both. Evaluations are written whenever +the phase evaluation schedule runs, and final policy training progress is always +written. Policy iteration clocks count committed updates, even when an iteration +contains several optimizer steps. Policy visualizations display the actual +monitor names on a linear scale and do not invent baseline measurements. + +Hyperparameter search accepts supervised plans. Policy and mixed plans are not yet +supported. Its default objective is the final phase's evaluation monitor; the +search direction must match. Explicit output destinations are isolated per +sampled run. + +## Inference and collection outputs + +Forecast and policy predictions share `sequenceId`, `subsequenceId`, +`windowStartOffset`, `itemPosition` and named prediction columns. Independent +policy rows receive synthetic sequence/item identities. Categorical probabilities +are separate target-specific files under +`outputs/probabilities///part-NNN.`, with decoded category +labels as column names. Policy predictions additionally include +`action_log_probability` when probabilities are requested. Conditional policy +probabilities remain conditional on the selected preceding actions. + +All inference routes accept `output_path` for one model. It overrides the first +prediction/embedding file; later chunks append `-part-NNN` to its stem. Probability +files keep their canonical target-specific destinations. + +Forecast and policy rollout use the shared validated trajectory record. +Training-time PPO and GRPO archives instead use the common +`sequifier.collection-archive` version-1 envelope with `algorithm` and +`collection` fields. The collection payload contains algorithm-specific replay +and diagnostic information. `read_collection_archive` reads the envelope; +`load_collection_record` additionally validates PPO collections. These archives +are diagnostics, not interchangeable offline training trajectories. diff --git a/release-notes/unreleased-training-artifact-contracts.md b/release-notes/unreleased-training-artifact-contracts.md new file mode 100644 index 00000000..c22b7a31 --- /dev/null +++ b/release-notes/unreleased-training-artifact-contracts.md @@ -0,0 +1,20 @@ +# Unified training artifact contracts + +- Honour `run.outputs` for supervised-only, policy-only and mixed plans, including + a canonical final portable model and per-phase model/report records. +- Reject overlapping output files, reserved phase/log/archive trees, protected + inputs and generated destinations escaping through symlinks before publication. +- Store exact recovery behind a shared version-4 `sequifier_run_checkpoint` + envelope with an embedded executable model and runner-owned recovery payload. +- Publish common version-3 run manifests and version-1 phase reports, retaining + supervised manifest discovery beside the rolling checkpoint. +- Read policy units from the model interface across every export/loading route. +- Write policy observations through the supervised scalar metric tables with + explicit version, phase, algorithm, iteration and primary-metric fields. Add + `run.logging.log_interval_iterations` and support policy visualization. +- Align prediction identity columns, categorical probability files and + `output_path` across inference tasks. Wrap algorithm-specific training collection + archives in a shared, versioned envelope. + +This is a new major-version contract. Previous native checkpoint formats remain +readable; earlier metric schemas are not migrated. diff --git a/src/sequifier/artifacts/checkpoint_envelope.py b/src/sequifier/artifacts/checkpoint_envelope.py new file mode 100644 index 00000000..fbca0038 --- /dev/null +++ b/src/sequifier/artifacts/checkpoint_envelope.py @@ -0,0 +1,43 @@ +"""Shared execution envelope around runner-owned exact recovery state.""" + +from sequifier.artifacts.model_artifact import ModelArtifact + +CHECKPOINT_FORMAT_VERSION = 4 + + +def checkpoint_envelope(recovery_type, recovery, model): + return { + "artifact_type": "sequifier_run_checkpoint", + "format_version": CHECKPOINT_FORMAT_VERSION, + "model": model.state_dict() if isinstance(model, ModelArtifact) else model, + "recovery_type": recovery_type, + "recovery": recovery, + } + + +def recovery_payload(payload, recovery_type): + """Validate the shared envelope before restoring runner-owned state.""" + legacy = { + "supervised": ("sequifier_run_checkpoint", 3), + "policy": ("sequifier_policy_run", 7), + "learning": ("sequifier_learning_run", 2), + } + if isinstance(payload, dict) and ( + payload.get("artifact_type"), + payload.get("format_version"), + ) == legacy.get(recovery_type): + return payload + if ( + isinstance(payload, dict) + and payload.get("artifact_type") == "sequifier_run_checkpoint" + ): + if payload.get("format_version") != CHECKPOINT_FORMAT_VERSION: + raise ValueError("Unsupported run checkpoint envelope version") + if payload.get("recovery_type", "supervised") != recovery_type: + raise ValueError( + f"Checkpoint does not contain {recovery_type} recovery state" + ) + if not isinstance(payload.get("recovery"), dict): + raise ValueError("Run checkpoint requires a recovery mapping") + return payload["recovery"] + raise ValueError("Expected a Sequifier run checkpoint envelope") diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index 5117d87e..b2f422bd 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -46,16 +46,11 @@ class ModelArtifact: metadata: ModelArtifactMetadata def validate(self) -> None: - if self.format_version not in (1, 2, MODEL_ARTIFACT_FORMAT_VERSION): + if self.format_version != MODEL_ARTIFACT_FORMAT_VERSION: raise ValueError( f"Unsupported model artifact format {self.format_version}; " f"expected {MODEL_ARTIFACT_FORMAT_VERSION}." ) - if self.format_version == 1 and any( - i.get("depth_layouts") - for i in self.model_config.values.get("interfaces", {}).values() - ): - raise ValueError("Depth execution requires model artifact format version 2") for name, interface in self.model_config.values.get("interfaces", {}).items(): if interface.get("depth_layouts") and "execution_schema" not in interface: raise ValueError( @@ -172,11 +167,15 @@ def read_artifact( ) artifact_type = payload.get("artifact_type") if isinstance(payload, dict) else None if expected_type is not None and artifact_type != expected_type: - if expected_type == "sequifier_run_checkpoint": - raise ValueError("Artifact is not a Sequifier run checkpoint.") - raise ValueError("Artifact is not a portable Sequifier model.") + raise ValueError(f"Expected {expected_type} artifact, received {artifact_type}") if artifact_type == "sequifier_run_checkpoint": - payload = payload["model"] + from sequifier.artifacts.checkpoint_envelope import CHECKPOINT_FORMAT_VERSION + + if payload.get("format_version") not in {3, CHECKPOINT_FORMAT_VERSION}: + raise ValueError("Unsupported run checkpoint envelope version") + payload = payload.get("model") + if payload is None: + raise ValueError("Checkpoint precedes the first executable model") if not isinstance(payload, dict) or not isinstance(artifact_type, str): raise ValueError( "Sequifier model artifact must be a mapping with an artifact_type" diff --git a/src/sequifier/artifacts/phase.py b/src/sequifier/artifacts/phase.py index a3414f23..99bc7f84 100644 --- a/src/sequifier/artifacts/phase.py +++ b/src/sequifier/artifacts/phase.py @@ -14,7 +14,7 @@ class PhaseArtifacts: model: Path @classmethod - def for_phase(cls, directory, index, name, filename="actor.pt"): + def for_phase(cls, directory, index, name, filename="model.pt"): return cls(Path(directory) / f"{index:03d}-{name}" / filename) @property diff --git a/src/sequifier/artifacts/reports.py b/src/sequifier/artifacts/reports.py new file mode 100644 index 00000000..74a24bcd --- /dev/null +++ b/src/sequifier/artifacts/reports.py @@ -0,0 +1,57 @@ +"""Common, versioned discovery records for every training runner.""" + +from pathlib import Path + +from sequifier.io.atomic import write_json_atomic + + +def phase_report(*, index, name, algorithm, model_path, checkpoint_path, **details): + return { + "artifact_type": "sequifier_phase_report", + "format_version": 1, + "index": index, + "name": name, + "algorithm": algorithm, + "model_path": str(model_path), + "checkpoint_path": str(checkpoint_path), + "monitor": None, + "direction": None, + "handoff": "last", + "selected_metric": None, + "selected_transaction": None, + "selected_weight_hash": None, + "clocks": {}, + "metrics": [], + "provenance": {}, + "stop_scope": None, + "completion_reason": "completed", + **details, + } + + +def run_report(*, model_path, checkpoint_path, phases, **details): + return { + "artifact_type": "sequifier_run_manifest", + "format_version": 3, + "model_path": str(model_path), + "checkpoint_path": str(checkpoint_path), + "phases": list(phases), + "run_id": None, + "session_id": None, + "status": "complete", + "completion_reason": "completed", + "exports": [], + "source_provenance": {}, + "initialization": {}, + "resume": {}, + **details, + } + + +def publish_run_report(report, path): + """Keep the supervised manifest discovery location on every route.""" + destination = Path(path).resolve() + manifest = Path(report["checkpoint_path"]).resolve().parent / "manifest.json" + write_json_atomic(destination, report) + if manifest != destination: + write_json_atomic(manifest, report) diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index a8e99e69..b4403fa7 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -60,8 +60,11 @@ def validate(self) -> None: def state_dict(self) -> dict[str, Any]: self.validate() - return { + from sequifier.artifacts.checkpoint_envelope import checkpoint_envelope + + recovery = { "artifact_type": "sequifier_run_checkpoint", + "recovery_type": "supervised", "format_version": self.format_version, "model": self.model.state_dict(), "optimization": asdict(self.optimization), @@ -73,15 +76,21 @@ def state_dict(self) -> dict[str, Any]: "initialization_seed_derivation_version": 1, "training_config": self.training_config.model_dump(mode="python"), } + return checkpoint_envelope("supervised", recovery, self.model) @classmethod def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": + from sequifier.artifacts.checkpoint_envelope import recovery_payload + + payload = recovery_payload(payload, "supervised") if payload.get("artifact_type") != "sequifier_run_checkpoint": raise ValueError("Unsupported checkpoint: expected new Sequifier format.") if payload.get("format_version") != RUN_CHECKPOINT_FORMAT_VERSION: raise ValueError( f"unsupported native checkpoint format {payload.get('format_version')}; expected {RUN_CHECKPOINT_FORMAT_VERSION}. Initialize from its portable model artifact to start a new run." ) + if payload.get("recovery_type", "supervised") != "supervised": + raise ValueError("Checkpoint belongs to another training runner") from sequifier.config.train_config import ResolvedSequifierConfig if ( @@ -115,16 +124,24 @@ class RunCheckpointStore: def __init__(self, training_config: Any, model_name: str) -> None: self.latest_path = checkpoint_path(training_config) self.model_name = model_name - - def path_for(self, suffix: str | None) -> Path: - return ( - self.latest_path - if suffix in {None, "latest"} - else self.latest_path.with_name(f"{self.model_name}-{suffix}.pt") + self.phase_directory = training_config.phase_directory + self.phases = training_config.training_plan + + def path_for(self, suffix: str | None, phase_index: int = 0) -> Path: + if suffix in {None, "latest"}: + return self.latest_path + from sequifier.artifacts.phase import PhaseArtifacts + + directory = ( + PhaseArtifacts.for_phase( + self.phase_directory, phase_index, self.phases[phase_index].name + ).model.parent + / "checkpoints" ) + return directory / f"{self.model_name}-{suffix}.pt" def save(self, checkpoint: RunCheckpoint, suffix: str | None) -> Path: - output_path = self.path_for(suffix) + output_path = self.path_for(suffix, checkpoint.run_state["phase_index"]) output_path.parent.mkdir(parents=True, exist_ok=True) write_run_checkpoint(checkpoint, output_path, self.latest_path) return output_path diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index 50681432..8754ca77 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -822,6 +822,28 @@ def _materialized_values(self, trial: Any | None, run_index: int) -> dict[str, A values["run"]["model_name"] = ( f"{self.run.study_name}-run-{run_index:0{len(str(trial_count))}d}" ) + outputs = values["run"].get("outputs", {}) + run_name = values["run"]["model_name"] + for key in ("model_path", "report_path"): + if outputs.get(key): + from pathlib import Path + + path = Path(outputs[key]) + outputs[key] = str( + path.with_name(f"{path.stem}-{run_name}{path.suffix}") + ) + if outputs.get("phase_directory"): + outputs["phase_directory"] = os.path.join( + outputs["phase_directory"], run_name + ) + recovery = values["run"].get("recovery", {}).get("resume", {}) + if recovery.get("checkpoint_path"): + from pathlib import Path + + path = Path(recovery["checkpoint_path"]) + recovery["checkpoint_path"] = str( + path.with_name(f"{path.stem}-{run_name}{path.suffix}") + ) return values @beartype @@ -894,6 +916,9 @@ def compile_canonical_hyperparameter_search_config( search_values["run"].setdefault("project_root", base_config.run.project_root) try: base_values = base_config.model_dump(mode="python") + base_values["run"] = base_config.run.model_dump( + mode="python", exclude_unset=True + ) search_config = CanonicalHyperparameterSearchConfig.model_validate( search_values ) diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index dae255e5..010ce084 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -17,6 +17,7 @@ class InferenceExecutionConfig(ConfigModel): model_interface: str | None = None read_format: str = "parquet" write_format: str = "csv" + output_path: str | None = None @property def project_root(self): @@ -41,6 +42,8 @@ def inference_batch_size(self): @model_validator(mode="after") def validate_resources(self): self.run.validate_for("infer") + if self.output_path is not None and len(getattr(self, "model_paths", ())) != 1: + raise ValueError("output_path requires exactly one model path") return self diff --git a/src/sequifier/config/ownership.py b/src/sequifier/config/ownership.py index 24dc8082..80859fe8 100644 --- a/src/sequifier/config/ownership.py +++ b/src/sequifier/config/ownership.py @@ -68,6 +68,7 @@ class RecoverySettings(Settings): class LoggingSettings(Settings): log_interval_batches: int = Field(default=10, gt=0) + log_interval_iterations: int = Field(default=10, gt=0) class OutputSettings(Settings): diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index a484c714..6649d104 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -29,7 +29,6 @@ class PolicyInferenceConfig(InferenceExecutionConfig): task: Literal["policy"] data_path: str - output_path: str | None = None output_probabilities: bool = False action_selection: Literal["deterministic", "stochastic"] = "deterministic" desired_return: float | None = Field(default=None, allow_inf_nan=False) @@ -131,6 +130,19 @@ def policy_inference_config( and request.rollout.environment.units != policy.units ): raise ValueError("rollout environment units differ from policy artifact") + reserved = {"sequenceId", "subsequenceId", "windowStartOffset", "itemPosition"} + if reserved & set(policy.actions) or ( + request.output_probabilities and "action_log_probability" in policy.actions + ): + raise ValueError( + "policy action names overlap prediction identity/probability columns" + ) + if request.output_probabilities and any( + len({str(value) for value in field.values}) != len(field.values) + for field in policy.actions.values() + if field.type == "categorical" + ): + raise ValueError("action labels must have unique probability column names") contract, metadata = _execution_source(execution, execution.dataset(selected)) for key, actual in contract.items(): if ( diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 0fe4a850..acccf153 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -129,6 +129,7 @@ class TrainingRuntimeSettings(ConfigModel): max_concat_tensors: int = Field(default=12, gt=0) log_interval_batches: int = Field(default=10, gt=0) + log_interval_iterations: int = Field(default=10, gt=0) latest_checkpoint_interval_minutes: Optional[float] = None resume: Optional[ResumeConfig] = None enforce_determinism: bool = False @@ -1831,6 +1832,11 @@ def resolve_sequifier_config( from sequifier.config.input_contract_resolution import validate_input_contracts validate_input_contracts(config, resolved) + from sequifier.io.output_paths import resolve_training_output_paths + + for name in ("model_path", "report_path", "phase_directory"): + setattr(resolved, name, getattr(config.run.outputs, name)) + resolve_training_output_paths(resolved) return resolved @@ -2319,7 +2325,13 @@ def validate_contract(self): "policy phases do not support run.execution settings: " f"{sorted(unsupported_execution)}" ) - if "log_interval_batches" in self.run.logging.model_fields_set: + if ( + all( + phase.algorithm != "supervised" + for phase in self.training_plan.phases + ) + and "log_interval_batches" in self.run.logging.model_fields_set + ): raise ValueError("policy phases do not support batch logging intervals") export = self.run.export unsupported_export = export.model_fields_set & { diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index bd20e58a..2140e99c 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -640,7 +640,8 @@ def policy_run_config( checkpoint = ( (root / resume.checkpoint_path).resolve() if resume and resume.checkpoint_path - else root / f"checkpoints/{config.model_name}/run.pt" + else root + / f"checkpoints/runs/{config.model_name}/{config.model_name}-latest.pt" ) if resuming is not None: checkpoint = ( @@ -666,6 +667,7 @@ def policy_run_config( source_validation_fractions=config.source_validation_fractions, latest_checkpoint_interval_minutes=training.latest_checkpoint_interval_minutes, episode_cache_size=training.episode_cache_size, + log_interval_iterations=training.log_interval_iterations, resume=ResumeConfig( checkpoint_path=str(checkpoint), policy=("if_exists" if resuming else "never") diff --git a/src/sequifier/hyperparameter_search.py b/src/sequifier/hyperparameter_search.py index 49d65b90..c35ba51e 100644 --- a/src/sequifier/hyperparameter_search.py +++ b/src/sequifier/hyperparameter_search.py @@ -1,12 +1,14 @@ import csv import ctypes import json +import math import os import signal import subprocess import sys import time import warnings +from pathlib import Path from typing import Any, Union import optuna @@ -21,10 +23,7 @@ from sequifier.config.hyperparameter_search_config import ( # noqa: E402 load_hyperparameter_search_config, ) -from sequifier.helpers import ( # noqa: E402 - get_best_model_path, - get_last_training_batch_timedelta, -) +from sequifier.helpers import get_last_training_batch_timedelta # noqa: E402 from sequifier.io.yaml import TrainModelDumper # noqa: E402 from sequifier.logging_paths import ( # noqa: E402 dataset_artifact_prefix, @@ -38,6 +37,12 @@ @beartype def _monitored_dataset(run_config: Any) -> tuple[str | None, int]: + if any( + phase.algorithm != "supervised" for phase in run_config.training_plan.phases + ): + raise ValueError( + "hyperparameter search for trajectory training is not yet supported" + ) dataset_names = tuple(run_config.dataset_training) dataset_count = len(dataset_names) if dataset_count == 1: @@ -97,28 +102,40 @@ def objective( ) os.makedirs(os.path.dirname(config_path), exist_ok=True) + final_phase = run_config.training_plan.phases[-1] + monitor = final_phase.evaluation.monitor if final_phase.evaluation else None + monitor_metric = monitor.metric if monitor is not None else "loss" + monitor_direction = monitor.direction if monitor is not None else "min" + expected_direction = (config.evaluation_metric_directions or ["min"])[0] + if not config.evaluation_script_path and expected_direction != monitor_direction: + raise ValueError( + "search direction must match the final phase evaluation monitor" + ) + serialized = run_config.model_dump(mode="python") + serialized["run"] = run_config.run.model_dump(mode="python", exclude_unset=True) with open(config_path, "w") as f: yaml.dump( - run_config.model_dump(mode="python"), + serialized, f, Dumper=TrainModelDumper, sort_keys=False, ) - log_dir = model_log_directory(config.run.project_root, run_name) + metrics_name = run_name + log_dir = model_log_directory(config.run.project_root, metrics_name) log_dir.mkdir(parents=True, exist_ok=True) validation_path = ( str( dataset_artifact_prefix( config.run.project_root, - run_name, + metrics_name, dataset_name=monitored_dataset, dataset_count=dataset_count, ) ) + "-validation-full.csv" ) - prune_path = str(log_dir / f"{run_name}.prune") + prune_path = str(log_dir / f"{metrics_name}.prune") consumed_evaluation_ids: set[str] = set() if os.path.exists(validation_path): with open(validation_path, "r", encoding="utf-8", newline="") as file: @@ -141,7 +158,7 @@ def objective( preexec_fn=set_pdeathsig if sys.platform.startswith("linux") else None, ) - best_val_loss = float("inf") + best_val_loss = float("inf") if monitor_direction == "min" else float("-inf") completed_epochs = 0 @beartype @@ -162,7 +179,8 @@ def consume_metrics( if ( not evaluation_id or evaluation_id in consumed_evaluation_ids - or data.get("metric") != "loss" + or data.get("metric") != monitor_metric + or (data.get("phase") and data["phase"] != final_phase.name) or data.get("target") != "__total__" or not data.get("value") or not data.get("global_step") @@ -173,7 +191,7 @@ def consume_metrics( val_loss = float(data["value"]) global_step = int(data["global_step"]) metric_epoch = int(data["epoch"]) - if data.get("evaluation_kind") == "epoch_end": + if data.get("evaluation_kind") in {"epoch_end", "iteration_end"}: completed_epochs = max(completed_epochs, metric_epoch) is_multi_objective = ( @@ -182,7 +200,9 @@ def consume_metrics( ) if not is_multi_objective: trial.report(val_loss, global_step) - best_val_loss = min(best_val_loss, val_loss) + best_val_loss = (min if monitor_direction == "min" else max)( + best_val_loss, val_loss + ) if config.pruning_warmup_batches is not None: warmup_complete = ( @@ -200,7 +220,7 @@ def consume_metrics( try: try: timedelta = get_last_training_batch_timedelta( - run_name, 0, config.run.project_root + metrics_name, 0, config.run.project_root ) timeout_val = (timedelta * 2) + 30 except (ValueError, FileNotFoundError): @@ -232,14 +252,21 @@ def consume_metrics( elif exit_code != 0: raise RuntimeError(f"Training failed with exit code {exit_code}") - model_type = "onnx" if run_config.run.export.onnx else "pt" - model_path, _last_epoch = get_best_model_path( - config.run.project_root, - run_name, - model_type, - dataset_name=(monitored_dataset if model_type == "onnx" else None), - dataset_count=(dataset_count if model_type == "onnx" else 1), + report_path = Path(config.run.project_root) / ( + run_config.run.outputs.report_path or f"outputs/training/{run_name}/report.json" ) + report = json.loads(report_path.read_text()) + phases = report["phases"] + if not phases: + raise ValueError("Training completed without a selected phase") + final_report = phases[-1] + best_val_loss = final_report.get("selected_metric") + if not config.evaluation_script_path: + if final_report["name"] != final_phase.name: + raise ValueError("Search plan stopped before its objective phase") + if best_val_loss is None or not math.isfinite(best_val_loss): + raise ValueError("Search requires a finite final phase selection metric") + model_path = report["model_path"] evaluation_id = os.path.splitext(os.path.basename(model_path))[0] if config.evaluation_inference_config_path: diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index 1baded50..1dd3d13f 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -846,8 +846,17 @@ def inference_output_path( model_id: str, data_id: int, target_column: Optional[str] = None, + output_path: Optional[str] = None, ) -> str: """Return a canonical inference output path and create its directory.""" + if output_path is not None and artifact_type in {"predictions", "embeddings"}: + destination = Path(project_root) / output_path + if data_id: + destination = destination.with_name( + f"{destination.stem}-part-{data_id:03d}{destination.suffix}" + ) + destination.parent.mkdir(parents=True, exist_ok=True) + return str(destination) output_dir = Path(project_root) / "outputs" / artifact_type / model_id if target_column is not None: output_dir /= target_column @@ -975,6 +984,7 @@ def infer_embedding( "embeddings", model_id, data_id, + output_path=config.output_path, ) logger.info(f"Writing embeddings to '{embeddings_path}'") write_data( @@ -1179,6 +1189,7 @@ def infer_generative( "predictions", model_id, data_id, + output_path=config.output_path, ) logger.info(f"Writing predictions to '{predictions_path}'") write_data( diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py index a12b6c38..f5e3722f 100644 --- a/src/sequifier/io/output_paths.py +++ b/src/sequifier/io/output_paths.py @@ -38,3 +38,147 @@ def require_directory_destinations(paths): for path in paths: if path.exists() and not path.is_dir(): raise ValueError(f"archive requires a directory: {path}") + + +def validate_training_destinations(config, checkpoint, protected): + """Reserve generated trees as well as explicit files before any publication.""" + checkpoint = Path(checkpoint).resolve() + files = unique_paths( + [config.model_path, config.report_path, checkpoint], + "training model, report and checkpoint paths must be distinct", + ) + manifest = checkpoint.parent / "manifest.json" + if manifest != Path(config.report_path).resolve(): + files = unique_paths([*files, manifest], "manifest overlaps a training output") + directories = [Path(config.phase_directory).resolve()] + from sequifier.logging_paths import model_log_directory + + directories.append( + model_log_directory(config.project_root, config.model_name).resolve() + ) + phases = getattr(config, "phases", getattr(config, "training_plan", ())) + from sequifier.artifacts.phase import PhaseArtifacts + + generated = [] + for index, phase in enumerate(phases): + artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) + for path in ( + artifacts.model, + artifacts.report, + artifacts.receipt, + artifacts.model.with_name("run.pt"), + artifacts.model.parent / "native", + artifacts.model.parent / "checkpoints", + ): + resolved = path.resolve() + if not contains_path(directories[0], resolved): + raise ValueError(f"phase destination escapes phase_directory: {path}") + generated.append(resolved) + validate_path_conflicts( + generated, protected, label="phase output", check_each_other=False + ) + directories.extend( + Path(phase.archive_directory).resolve() + for phase in phases + if getattr(phase, "archive_directory", None) + ) + directories = unique_paths( + directories, "training output directories must be distinct" + ) + require_file_destinations(files) + require_directory_destinations(directories) + # A generated snapshot may be selected as the rolling checkpoint on resume. + snapshot_dirs = { + ( + PhaseArtifacts.for_phase( + config.phase_directory, index, phase.name + ).model.parent + / "checkpoints" + ).resolve() + for index, phase in enumerate(phases) + } + resume = getattr(getattr(config, "runtime_settings", None), "resume", None) + snapshot_resume = ( + resume is not None + and resume.policy != "never" + and checkpoint.is_file() + and checkpoint.parent in snapshot_dirs + and checkpoint.name.startswith(f"{config.model_name}-") + and checkpoint.suffix == ".pt" + and manifest != Path(config.report_path).resolve() + ) + if snapshot_resume: + external_files = [path for path in files if path not in {checkpoint, manifest}] + validate_path_conflicts( + [*external_files, *directories], protected, label="training output" + ) + validate_path_conflicts( + [checkpoint, manifest], + protected | set(external_files) | set(directories[1:]), + label="training output", + ) + else: + validate_path_conflicts( + [*files, *directories], protected, label="training output" + ) + if any(phase.algorithm == "supervised" for phase in phases): + import re + + export_directory = (Path(config.project_root) / "models").resolve() + names = [config.model_name] + [ + f"{config.model_name}-{name}" for name in config.dataset_training + ] + pattern = re.compile( + "(?:" + + "|".join(re.escape(name) for name in names) + + r")-(?:best|last)(?:-embedding)?-\d+\.(?:pt|onnx)" + ) + for path in [*files, *directories, *(Path(p).resolve() for p in protected)]: + if contains_path(path, export_directory) or ( + path.parent == export_directory and pattern.fullmatch(path.name) + ): + raise ValueError( + f"training destination/input overlaps generated exports: {path}" + ) + + +def supervised_input_paths(config): + """Include split directories, metadata and media that native loaders consume.""" + root = Path(config.project_root) + paths = set() + if config.initialize_from: + paths.add((root / config.initialize_from).resolve()) + for dataset in config.dataset_training.values(): + for part in dataset.parts.values(): + for name in ( + "metadata_config_path", + "training_data_path", + "validation_data_path", + ): + if value := getattr(part, name, None): + paths.add((root / value).resolve()) + interface = dataset.interface + if interface.static_features_path: + paths.add((root / interface.static_features_path).resolve()) + for source in interface.image_sources.values(): + if source.path: + paths.add((root / source.path).resolve()) + if interface.preconditioning_image: + paths.update( + (root / path).resolve() + for path in (interface.preconditioning_image.paths or {}).values() + ) + return paths + + +def resolve_training_output_paths(config): + root = Path(config.project_root) + defaults = { + "model_path": f"models/{config.model_name}.pt", + "report_path": f"outputs/training/{config.model_name}/report.json", + "phase_directory": f"outputs/training/{config.model_name}/phases", + } + for name, default in defaults.items(): + setattr( + config, name, str((root / (getattr(config, name, "") or default)).resolve()) + ) diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 73ef7410..6c39a835 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -1,4 +1,4 @@ -"""Policies use the same portable model and run-checkpoint artifacts as forecasts.""" +"""Policy views over shared portable models and execution checkpoints.""" from pathlib import Path @@ -37,7 +37,6 @@ def save_policy( policy.execution_config, provenance={ **(metadata or {}), - "policy_units": dict(getattr(policy, "units", {})), "transfer": getattr(policy, "transfer_report", None), }, state_dict={ @@ -89,7 +88,7 @@ def load_policy( policy = attach_policy(network, config, interface_name, binding) else: policy = Policy.from_network(network, config, interface_name) - policy.units = dict(artifact.metadata.provenance.get("policy_units", {})) + policy.units = dict(policy.config.units) validate_policy_weights(policy) policy.eval() return policy diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index f94e7592..6e158f8b 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -117,6 +117,7 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): session.step() session.transaction = update final = update == phase.iterations + session.record_training(final=final, training_dpo_loss=loss_total) if session.should_evaluate(final): held_out = evaluate_dpo(actor, reference, data.validation, phase) session.record_evaluation( diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 1b38cb0e..1f976334 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -25,6 +25,7 @@ from sequifier.rl.grpo_update import update_grpo from sequifier.rl.ppo_evaluation import evaluate_policy from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal +from sequifier.trajectory.archives import collection_archive def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): @@ -109,7 +110,10 @@ def evaluate(): ) if not torch.isfinite(torch.tensor(score)): raise ValueError("GRPO evaluation score exceeds finite FP32 range") - session.observe(score) + session.observe( + score, + **{key: value for key, value in result.items() if key != "score_mean"}, + ) evaluations.append( {**result, "score_mean": score, "transaction": session.transaction} ) @@ -143,13 +147,16 @@ def evaluate(): write_json_atomic( Path(phase.archive_directory) / f"groups-{session.transaction:08d}.json", - { - "format": "sequifier.grpo-archive", - "version": 1, - "reference_identity": context.reference_identity, - "generation": buffer.generation, - "trajectories": [t.as_dict() for t in buffer.trajectories], - }, + collection_archive( + "grpo", + { + "format": "sequifier.grpo-archive", + "version": 1, + "reference_identity": context.reference_identity, + "generation": buffer.generation, + "trajectories": [t.as_dict() for t in buffer.trajectories], + }, + ), ) evaluation_start = perf_counter() if session.should_evaluate(final): @@ -184,7 +191,7 @@ def evaluate(): ) record.update(image_cache_statistics(actor, context.reference)) release_image_cache_entries(actor, context.reference) - session.metrics.append(record) + session.record_training(final=final, **record) logger.info( "GRPO iteration {}: {} steps, collection {:.2f}s ({:.1f} steps/s), " "update {:.2f}s, evaluation {:.2f}s", diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 487e5aaf..a760c428 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -131,11 +131,15 @@ def policy_windows(data, spec, config): @torch.no_grad() def predict_policy(policy, data, config): spec = policy.config + row_key = policy_row_key(data, spec) windows = policy_windows(data, spec, config) generator = action_generator(policy.device, config.seed) outputs = [] + probability_rows = { + name: [] for name, field in spec.actions.items() if field.type == "categorical" + } while batch := list(islice(windows, config.inference_batch_size)): - rows, histories = zip(*batch) + rows, histories, identities = zip(*batch) inputs = cast(SequifierInputs, collate_model_inputs(histories)).to( policy.device ) @@ -151,7 +155,8 @@ def predict_policy(policy, data, config): ) for index, (row_index, action) in enumerate(zip(rows, actions)): output = { - "__policy_row": row_index, + row_key: row_index, + **identities[index], **decode_action(action, spec), } if config.output_probabilities: @@ -159,32 +164,61 @@ def predict_policy(policy, data, config): distribution.log_prob(sampled)[index] ) for name, values in probabilities.items(): - for action_id, value in enumerate(values[index]): - output[f"{name}__probability_{action_id}"] = float(value) + probability_rows[name].append( + { + row_key: row_index, + **{ + str(label): float(values[index][action_id]) + for action_id, label in enumerate(spec.actions[name].values) + }, + } + ) outputs.append(output) - predictions = pl.DataFrame(outputs).sort("__policy_row").drop("__policy_row") - identity = [ - column - for column in (config.sequence_id_column, config.item_position_column) - if column in data.columns and column not in predictions.columns - ] - predictions = ( - pl.concat([data.select(identity), predictions], how="horizontal") - if identity - else predictions - ) + predictions = pl.DataFrame(outputs).sort(row_key).drop(row_key) + if config.output_probabilities: + from sequifier.infer import inference_output_path + + for name, records in probability_rows.items(): + if not records: + continue + destination = inference_output_path( + config.project_root, + config.write_format, + "probabilities", + Path(config.model_paths[0]).stem, + 0, + name, + ) + write_data( + pl.DataFrame(records).sort(row_key).drop(row_key), + destination, + config.write_format, + ) return predictions +def policy_row_key(data, spec): + reserved = ( + set(data.columns) + | set(spec.actions) + | {str(value) for field in spec.actions.values() for value in field.values} + ) + key = "__policy_row" + while key in reserved: + key += "_" + return key + + def joint_policy_windows(data, spec, config): - indexed = data.with_row_index("__policy_row") + row_key = policy_row_key(data, spec) + indexed = data.with_row_index(row_key) groups = ( indexed.partition_by(config.sequence_id_column, maintain_order=True) if config.sequence_id_column in data.columns else [indexed.slice(i, 1) for i in range(len(indexed))] ) view = PolicyWindowView(spec) - for group in groups: + for group_index, group in enumerate(groups): if config.item_position_column not in group.columns: group = group.with_row_index(config.item_position_column) group = ordered_sequence_rows(group, config.item_position_column) @@ -217,11 +251,19 @@ def joint_policy_windows(data, spec, config): if spec.return_conditioned: budgets.append(desired_return(config, spec, row)) yield ( - row["__policy_row"], + row[row_key], view.inputs( observations, previous, budgets if spec.return_conditioned else None, {name: row[name] for name in spec.static_columns}, ), + { + "sequenceId": row.get(config.sequence_id_column, group_index), + "subsequenceId": index, + "windowStartOffset": rows[max(0, index + 1 - spec.context_length)][ + config.item_position_column + ], + "itemPosition": row[config.item_position_column], + }, ) diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 40ee6c50..e26340b3 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -149,6 +149,10 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): _polyak(targets[1], q2, phase.target_update_rate) session.transaction = update final = update == phase.iterations + session.record_training( + final=final, + **{f"{role}_loss": float(loss.detach()) for role, loss in losses.items()}, + ) if session.should_evaluate(final): held_out = _evaluate( actor, diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 3c35d9e3..d0b58adb 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -7,8 +7,7 @@ from sequifier.artifacts.phase import PhaseArtifacts from sequifier.fingerprints import fingerprint -from sequifier.helpers import configure_determinism -from sequifier.io.atomic import write_json_atomic +from sequifier.helpers import configure_determinism, configure_logger from sequifier.rl.checkpoint import load_policy, phase_actor_path, save_policy from sequifier.rl.preparation import ( build_actor, @@ -110,6 +109,7 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: inputs.revalidate(inputs.identity, "run.source_identity") data_identity = inputs.identity validate_output_paths(config, inputs) + configure_logger(config.project_root, config.model_name, 0) checkpoint = Path(config.checkpoint_path) recovery = RunCheckpoint.load(checkpoint, config, plan, data_identity) resuming = recovery is not None @@ -161,6 +161,9 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: clone_weights(actor.network), capture_rng_state(actor.device), ) + from sequifier.rl.metrics import PolicyMetrics + + config._metrics_service = PolicyMetrics(config, recovery.run_id) references = PolicyReferences(config, recovery, resuming=resuming) saved_random = capture_rng_state(actor.device) step_budgets = preflight_optimization(config, actor, data) @@ -169,7 +172,7 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: if not resuming: # Persist the complete fitted representation before the first update. recovery.save(checkpoint, actor.device) - recovery.write_reports() + recovery.write_reports(checkpoint) while recovery.phase_index < len(config.phases) and not recovery.stopped: phase_index = recovery.phase_index current_state = recovery.phase_state @@ -223,23 +226,28 @@ def commit(state): recovery.continuation = getattr(actor, "_phase_continuation", None) recovery.finish_phase(actor, result) recovery.save(checkpoint, actor.device) - recovery.write_reports() - report = { - "format": "sequifier.policy-run-report", - "version": 1, - "plan": plan, - "plan_fingerprint": fingerprint(plan), - "representation_fingerprint": representation.identity, - "source_provenance": data_identity, - "reference_provenance": recovery.reference_contract, - "initialization": recovery.initialization, - "resume": config.resume.model_dump(mode="json"), - "phases": [r.as_dict() for r in recovery.results], - "model_path": config.model_path, - "checkpoint_path": config.checkpoint_path, - } + recovery.write_reports(checkpoint) + from sequifier.artifacts.reports import publish_run_report, run_report + + report = run_report( + model_path=config.model_path, + checkpoint_path=checkpoint, + phases=recovery.phase_reports(checkpoint), + run_id=recovery.run_id, + session_id=config._metrics_service.session_id, + status="complete", + completion_reason="early_stopping" if recovery.stopped else "completed", + exports=[config.model_path], + plan=plan, + plan_fingerprint=fingerprint(plan), + representation_fingerprint=representation.identity, + source_provenance=data_identity, + reference_provenance=recovery.reference_contract, + initialization=recovery.initialization, + resume=config.resume.model_dump(mode="json"), + ) save_policy(actor.eval(), config.model_path, metadata=report) - write_json_atomic(config.report_path, report) + publish_run_report(report, config.report_path) recovery.save(checkpoint, actor.device) return PolicyRunResult( actor, diff --git a/src/sequifier/rl/metrics.py b/src/sequifier/rl/metrics.py new file mode 100644 index 00000000..0ee42907 --- /dev/null +++ b/src/sequifier/rl/metrics.py @@ -0,0 +1,90 @@ +"""Policy observations in the native supervised metric tables.""" + +import math +import time +import uuid + +from sequifier.training.metrics import StructuredMetricWriters + + +class PolicyMetrics: + def __init__(self, config, run_id): + self.writer = StructuredMetricWriters( + config.project_root, + config.model_name, + 0, + dataset_name=config.policy_interface, + ) + self.run_id = run_id + self.session_id = uuid.uuid4().hex + self.started = time.monotonic() + self.pending = None + + def record( + self, + session, + values, + *, + validation=False, + primary=None, + initial=False, + final=False, + ): + context = session.context + if not validation: + self.pending = (session, values) + if ( + not validation + and not final + and not session.stopping.triggered + and session.transaction % context.run.log_interval_iterations + ): + return + scalars = { + name: float(value) + for name, value in values.items() + if isinstance(value, (float, int)) + and not isinstance(value, bool) + and math.isfinite(value) + } + if not scalars: + return + if primary is None: + primary = next( + ( + name + for name in ( + "training_action_nll", + "actor_loss", + "training_dpo_loss", + "loss", + "policy_loss", + ) + if name in scalars + ), + next(iter(scalars)), + ) + self.writer.phase = context.phase.name + self.writer.algorithm = context.phase.algorithm + now = time.monotonic() + self.writer.write_observation( + validation=validation, + run_id=self.run_id, + session_id=self.session_id, + epoch=context.index + 1, + iteration=session.transaction, + batches_total=session.iteration_budget, + global_step=sum(session.steps.values()), + values=scalars, + primary=primary, + learning_rate=float(session.optimization["actor"][0].param_groups[0]["lr"]), + elapsed_seconds=now - self.started, + initial=initial, + ) + if not validation: + self.started = now + self.pending = None + + def flush(self, session): + if self.pending is not None and self.pending[0] is session: + self.record(session, self.pending[1], final=True) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 196be41a..91a42b6a 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -69,6 +69,7 @@ def from_network(cls, network, execution_config, interface_name: str): ) policy = cls() policy.config = interface.policy + policy.units = dict(interface.policy.units) policy.execution_config = execution_config policy.interface_name = interface_name policy.network = network diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index 4123b98d..af849134 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -68,7 +68,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): initial = evaluate_policy( actor, phase.evaluation, forbidden_adapter=adapter ) - session.observe(initial["selection_value"]) + session.observe(initial["selection_value"], **initial) evaluations.append( {**initial, "transaction": session.transaction, "initial": True} ) @@ -89,7 +89,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): result = evaluate_policy( actor, phase.evaluation, forbidden_adapter=adapter ) - session.observe(result["selection_value"]) + session.observe(result["selection_value"], **result) evaluations.append( { **result, @@ -97,14 +97,15 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): "env_steps": env_steps, } ) - session.metrics.append( - { + session.record_training( + final=final, + **{ **record, "transaction": session.transaction, "env_steps": env_steps, "policy_version": actor.ppo_policy_version, "episodes": episodes, - } + }, ) if session.should_checkpoint(final): environment_state = None diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py index f5f82bbd..f06f640b 100644 --- a/src/sequifier/rl/ppo_buffer.py +++ b/src/sequifier/rl/ppo_buffer.py @@ -2,7 +2,6 @@ import json from dataclasses import asdict, dataclass -from pathlib import Path from typing import cast import torch @@ -142,12 +141,16 @@ def archive(self, path, *, include_scores): "frames": [frame.state_dict() for frame in self.frames], "decisions": decisions, } - write_json_atomic(path, payload) + from sequifier.trajectory.archives import collection_archive + + write_json_atomic(path, collection_archive("ppo", payload)) def load_collection_record(path): """Read the independent archival format without changing v1 episode readers.""" - payload = json.loads(Path(path).read_text(encoding="utf-8")) + from sequifier.trajectory.archives import read_collection_archive + + payload = read_collection_archive(path, "ppo") if payload.get("format") != "sequifier.ppo-collection" or payload.get( "version" ) not in {1, 2}: diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 7261f454..dfe94238 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -56,6 +56,9 @@ def validate_output_paths(config, inputs): ) require_file_destinations(outputs) validate_path_conflicts(outputs, protected, label="policy output") + from sequifier.io.output_paths import validate_training_destinations + + validate_training_destinations(config, config.checkpoint_path, protected) require_directory_destinations(archives) validate_path_conflicts(archives, protected | set(outputs), label="archive") diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index cee4e5ed..31f4ff73 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -1,11 +1,21 @@ """Policy recovery with artifact-backed, authoritative committed phase boundaries.""" import math +import uuid from dataclasses import asdict, dataclass, field, fields from pathlib import Path import torch +from sequifier.artifacts.checkpoint_envelope import ( + checkpoint_envelope, + recovery_payload, +) +from sequifier.artifacts.model_artifact import ( + ModelArtifact, + ModelArtifactMetadata, + ModelExecutionConfig, +) from sequifier.artifacts.phase import PhaseArtifacts from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save, write_json_atomic @@ -57,6 +67,7 @@ class RunCheckpoint: phase_state: dict | None = None continuation: dict | None = None stopped: bool = False + run_id: str = field(default_factory=lambda: uuid.uuid4().hex) @property def results(self): @@ -75,7 +86,9 @@ def load(cls, path, config, plan, data_identity): if config.resume.policy == "required": raise FileNotFoundError(path) return None - payload = torch.load(path, map_location="cpu", weights_only=False) + payload = recovery_payload( + torch.load(path, map_location="cpu", weights_only=False), "policy" + ) if ( not isinstance(payload, dict) or payload.get("artifact_type") != "sequifier_policy_run" @@ -96,6 +109,7 @@ def load(cls, path, config, plan, data_identity): f.name: payload[f.name] for f in fields(cls) if f.name not in {"representation", "completed"} + and f.name in payload }, representation=ActorRepresentation(payload["representation"]), completed=completed, @@ -209,20 +223,30 @@ def save(self, path, device, state=None): """Atomically commit the run and its algorithm-owned recovery state.""" self.random = capture_rng_state(device) self.phase_state = state + model = ModelArtifact( + 3, + ModelExecutionConfig(self.representation.values["execution"]), + state["networks"]["actor"] if state else self.incoming_actor, + ModelArtifactMetadata(), + ) atomic_save( path, - { - "artifact_type": "sequifier_policy_run", - "format_version": 7, - **{ - f.name: getattr(self, f.name) - for f in fields(self) - if f.name not in {"representation", "completed"} + checkpoint_envelope( + "policy", + { + "artifact_type": "sequifier_policy_run", + "format_version": 7, + **{ + f.name: getattr(self, f.name) + for f in fields(self) + if f.name not in {"representation", "completed"} + }, + "representation": self.representation.canonical, + "representation_fingerprint": self.representation.identity, + "completed": [record.as_dict() for record in self.completed], }, - "representation": self.representation.canonical, - "representation_fingerprint": self.representation.identity, - "completed": [record.as_dict() for record in self.completed], - }, + model, + ), ) def finish_phase(self, actor, result): @@ -244,10 +268,31 @@ def finish_phase(self, actor, result): self.incoming_actor = weights self.phase_state = None - def write_reports(self): + def phase_reports(self, checkpoint_path): + from sequifier.artifacts.reports import phase_report + + return [ + phase_report( + model_path=result.provenance["actor_path"], + checkpoint_path=checkpoint_path, + **result.as_dict(), + ) + for result in self.results + ] + + def write_reports(self, checkpoint_path): """Reports are derived views; deleting one does not invalidate recovery.""" for result in self.results: - write_json_atomic(result.provenance["report_path"], result.as_dict()) + from sequifier.artifacts.reports import phase_report + + write_json_atomic( + result.provenance["report_path"], + phase_report( + model_path=result.provenance["actor_path"], + checkpoint_path=checkpoint_path, + **result.as_dict(), + ), + ) def validate_finite_state(value, field): diff --git a/src/sequifier/rl/run_config.py b/src/sequifier/rl/run_config.py index 18e2ca5b..fc1cffb9 100644 --- a/src/sequifier/rl/run_config.py +++ b/src/sequifier/rl/run_config.py @@ -14,6 +14,7 @@ class ResolvedPolicyRun(BaseModel): _prepared_inputs: Any = PrivateAttr(default=None) + _metrics_service: Any = PrivateAttr(default=None) model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) project_root: str @@ -32,6 +33,7 @@ class ResolvedPolicyRun(BaseModel): units: dict[str, str] phases: list[PolicyPhase] latest_checkpoint_interval_minutes: float | None = None + log_interval_iterations: int = Field(default=10, gt=0) source_validation_fractions: dict[str, float] = Field(default_factory=dict) validation_fraction: float episode_cache_size: int diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index fed4c53d..546efbad 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -292,6 +292,18 @@ def __init__(self, context, *, modules=None, frozen=None): self.sampler = torch.Generator().manual_seed(context.run.seed + context.index) self.state = context.resume_state or {} self.transaction = self.state.get("transaction", 0) + phase = context.phase + self.iteration_budget = ( + phase.iterations + if hasattr(phase, "iterations") + else (phase.total_env_steps + phase.rollout_env_steps - 1) + // phase.rollout_env_steps + if hasattr(phase, "total_env_steps") + else (phase.total_groups + phase.groups_per_iteration - 1) + // phase.groups_per_iteration + if hasattr(phase, "total_groups") + else context.step_budget or 1 + ) self.stopping = StoppingState(**self.state.get("stopping", {})) self.initial_evaluated = self.state.get("initial_evaluated", False) self._count_stopping = True @@ -392,7 +404,16 @@ def should_evaluate(self, final): self.snapshot_due() and self.context.phase.evaluation.on_snapshot ) - def observe(self, metric): + def observe(self, metric, **record): + if self.context.run._metrics_service is not None: + values = {**record, self.selection.monitor: metric} + self.context.run._metrics_service.record( + self, + values, + validation=True, + primary=self.selection.monitor, + initial=self.transaction == 0, + ) self.selection.observe(self.context.actor, metric, self.transaction) if self._count_stopping: self.stopping.observe( @@ -403,8 +424,13 @@ def observe(self, metric): ) def record_evaluation(self, metric, **record): - self.observe(metric) + self.observe(metric, **record) + self.metrics.append({"transaction": self.transaction, **record}) + + def record_training(self, *, final=False, **record): self.metrics.append({"transaction": self.transaction, **record}) + if self.context.run._metrics_service is not None: + self.context.run._metrics_service.record(self, record, final=final) def optimization_state(self, *, include_steps=False): return { @@ -481,6 +507,8 @@ def checkpoint(self, final, **extra): self.last_snapshot_time = self.last_checkpoint_time def finish(self, clocks, provenance): + if self.context.run._metrics_service is not None: + self.context.run._metrics_service.flush(self) next_index = self.context.index + 1 preserve = ( next_index < len(self.context.run.phases) diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index c7c511f9..a0b9b3e7 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -73,6 +73,7 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin session.step() session.transaction += 1 final = batches.epoch == phase.epochs + session.record_training(final=final, training_action_nll=total) if session.should_evaluate(final): score = evaluate_imitation( actor, data.validation, phase.evaluation.batch_size diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index fd38d7d3..4366dcb7 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -169,6 +169,22 @@ def build( *, initial_artifact=None, ) -> TrainingRun: + from sequifier.artifacts.run_checkpoint import ( + checkpoint_path as output_checkpoint_path, + ) + from sequifier.io.output_paths import ( + resolve_training_output_paths, + supervised_input_paths, + validate_training_destinations, + ) + + resolve_training_output_paths(config) + protected = supervised_input_paths(config) + if initial_artifact is not None: + protected.add(Path(initial_artifact).resolve()) + validate_training_destinations( + config, output_checkpoint_path(config), protected + ) precision = PrecisionPolicy.from_settings( config.runtime_settings, execution.device ) diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 3771d588..00f8c502 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -87,6 +87,15 @@ def train_worker( ) -> None: """Initialize one execution environment and run the composed runtime.""" + from sequifier.artifacts.run_checkpoint import checkpoint_path + from sequifier.io.output_paths import ( + supervised_input_paths, + validate_training_destinations, + ) + + validate_training_destinations( + config, checkpoint_path(config), supervised_input_paths(config) + ) configure_logger( config.project_root, config.model_name, @@ -170,6 +179,10 @@ def run_training( ) -> Any: """Launch canonical service-composed training locally or across workers.""" + if isinstance(config, TrainModel): + from sequifier.io.output_paths import resolve_training_output_paths + + resolve_training_output_paths(config) if isinstance(config, TrainModel) and any( phase.algorithm != "supervised" for phase in config.training_plan ): diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index 016c7bc7..7eb1083a 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -189,7 +189,7 @@ def __init__( self.compatibility = compatibility or CheckpointCompatibility() def save(self, request: CheckpointRequest, run: Any) -> Path | None: - path = self.store.path_for(request.suffix) + path = self.store.path_for(request.suffix, run.state.phase_index) access = run.optimization.access(run.network) run.integrations.emit( CheckpointSaving(access=access, identity=request.identity, path=path) diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 754c731d..580e637f 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -5,26 +5,27 @@ Completing a phase publishes its selected artifact before advancing the envelope. """ +import uuid from dataclasses import dataclass from pathlib import Path from typing import Any import torch +from sequifier.artifacts.checkpoint_envelope import ( + checkpoint_envelope, + recovery_payload, +) from sequifier.artifacts.model_artifact import ( build_model_artifact, read_execution_artifact, save_model_artifact, ) from sequifier.artifacts.phase import PhaseArtifacts, verify_artifact +from sequifier.artifacts.reports import phase_report, publish_run_report, run_report from sequifier.config.train_config import SequifierConfig, resolve_sequifier_config from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save, write_json_atomic -from sequifier.io.output_paths import ( - contains_path, - unique_paths, - validate_path_conflicts, -) from sequifier.trajectory.sources import file_checksum @@ -67,7 +68,13 @@ def _supervised_config(config, phase, index, incoming, resume): training.forward_attention = forward_attention from sequifier.config.components import ResumeConfig - training.resume = ResumeConfig(policy="if_exists" if resume else "never") + native = PhaseArtifacts.for_phase( + config.phase_directory, index, phase.name + ).model.parent + training.resume = ResumeConfig( + policy="if_exists" if resume else "never", + checkpoint_path=str(native / "run.pt"), + ) values = authored.model_dump(mode="python") run = values["run"] run["project_root"] = config.project_root @@ -76,6 +83,12 @@ def _supervised_config(config, phase, index, incoming, resume): run["reproducibility"]["seed"] = config.seed + index run["recovery"]["resume"] = training.resume.model_dump() run["export"]["pt"] = True + run["logging"].pop("log_interval_iterations", None) + run["outputs"] = { + "model_path": str(native / "native" / "model.pt"), + "report_path": str(native / "native" / "report.json"), + "phase_directory": str(native / "native" / "phases"), + } model_values = model.model_dump() model_values["execution"] = { key: getattr(training, key) @@ -109,6 +122,9 @@ def _supervised_config(config, phase, index, incoming, resume): for part_name, part in dataset.parts.items() } resolved = resolve_sequifier_config(authored, metadata) + resolved.model_path = run["outputs"]["model_path"] + resolved.report_path = run["outputs"]["report_path"] + resolved.phase_directory = run["outputs"]["phase_directory"] resolved.ownership = config.ownership resolved.source_validation_fractions = config.source_validation_fractions resolved.normalization_by_interface = config.normalization_by_interface @@ -145,6 +161,15 @@ def _run_supervised(config, phase, index, incoming, destination, resume): from sequifier.training.engine import TrainingEngine resolved = _supervised_config(config, phase, index, incoming, resume) + from sequifier.artifacts.run_checkpoint import checkpoint_path + from sequifier.io.output_paths import ( + supervised_input_paths, + validate_training_destinations, + ) + + validate_training_destinations( + resolved, checkpoint_path(resolved), supervised_input_paths(resolved) + ) configure_logger(config.project_root, resolved.model_name, 0) configure_determinism( config.seed + index, resolved.runtime_settings.enforce_determinism @@ -166,6 +191,7 @@ def _run_supervised(config, phase, index, incoming, destination, resume): run.network.load_state_dict(run.state.best_model_state_dict, strict=True) save_model_artifact(build_model_artifact(run.network, resolved), destination) return { + **run.state.phase_reports[-1], "completion_reason": result.completion_reason, "resolved_seed": config.seed + index, "stop_scope": phase.stopping.on_trigger @@ -205,7 +231,8 @@ def run_learning_plan(config) -> LearningRunResult: resume = resume or ResumeConfig() checkpoint = Path(config.project_root) / ( - resume.checkpoint_path or f"checkpoints/{config.model_name}/learning-run.pt" + resume.checkpoint_path + or f"checkpoints/runs/{config.model_name}/{config.model_name}-latest.pt" ) plan = config.model_dump(mode="json") plan["runtime_settings"].pop("resume", None) @@ -221,9 +248,12 @@ def run_learning_plan(config) -> LearningRunResult: if config.initialize_from else None, "active": False, + "run_id": uuid.uuid4().hex, } if resume.policy != "never" and checkpoint.is_file(): - state = torch.load(checkpoint, map_location="cpu", weights_only=False) + state = recovery_payload( + torch.load(checkpoint, map_location="cpu", weights_only=False), "learning" + ) if ( state.get("artifact_type") != "sequifier_learning_run" or state.get("format_version") != 2 @@ -232,6 +262,8 @@ def run_learning_plan(config) -> LearningRunResult: raise ValueError("learning run checkpoint differs from the requested plan") elif resume.policy == "required": raise FileNotFoundError(checkpoint) + session_id = uuid.uuid4().hex + state.setdefault("run_id", uuid.uuid4().hex) if not 0 <= len(state["completed"]) <= len(phases): raise ValueError("learning run checkpoint exceeds the phase plan") verify_artifact( @@ -240,16 +272,24 @@ def run_learning_plan(config) -> LearningRunResult: "committed incoming model artifact changed", ) # Validate model/source/environment contracts for the complete plan first. - from sequifier.rl.preparation import preflight_environments, prepare_data + from sequifier.rl.preparation import ( + preflight_environments, + prepare_data, + validate_output_paths, + ) from sequifier.rl.prepared_inputs import PreparedTrainingInputs prepared_inputs = PreparedTrainingInputs.for_run(config) dependencies = {} + protected_inputs = set() for index, phase in enumerate(phases): if phase.algorithm == "supervised": resolved = _supervised_config( config, phase, index, state["incoming"], False ) + from sequifier.io.output_paths import supervised_input_paths + + protected_inputs.update(supervised_input_paths(resolved)) for dataset in resolved.dataset_training.values(): for part in dataset.parts.values(): path = str(Path(config.project_root) / part.metadata_config_path) @@ -258,16 +298,18 @@ def run_learning_plan(config) -> LearningRunResult: destination = PhaseArtifacts.for_phase( config.phase_directory, index, phase.name, "model.pt" ).model + if index == 0: + phase_incoming = config.initialize_from + elif index <= len(state["completed"]): + phase_incoming = state["completed"][index - 1]["model_path"] + else: + phase_incoming = state["incoming"] run = _policy_config( - config, - phase, - index, - state["incoming"], - destination, - False, + config, phase, index, phase_incoming, destination, False ) inputs = prepared_inputs.resolve(run) inputs.revalidate(inputs.identity, f"phases.{phase.name}.source_identity") + validate_output_paths(run, inputs) dependencies.update(inputs.identity) preflight_environments(run) prepare_data(run, run.policy, inputs) @@ -276,23 +318,23 @@ def run_learning_plan(config) -> LearningRunResult: if "dependencies" in state and state["dependencies"] != dependencies: raise ValueError("learning plan source identities changed since its checkpoint") state["dependencies"] = dependencies - protected = {Path(path).resolve() for path in dependencies} + protected = {Path(path).resolve() for path in dependencies} | protected_inputs + for phase in phases: + reference = getattr(phase, "reference", None) + if reference is not None and reference.path: + protected.add(Path(reference.path).resolve()) if config.initialize_from: protected.add(Path(config.initialize_from).resolve()) - outputs = unique_paths( - [ - checkpoint, - Path(config.model_path), - Path(config.report_path), - ], - "learning run output paths must be distinct", - ) - validate_path_conflicts( - outputs, protected, label="learning output", check_each_other=False - ) - directory = Path(config.phase_directory).resolve() - if any(contains_path(directory, source) for source in protected): - raise ValueError("phase_directory overlaps a learning input") + from sequifier.io.output_paths import validate_training_destinations + + validate_training_destinations(config, checkpoint, protected) + + def save_boundary(): + model = ( + read_execution_artifact(state["incoming"]) if state["incoming"] else None + ) + atomic_save(checkpoint, checkpoint_envelope("learning", state, model)) + for index in range(len(state["completed"]), len(phases)): if state.get("stopped"): break @@ -310,7 +352,7 @@ def run_learning_plan(config) -> LearningRunResult: resuming = state["active"] and resume.policy != "never" if not state["active"]: state["active"] = True - atomic_save(checkpoint, state) + save_boundary() phase_identity = fingerprint( { "phase": phase.model_dump(mode="json"), @@ -323,7 +365,29 @@ def run_learning_plan(config) -> LearningRunResult: else: runner = _run_supervised if phase.algorithm == "supervised" else _run_policy report = runner(config, phase, index, incoming, destination, resuming) + report = phase_report( + index=index, + name=phase.name, + algorithm=phase.algorithm, + model_path=destination, + checkpoint_path=destination.with_name("run.pt"), + **{ + key: value + for key, value in report.items() + if key + not in { + "index", + "name", + "algorithm", + "model_path", + "checkpoint_path", + "artifact_type", + "format_version", + } + }, + ) artifacts.write_receipt(phase_identity, report) + write_json_atomic(artifacts.report, report) state["stopped"] = report.get("stop_scope") == "end_plan" state["completed"].append( { @@ -338,18 +402,25 @@ def run_learning_plan(config) -> LearningRunResult: incoming_sha256=file_checksum(destination), active=False, ) - atomic_save(checkpoint, state) + save_boundary() final = read_execution_artifact(state["incoming"]) save_model_artifact(final, config.model_path) - write_json_atomic( + publish_run_report( + run_report( + model_path=config.model_path, + checkpoint_path=checkpoint, + phases=[item["report"] for item in state["completed"]], + plan_fingerprint=identity, + run_id=state["run_id"], + session_id=session_id, + source_provenance=dependencies, + initialization={"path": config.initialize_from}, + resume=resume.model_dump(mode="json"), + exports=[config.model_path], + status="complete", + completion_reason="early_stopping" if state.get("stopped") else "completed", + ), config.report_path, - { - "format": "sequifier.learning-run-report", - "version": 1, - "plan_fingerprint": identity, - "phases": state["completed"], - "model_path": config.model_path, - }, ) return LearningRunResult( config.model_path, diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 57647d96..1bdcc077 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -13,7 +13,9 @@ import torch from sequifier.artifacts.backbone_repository import publish_revision -from sequifier.artifacts.manifests import write_manifest +from sequifier.artifacts.model_artifact import build_model_artifact, save_model_artifact +from sequifier.artifacts.phase import PhaseArtifacts +from sequifier.artifacts.reports import phase_report, publish_run_report, run_report from sequifier.artifacts.run_checkpoint import LoaderState from sequifier.artifacts.state_dict import canonicalize_state_dict from sequifier.evaluation.compatibility import compatible_supervised_phases @@ -28,6 +30,7 @@ StepIdentity, ValidationCompleted, ) +from sequifier.io.atomic import write_json_atomic from sequifier.logging_paths import model_log_directory from sequifier.runtime.random_state import RandomState from sequifier.runtime.stopping import StoppingState @@ -185,6 +188,7 @@ def _update_monitor( if not math.isfinite(value): return value, False + run.state.last_validation_metric = value improved = ( value < run.state.best_validation_loss if monitor.direction == "min" @@ -192,6 +196,7 @@ def _update_monitor( ) if improved: run.state.best_validation_loss = value + run.state.best_optimizer_step = run.state.optimizer_step if count_patience: run.state.epochs_without_improvement = 0 state = run.distributed.capture_model_state(run.network) @@ -225,22 +230,78 @@ def _manifest( ) -> None: if run.distributed.rank != 0: return - path = run.checkpoints.store.latest_path.parent / "manifest.json" - write_manifest( - path, - { - "artifact_type": "sequifier_run_manifest", - "format_version": 2, - "run_id": run.state.run_id, - "session_id": run.state.session_id, - "status": status, - "completion_reason": result.completion_reason, - "source_epoch": run.state.epoch, - "exports": [str(path) for path in result.exports], - "backbone_parent_revision_id": run.state.backbone_parent_revision_id, - "backbone_publication": result.publication, + publish_run_report( + run_report( + model_path=run.config.model_path, + checkpoint_path=run.checkpoints.store.latest_path, + phases=run.state.phase_reports, + run_id=run.state.run_id, + session_id=run.state.session_id, + status=status, + completion_reason=result.completion_reason, + source_epoch=run.state.epoch, + exports=[str(path) for path in result.exports] + + [run.config.model_path], + backbone_parent_revision_id=run.state.backbone_parent_revision_id, + backbone_publication=result.publication, + ), + run.config.report_path, + ) + + def _publish_phase(self, run, index, phase): + """Publish the selected portable artifact on the supervised phase boundary.""" + from sequifier.rl.run_state import weight_fingerprint + + last = run.distributed.capture_model_state(run.network) + if run.distributed.rank != 0: + return + selected = ( + (run.state.best_model_state_dict or last) + if phase.handoff == "best" + else last + ) + artifacts = PhaseArtifacts.for_phase( + run.config.phase_directory, index, phase.name + ) + save_model_artifact( + build_model_artifact(run.network, run.config, state_dict=selected), + artifacts.model, + ) + metric = run.state.best_validation_loss + import math + + report = phase_report( + index=index, + name=phase.name, + algorithm="supervised", + model_path=artifacts.model, + checkpoint_path=run.checkpoints.store.latest_path, + handoff=phase.handoff, + monitor=phase.evaluation_monitor.metric + if phase.evaluation_monitor + else None, + direction=phase.evaluation_monitor.direction + if phase.evaluation_monitor + else None, + selected_metric=metric + if math.isfinite(metric) and phase.handoff == "best" + else run.state.last_validation_metric, + selected_transaction=run.state.best_optimizer_step + if phase.handoff == "best" + else run.state.optimizer_step, + selected_weight_hash=weight_fingerprint(selected), + clocks={ + "epochs": run.state.phase_epoch, + "optimizer_steps": run.state.optimizer_step, }, + stop_scope=phase.stopping.on_trigger + if phase.stopping and run.state.stopping_state.get("triggered") + else None, ) + write_json_atomic(artifacts.report, report) + run.state.phase_reports = [ + r for r in run.state.phase_reports if r["index"] != index + ] + [report] def run(self, run: Any) -> RunResult: config = run.config @@ -346,7 +407,25 @@ def flush_metrics( current_batch = sum(state.iterator_positions.values()) current_batches_total = 1 try: + # A checkpoint may have been committed immediately before phase + # publication. Rebuild that derived record before advancing. + if ( + state.phase_epoch_complete + and ( + state.plan_stopped + or state.phase_epoch + >= config.training_plan[state.phase_index].epochs + or state.stopping_state.get("triggered") + ) + and not any( + r["index"] == state.phase_index for r in state.phase_reports + ) + ): + self._publish_phase( + run, state.phase_index, config.training_plan[state.phase_index] + ) for phase_index, phase in enumerate(config.training_plan): + run.metrics.set_phase(phase) if state.plan_stopped: completion_reason = "early_stopping" break @@ -370,6 +449,7 @@ def flush_metrics( state.best_model_state_dict = None state.epochs_without_improvement = 0 state.stopping_state = {} + state.last_validation_metric = None last_snapshot = time.monotonic() last_snapshot_step = state.global_batch_step state.last_snapshot_batch_step = last_snapshot_step @@ -740,6 +820,8 @@ def flush_metrics( ) if stop_requested or phase_stop: break + self._publish_phase(run, phase_index, phase) + run.checkpoints.save(CheckpointRequest("latest", last_identity), run) if stop_requested: break except BaseException: @@ -781,6 +863,16 @@ def flush_metrics( best_export.paths + last_export.paths, publication, ) + if run.distributed.rank == 0: + selected = ( + best_state + if config.training_plan[state.phase_index].handoff == "best" + else last_state + ) + save_model_artifact( + build_model_artifact(run.network, config, state_dict=selected), + config.model_path, + ) self._manifest(run, result) run.integrations.emit( RunCompleted(access=access, completion_reason=completion_reason) diff --git a/src/sequifier/training/metrics.py b/src/sequifier/training/metrics.py index eada6a65..f361115c 100644 --- a/src/sequifier/training/metrics.py +++ b/src/sequifier/training/metrics.py @@ -18,7 +18,13 @@ TOTAL_TARGET = "__total__" +METRICS_SCHEMA_VERSION = 3 + COMMON_FIELDS = [ + "schema_version", + "phase", + "algorithm", + "iteration", "run_id", "session_id", "timestamp_utc", @@ -37,6 +43,8 @@ ] TRAINING_FIELDS = FULL_COMMON_FIELDS + [ + "metric", + "is_primary", "window_batches", "target", "value", @@ -45,6 +53,7 @@ ] TRAINING_CONDENSED_FIELDS = [ + "schema_version", "run_id", "epoch", "batch", @@ -55,6 +64,7 @@ ] VALIDATION_FIELDS = FULL_COMMON_FIELDS + [ + "is_primary", "evaluation_id", "evaluation_kind", "metric", @@ -65,6 +75,7 @@ ] VALIDATION_CONDENSED_FIELDS = [ + "schema_version", "run_id", "epoch", "batch", @@ -146,7 +157,19 @@ def append(self, rows: Iterable[Mapping[str, Any]], *, durable: bool) -> None: return with self.path.open("a", encoding="utf-8", newline="") as file: writer = csv.DictWriter(file, fieldnames=self.fields, extrasaction="raise") - writer.writerows(row_group) + for row in row_group: + row = dict(row) + if "schema_version" in self.fields: + row.setdefault("schema_version", METRICS_SCHEMA_VERSION) + if "metric" in self.fields: + row.setdefault("metric", "loss") + if "is_primary" in self.fields: + row.setdefault( + "is_primary", + row.get("metric") == "loss" + and row.get("target") == TOTAL_TARGET, + ) + writer.writerow(row) file.flush() if durable: os.fsync(file.fileno()) @@ -174,6 +197,9 @@ def __init__( dataset_count=dataset_count, ) self.model_name = model_name + self.phase = "" + self.algorithm = "supervised" + self.iteration = 0 self.dataset_name = dataset_name self.rank = rank self.class_share_columns = tuple(class_share_columns) @@ -216,6 +242,10 @@ def _common( part: str | None = None, ) -> dict[str, Any]: return { + "schema_version": METRICS_SCHEMA_VERSION, + "phase": self.phase, + "algorithm": self.algorithm, + "iteration": self.iteration, "run_id": run_id, "session_id": session_id, "timestamp_utc": timestamp, @@ -448,3 +478,93 @@ def write_validation( if self._class_shares is not None: self._class_shares.append(class_rows, durable=True) return evaluation_id + + def write_observation( + self, + *, + validation, + run_id, + session_id, + epoch, + iteration, + batches_total, + global_step, + values, + primary, + learning_rate, + elapsed_seconds=0.0, + initial=False, + ): + """Use the supervised tables for named algorithm-specific scalar metrics.""" + self.iteration = iteration + common = self._common( + run_id=run_id, + session_id=session_id, + timestamp=_timestamp_utc(), + epoch=epoch, + batch=iteration, + batches_total=max(1, batches_total), + global_step=global_step, + ) + common = { + key: value for key, value in common.items() if key in FULL_COMMON_FIELDS + } + evaluation_id = uuid.uuid4().hex + rows = [] + for name, value in values.items(): + row = { + **common, + "metric": name, + "target": TOTAL_TARGET, + "value": float(value), + "is_primary": name == primary, + "learning_rate": learning_rate, + } + if validation: + row.update( + evaluation_id=evaluation_id, + evaluation_kind="initial" if initial else "iteration_end", + elapsed_seconds=elapsed_seconds, + ) + else: + row.update(window_batches=1, seconds_per_batch=elapsed_seconds) + rows.append(row) + full = self._validation_full if validation else self._training_full + condensed = self._validation if validation else self._training + if full is None: + full = _CsvAppender( + self.validation_full_path if validation else self.training_full_path, + VALIDATION_FIELDS if validation else TRAINING_FIELDS, + ) + if validation: + self._validation_full = full + else: + self._training_full = full + if condensed is None: + condensed = _CsvAppender( + self.validation_path if validation else self.training_path, + VALIDATION_CONDENSED_FIELDS + if validation + else TRAINING_CONDENSED_FIELDS, + ) + if validation: + self._validation = condensed + else: + self._training = condensed + full.append(rows, durable=validation) + fields = ( + VALIDATION_CONDENSED_FIELDS if validation else TRAINING_CONDENSED_FIELDS + ) + condensed.append( + [{key: row.get(key, "") for key in fields} for row in rows], + durable=validation, + ) + if primary in values: + logger.info( + "{} {} | Iteration {} | {}: {}", + self.phase, + "Validation" if validation else "Training", + iteration, + primary, + values[primary], + ) diff --git a/src/sequifier/training/metrics_service.py b/src/sequifier/training/metrics_service.py index 14271122..600a679f 100644 --- a/src/sequifier/training/metrics_service.py +++ b/src/sequifier/training/metrics_service.py @@ -56,6 +56,11 @@ def __init__(self, context: Any, config: Any) -> None: else {} ) + def set_phase(self, phase): + for writer in self.writers.values(): + writer.phase = phase.name + writer.algorithm = phase.algorithm + def record_training(self, result: TrainingMetrics) -> None: if self.context.rank != 0: return diff --git a/src/sequifier/training/state.py b/src/sequifier/training/state.py index 27cb477c..e5ee103e 100644 --- a/src/sequifier/training/state.py +++ b/src/sequifier/training/state.py @@ -20,6 +20,9 @@ class RunState: run_id: str = field(default_factory=lambda: uuid.uuid4().hex) session_id: str = field(default_factory=lambda: uuid.uuid4().hex) stopping_state: dict[str, Any] = field(default_factory=dict) + phase_reports: list[dict[str, Any]] = field(default_factory=list) + last_validation_metric: float | None = None + best_optimizer_step: int | None = None plan_stopped: bool = False initial_evaluated_phases: list[int] = field(default_factory=list) phase_index: int = 0 diff --git a/src/sequifier/trajectory/archives.py b/src/sequifier/trajectory/archives.py new file mode 100644 index 00000000..c264f1c5 --- /dev/null +++ b/src/sequifier/trajectory/archives.py @@ -0,0 +1,38 @@ +"""Collection archive envelopes with algorithm-owned payloads.""" + +import json +from pathlib import Path + + +def collection_archive(algorithm, payload): + return { + "format": "sequifier.collection-archive", + "version": 1, + "algorithm": algorithm, + "collection": payload, + } + + +def read_collection_archive(path, algorithm=None): + payload = json.loads(Path(path).read_text(encoding="utf-8")) + legacy_formats = { + "ppo": "sequifier.ppo-collection", + "grpo": "sequifier.grpo-archive", + } + if isinstance(payload, dict) and payload.get("format") in legacy_formats.values(): + if algorithm is not None and payload["format"] != legacy_formats.get(algorithm): + raise ValueError(f"Expected {algorithm} collection archive") + return payload + if payload.get("format") == "sequifier.collection-archive": + if payload.get("version") != 1: + raise ValueError("Unsupported collection archive envelope version") + if algorithm is not None and payload.get("algorithm") != algorithm: + raise ValueError(f"Expected {algorithm} collection archive") + if payload.get("algorithm") not in {"ppo", "grpo"} or not isinstance( + payload.get("collection"), dict + ): + raise ValueError( + "Collection archive requires a known algorithm and collection mapping" + ) + return payload["collection"] + raise ValueError("Expected a Sequifier collection archive envelope") diff --git a/src/sequifier/visualize_training.py b/src/sequifier/visualize_training.py index 38acf6a6..641ce48a 100644 --- a/src/sequifier/visualize_training.py +++ b/src/sequifier/visualize_training.py @@ -33,6 +33,8 @@ class TrainingMetrics: default_factory=dict ) train_window_batches: dict[float, dict[int, int]] = field(default_factory=dict) + training_label: str = "Training Loss" + validation_label: str = "Validation Loss" class StructuredMetricsParser: @@ -47,14 +49,49 @@ def parse_files(self, training_file: str, validation_file: str) -> TrainingMetri training_rows = self._read_rows(training_file) validation_rows = self._read_rows(validation_file) total_training_rows = [ - row for row in training_rows if row.get("target") == TOTAL_TARGET + row + for row in training_rows + if row.get("target") == TOTAL_TARGET + and ( + row.get("is_primary", "").lower() == "true" + or (not row.get("is_primary") and row.get("metric", "loss") == "loss") + ) ] if not total_training_rows: raise DataContinuityError( f"[{self.model}]: No valid training loss data found." ) run_id = total_training_rows[-1]["run_id"] + total_training_rows = [ + row for row in total_training_rows if row["run_id"] == run_id + ] + last_row = total_training_rows[-1] + policy_rows = last_row.get("algorithm") not in {None, "", "supervised"} + if policy_rows: + phase = last_row.get("phase") + training_metric = last_row.get("metric") + total_training_rows = [ + row + for row in total_training_rows + if row.get("phase") == phase and row.get("metric") == training_metric + ] + validation_rows = [ + row for row in validation_rows if row.get("phase") == phase + ] + primary_validation = [ + row + for row in validation_rows + if row.get("run_id") == run_id + and row.get("is_primary", "").lower() == "true" + ] + validation_metric = ( + primary_validation[-1].get("metric") if primary_validation else None + ) + else: + validation_metric = "loss" metrics = TrainingMetrics() + if total_training_rows[-1].get("algorithm") not in {None, "", "supervised"}: + metrics.training_label = total_training_rows[-1].get("metric", "Training") for row in total_training_rows: if row.get("run_id") != run_id: @@ -83,8 +120,17 @@ def parse_files(self, training_file: str, validation_file: str) -> TrainingMetri metric = row.get("metric") target = row.get("target") value = float(row["value"]) - if metric == "loss" and target == TOTAL_TARGET: + if ( + metric == validation_metric + and (not policy_rows or row.get("is_primary", "").lower() == "true") + ) and target == TOTAL_TARGET: metrics.val_losses[epoch_position] = value + if metric is not None and row.get("algorithm") not in { + None, + "", + "supervised", + }: + metrics.validation_label = metric elif metric == "baseline_loss" and target == TOTAL_TARGET: metrics.baseline_losses[epoch_position] = value elif metric == "loss" and target: @@ -94,12 +140,12 @@ def parse_files(self, training_file: str, validation_file: str) -> TrainingMetri raise DataContinuityError( f"[{self.model}]: No valid validation loss data found." ) - if not metrics.baseline_losses: + if not metrics.baseline_losses and not policy_rows: raise DataContinuityError(f"[{self.model}]: No baseline loss data found.") missing_baselines = sorted( set(metrics.val_losses).difference(metrics.baseline_losses) ) - if missing_baselines: + if missing_baselines and not policy_rows: raise DataContinuityError( f"[{self.model}]: Missing baseline loss data for validation " f"positions {missing_baselines!r}." @@ -119,7 +165,7 @@ def _epoch_position(row: dict[str, str]) -> float: batch = int(row["batch"]) batches_total = int(row["batches_total"]) if row.get("evaluation_kind") == "initial": - return 0.0 + return float(epoch - 1) if row.get("algorithm") != "supervised" else 0.0 if batch > 0 and batches_total > 0: return round(epoch - 1 + batch / batches_total, 8) return float(epoch) @@ -181,7 +227,11 @@ def format_plot_data( """Convert parsed metrics into Plotly-ready arrays.""" val_x = sorted(list(metrics.val_losses.keys())) val_y = [metrics.val_losses[e] for e in val_x] - base_y = [metrics.baseline_losses[e] for e in val_x] + base_y = ( + [metrics.baseline_losses[e] for e in val_x] + if all(e in metrics.baseline_losses for e in val_x) + else [] + ) train_x, train_y = [], [] @@ -259,6 +309,8 @@ def append_chunk() -> None: "train_x": train_x, "train_y": train_y, "var_losses": metrics.var_losses, + "training_label": metrics.training_label, + "validation_label": metrics.validation_label, } @@ -290,11 +342,17 @@ def _generate_single_model_plot( invalid_runs: Optional[dict[str, str]] = None, ) -> None: """Write a single-model training report.""" + policy_metrics = data.get("training_label", "Training Loss") != "Training Loss" + if policy_metrics: + yaxis_type = "linear" has_var_losses = bool(data.get("var_losses")) subplot_titles = ( ("Global Losses", "Normalized Variable Validation Losses") if has_var_losses - else ("Global Losses", "") + else ( + "Training and validation metrics" if policy_metrics else "Global Losses", + "", + ) ) fig = make_subplots(rows=1, cols=2, subplot_titles=subplot_titles) @@ -304,7 +362,7 @@ def _generate_single_model_plot( x=data["val_x"], y=data["val_y"], mode="lines", - name="Validation Loss", + name=data.get("validation_label", "Validation Loss"), hovertemplate=f"{model}
Val Loss: %{{y}}
Epoch: %{{x}}", ), row=1, @@ -315,7 +373,7 @@ def _generate_single_model_plot( x=data["train_x"], y=data["train_y"], mode="lines", - name="Training Loss", + name=data.get("training_label", "Training Loss"), hovertemplate=f"{model}
Train Loss: %{{y}}
Epoch: %{{x}}", ), row=1, @@ -391,6 +449,11 @@ def _generate_multi_model_plot( fig = make_subplots( rows=1, cols=2, subplot_titles=("Validation Losses", "Training Losses") ) + if any( + data.get("training_label", "Training Loss") != "Training Loss" + for data in all_data.values() + ): + yaxis_type = "linear" baseline_val = None colors = pc.qualitative.Plotly From afdf8039305133043700c7be2993a201c6e820d3 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 17 Nov 2026 17:35:53 +0100 Subject: [PATCH 45/71] fix path --- src/sequifier/hyperparameter_search.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sequifier/hyperparameter_search.py b/src/sequifier/hyperparameter_search.py index c35ba51e..337e7251 100644 --- a/src/sequifier/hyperparameter_search.py +++ b/src/sequifier/hyperparameter_search.py @@ -270,6 +270,7 @@ def consume_metrics( evaluation_id = os.path.splitext(os.path.basename(model_path))[0] if config.evaluation_inference_config_path: + inference_model_path = os.path.relpath(model_path, config.run.project_root) evaluation_inference_config_path = config.evaluation_inference_config_path if not os.path.isabs(evaluation_inference_config_path) and not os.path.exists( evaluation_inference_config_path @@ -283,7 +284,7 @@ def consume_metrics( "sequifier", "infer", f"--config-path={evaluation_inference_config_path}", - f"--model-paths={model_path}", + f"--model-paths={inference_model_path}", ], check=True, ) From 5ff782f2fc70b777fd20782277772375f5652588 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 17 Nov 2026 20:56:10 +0100 Subject: [PATCH 46/71] clean up --- src/sequifier/artifacts/model_config.py | 26 ++- src/sequifier/artifacts/model_export.py | 8 +- src/sequifier/config/learning_config.py | 15 ++ src/sequifier/config/sources.py | 17 ++ src/sequifier/config/train_config.py | 144 +++++---------- src/sequifier/config/training_resolution.py | 96 +++------- src/sequifier/model/execution.py | 75 ++++++-- src/sequifier/rl/dpo.py | 30 ++-- src/sequifier/rl/group_collection.py | 190 ++++++++++++++++++++ src/sequifier/rl/grpo.py | 22 +-- src/sequifier/rl/grpo_collector.py | 165 ++++------------- src/sequifier/rl/iql.py | 75 ++++---- src/sequifier/rl/learner.py | 20 +-- src/sequifier/rl/parallel_grpo_collector.py | 161 ++--------------- src/sequifier/rl/policy.py | 25 +-- src/sequifier/rl/ppo.py | 26 +-- src/sequifier/rl/preparation.py | 2 - src/sequifier/rl/run_checkpoint.py | 13 +- src/sequifier/rl/run_state.py | 58 ++++++ src/sequifier/rl/training.py | 30 ++-- src/sequifier/training/coordinator.py | 140 ++++++--------- src/sequifier/training/runtime.py | 8 +- 22 files changed, 629 insertions(+), 717 deletions(-) create mode 100644 src/sequifier/rl/group_collection.py diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 6fcef2a1..dfda2991 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -12,6 +12,7 @@ TrainingRuntimeSettings, ) from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view +from sequifier.model.execution import ModelExecution from sequifier.special_tokens import SPECIAL_TOKEN_IDS from sequifier.typechecking import beartype @@ -22,8 +23,8 @@ def validate_execution_attachment(source: dict, current: dict) -> None: from sequifier.artifacts.model_export import model_execution_config - source_config, _ = resolved_config_from_model_config(source, device="cpu") - current_config, _ = resolved_config_from_model_config(current, device="cpu") + source_config, _ = model_execution_from_model_config(source, device="cpu") + current_config, _ = model_execution_from_model_config(current, device="cpu") source = model_execution_config(source_config) current = model_execution_config(current_config) if source["backbone"]["architecture"] != current["backbone"]["architecture"]: @@ -90,7 +91,22 @@ def resolved_config_from_model_config( device: str, interface_name: str | None = None, ) -> tuple[ResolvedSequifierConfig, str]: - """Build an execution-only config and return its selected interface name.""" + """Public compatibility projection of a portable model's execution facts.""" + execution, selected = model_execution_from_model_config( + values, device=device, interface_name=interface_name, validate=False + ) + return execution.training_config(), selected + + +@beartype +def model_execution_from_model_config( + values: dict, + *, + device: str, + interface_name: str | None = None, + validate: bool = True, +) -> tuple[ModelExecution, str]: + """Resolve portable model execution without inventing datasets or losses.""" interface_values = values.get("interfaces") if not isinstance(interface_values, dict) or not interface_values: @@ -195,7 +211,7 @@ def resolved_config_from_model_config( ): raise ValueError(f"Execution schema contradicts interface {name!r}") resolved_interfaces[name] = resolved - config = execution_config( + config = ModelExecution( ModelSpecModel(backbone=backbone, interfaces=authored_interfaces), resolved_interfaces, global_spec, @@ -204,7 +220,7 @@ def resolved_config_from_model_config( "embedding_layer_names", ["backbone.final_norm"] ), ) - return config, interface_name + return config.validate() if validate else config, interface_name def execution_config(model, interfaces, runtime_settings, *, device="cpu", **options): diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index 793989c8..c902736c 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -7,11 +7,15 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: """Return only configuration required to reconstruct model execution.""" - if not hasattr(training_config, "dataset_training"): - raise TypeError("Model export requires a canonical training config") from sequifier.model.execution import ModelExecution from sequifier.model.execution_schema import ExecutionSchema + if not isinstance(training_config, ModelExecution) and not hasattr( + training_config, "dataset_training" + ): + raise TypeError( + "Model export requires a canonical training config or model execution" + ) execution = ModelExecution.from_config(training_config) interfaces = {} for interface in execution.interfaces.values(): diff --git a/src/sequifier/config/learning_config.py b/src/sequifier/config/learning_config.py index ac3ca123..2ab89ba1 100644 --- a/src/sequifier/config/learning_config.py +++ b/src/sequifier/config/learning_config.py @@ -17,6 +17,21 @@ class DatasetLearningConfig(ConfigModel): loss_weights: Optional[dict[str, float]] = None class_share_log_columns: list[str] = Field(default_factory=list) + def validate_targets(self, targets, usage): + targets = set(targets) + if set(self.criterion) != targets: + raise ValueError(f"{usage}: criterion keys must equal interface targets") + for label in ("class_weights", "loss_weights"): + weights = getattr(self, label) + if weights is not None and weights.keys() - targets: + raise ValueError(f"{usage}: {label} references unknown targets") + if self.loss_weights is not None and all( + self.loss_weights.get(target, 1.0) == 0 for target in targets + ): + raise ValueError( + f"{usage}: at least one target must have positive loss weight" + ) + @model_validator(mode="after") def losses(self): for name in self.criterion.values(): diff --git a/src/sequifier/config/sources.py b/src/sequifier/config/sources.py index 8a41ffa9..c27b7476 100644 --- a/src/sequifier/config/sources.py +++ b/src/sequifier/config/sources.py @@ -9,6 +9,23 @@ from sequifier.typechecking import beartype +def phase_sources(phase): + """All authored references, including held-out and preference sources.""" + refs = list(getattr(phase, "sources", [])) + refs.extend(getattr(phase.evaluation, "sources", None) or []) + if (preference := getattr(phase, "preferences", None)) is not None: + refs.append(preference) + return refs + + +def validate_source(config, source, usage): + name, _, part = source.partition(".") + if name not in config.dataset_training or ( + part and part not in config.dataset_training[name].parts + ): + raise ValueError(f"unknown {usage} source {source!r}") + + class TrainingSourceSpecModel(ConfigModel): model_config = ConfigDict(extra="forbid") diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index acccf153..384e60f5 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -60,7 +60,8 @@ ) from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.config.prediction_config import PredictionDistributionConfig -from sequifier.config.sources import TrainingSourceSpecModel +from sequifier.config.sources import TrainingSourceSpecModel, phase_sources +from sequifier.config.sources import validate_source as _validate_source from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -572,25 +573,12 @@ def validate_unique_names(self): return self -def _validate_source(config, source, usage): - name, _, part = source.partition(".") - if name not in config.dataset_training or ( - part and part not in config.dataset_training[name].parts - ): - raise ValueError(f"unknown {usage} source {source!r}") - - def validate_training_contract(config): if config.model is None and config.initialize_from is None: raise ValueError("a run requires model or initialize_from") for phase in config.training_plan.phases: evaluation = phase.evaluation - refs = list(getattr(phase, "sources", [])) + list( - getattr(evaluation, "sources", None) or [] - ) - preference = getattr(phase, "preferences", None) - if preference is not None: - refs.append(preference) + refs = phase_sources(phase) for ref in refs: _validate_source(config, ref.source, f"phase {phase.name!r}") if phase.algorithm != "supervised" and ( @@ -655,31 +643,7 @@ def validate_training_contract(config): referenced_interfaces.add(dataset.model_interface) interface = model.interfaces[dataset.model_interface] learning = config.dataset_learning(dataset_name) - if set(learning.criterion) != set(interface.target_columns): - raise ValueError( - f"Dataset {dataset_name!r} criterion keys must equal interface " - f"target_columns." - ) - if learning.loss_weights is not None and set(learning.loss_weights) - set( - interface.target_columns - ): - raise ValueError( - f"Dataset {dataset_name!r} loss_weights references unknown targets." - ) - if learning.loss_weights is not None and all( - learning.loss_weights.get(target, 1.0) == 0.0 - for target in interface.target_columns - ): - raise ValueError( - f"Dataset {dataset_name!r} must have at least one target with " - "a positive loss weight." - ) - if learning.class_weights is not None and set(learning.class_weights) - set( - interface.target_columns - ): - raise ValueError( - f"Dataset {dataset_name!r} class_weights references unknown targets." - ) + learning.validate_targets(interface.target_columns, f"Dataset {dataset_name!r}") unreferenced = set(model.interfaces) - referenced_interfaces if unreferenced: warnings.warn( @@ -1052,22 +1016,14 @@ def validate_next_occurrence_phases(self, dataset_interfaces=None): def validate_model_execution_plans(self): """Compile each route while validation errors still retain Pydantic context.""" - from sequifier.model.decoders import resolve_decoding_plan - from sequifier.model.ingestion_compiler import resolve_ingestion_plan + from sequifier.model.execution import ModelExecution - validated_interfaces: set[str] = set() for dataset in self.dataset_training.values(): - if dataset.model_interface in validated_interfaces: - continue - if dataset.interface.depth_layouts: - if any(part.read_format != "pt" for part in dataset.parts.values()): - raise ValueError( - "Preprocessed depth inputs require read_format: pt" - ) - view = interface_build_view(self, dataset.interface) - resolve_ingestion_plan(view) - resolve_decoding_plan(view) - validated_interfaces.add(dataset.model_interface) + if dataset.interface.depth_layouts and any( + part.read_format != "pt" for part in dataset.parts.values() + ): + raise ValueError("Preprocessed depth inputs require read_format: pt") + ModelExecution.from_config(self).validate() return self @property @@ -1556,6 +1512,8 @@ def _interface_semantics(interface: ResolvedModelInterface) -> dict[str, Any]: def resolve_sequifier_config( config: Any, metadata: DatasetMetadata | dict[str, DatasetMetadata], + *, + execution=None, ) -> ResolvedSequifierConfig: """Resolve every dataset part, compatibility contract, and source.""" @@ -1685,12 +1643,22 @@ def resolve_sequifier_config( if first_training_layout is None: first_training_layout = layout assert first_training_layout is not None - interface = _resolve_interface( - dataset_spec.model_interface, - interface_spec, - first_metadata, - config.runtime_settings, - first_training_layout, + interface = ( + execution.dataset_interface( + dataset_spec.model_interface, + interface_spec, + first_metadata, + first_training_layout, + config.runtime_settings, + ) + if execution is not None + else _resolve_interface( + dataset_spec.model_interface, + interface_spec, + first_metadata, + config.runtime_settings, + first_training_layout, + ) ) for part_name, resolved_part in resolved_parts.items(): ref = f"{dataset_name}.{part_name}" @@ -2073,7 +2041,7 @@ class SelectedDatasetPartConfig(SelectedInterfaceConfig): @beartype def interface_build_view( - config: ResolvedSequifierConfig, interface: ResolvedModelInterface + config: Any, interface: ResolvedModelInterface ) -> SelectedInterfaceConfig: """Select one interface for model and objective construction.""" @@ -2173,6 +2141,21 @@ class OwnedModelSpec(ConfigModel): execution: ModelExecutionSettings = Field(default_factory=ModelExecutionSettings) interfaces: dict[str, OwnedModelInterfaceSpec] = Field(default_factory=dict) + @classmethod + def from_execution(cls, execution, declared=None, *, settings=None): + """Project model facts without discarding authored input assertions.""" + values = execution.model.model_dump() + values["execution"] = settings or ( + declared.execution.model_dump(exclude_unset=True) if declared else {} + ) + if declared: + for name, interface in declared.interfaces.items(): + if name in values["interfaces"]: + values["interfaces"][name]["input_contract"] = ( + interface.input_contract + ) + return cls.model_validate(values) + def architecture_model(self): if self.backbone is None: raise ValueError("model backbone has not been resolved") @@ -2268,19 +2251,9 @@ def validate_contract(self): for part in dataset.parts: _identifier(part, "Part name") for phase in self.training_plan.phases: - refs = list(getattr(phase, "sources", [])) + list( - getattr(phase.evaluation, "sources", None) or [] - ) - if phase.algorithm == "dpo": - refs.append(phase.preferences) + refs = phase_sources(phase) for ref in refs: - name, _, part = ref.source.partition(".") - if name not in self.dataset_training or ( - part and part not in self.dataset_training[name].parts - ): - raise ValueError( - f"{phase.name}: unknown dataset source {ref.source!r}" - ) + _validate_source(self, ref.source, f"phase {phase.name!r}") if phase.algorithm == "supervised": for name in {ref.source.partition(".")[0] for ref in refs}: if name not in phase.learning.datasets: @@ -2289,29 +2262,10 @@ def validate_contract(self): ) interface_name = self.dataset_training[name].model_interface if self.model and interface_name in self.model.interfaces: - targets = set( - self.model.interfaces[interface_name].target_columns + phase.learning.datasets[name].validate_targets( + self.model.interfaces[interface_name].target_columns, + f"{phase.name}: learning.datasets.{name}", ) - learning = phase.learning.datasets[name] - if set(learning.criterion) != targets: - raise ValueError( - f"{phase.name}: learning.datasets.{name}.criterion keys must equal interface targets" - ) - for label, weights in ( - ("class_weights", learning.class_weights), - ("loss_weights", learning.loss_weights), - ): - if weights is not None and weights.keys() - targets: - raise ValueError( - f"{phase.name}: {label} references unknown targets" - ) - if learning.loss_weights is not None and all( - learning.loss_weights.get(target, 1.0) == 0 - for target in targets - ): - raise ValueError( - f"{phase.name}: at least one target must have positive loss weight" - ) if any(phase.algorithm != "supervised" for phase in self.training_plan.phases): execution = self.run.execution unsupported_execution = execution.model_fields_set & { diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 2140e99c..53290ff7 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -294,10 +294,8 @@ def _supervised_subset(config): def resolve_training_run(authored: SequifierConfig): from sequifier.artifacts.model_artifact import read_execution_artifact - from sequifier.artifacts.model_config import ( - execution_config, - resolved_config_from_model_config, - ) + from sequifier.artifacts.model_config import model_execution_from_model_config + from sequifier.model.execution import ModelExecution from sequifier.rl.attachment import apply_binding, resolve_binding from sequifier.rl.prepared_inputs import PreparedTrainingInputs @@ -309,7 +307,7 @@ def resolve_training_run(authored: SequifierConfig): source = None if config.initialize_from: config.initialize_from = str((root / config.initialize_from).resolve()) - source, _ = resolved_config_from_model_config( + source, _ = model_execution_from_model_config( read_execution_artifact(config.initialize_from).model_config.values, device=config.run.execution.device, ) @@ -317,18 +315,7 @@ def resolve_training_run(authored: SequifierConfig): from sequifier.artifacts.model_config import extend_prediction_distributions extend_prediction_distributions(source, config.model) - declared = config.model - values = source.model.model_dump() - values["execution"] = ( - declared.execution.model_dump(exclude_unset=True) if declared else {} - ) - if declared: - for name, interface in declared.interfaces.items(): - if name in values["interfaces"]: - values["interfaces"][name]["input_contract"] = ( - interface.input_contract.model_dump() - ) - config.model = OwnedModelSpec.model_validate(values) + config.model = OwnedModelSpec.from_execution(source, config.model) for field in ( "context_length", "training_objective", @@ -446,24 +433,11 @@ def resolve_training_run(authored: SequifierConfig): interfaces[name] = _resolve_interface( name, interface, value, training, value.storage_layout ) - resolved = execution_config( - config.model.architecture_model(), - interfaces, - training, - device=config.run.execution.device, - project_root=str(root), - model_name=config.run.model_name, - seed=config.run.reproducibility.seed, - ) + execution = ModelExecution(config.model.architecture_model(), interfaces, training) for name, binding in config.policy_bindings.items(): if name not in interfaces: raise ValueError(f"policy_bindings: unknown interface {name!r}") - fitted = resolve_binding(resolved, name, binding) - apply_binding(resolved, name, fitted) - resolved.policy_bindings[name] = fitted - for name, interface in interfaces.items(): - if interface.policy is not None: - resolved.policy_bindings.setdefault(name, interface.policy) + apply_binding(execution, name, resolve_binding(execution, name, binding)) sequences, preferences = TrainingSources(config, root).resolve() for phase in config.training_plan.phases: @@ -478,7 +452,7 @@ def resolve_training_run(authored: SequifierConfig): continue name = phase_interface(config, phase) phase.model_interface = name - if name not in resolved.policy_bindings: + if interfaces[name].policy is None: raise ValueError( f"{phase.name}: interface {name!r} requires a policy binding" ) @@ -489,10 +463,7 @@ def resolve_training_run(authored: SequifierConfig): getattr(phase, "environment", None), getattr(getattr(phase, "evaluation", None), "environment", None), ): - if ( - environment - and environment.units != resolved.policy_bindings[name].units - ): + if environment and environment.units != interfaces[name].policy.units: raise ValueError( f"{phase.name}: environment units differ from policy units" ) @@ -513,15 +484,15 @@ def resolve_training_run(authored: SequifierConfig): raise ValueError("optimizer continuation cannot change model_interface") if all(phase.algorithm == "supervised" for phase in config.training_plan.phases): subset = _supervised_subset(config).model_copy(update={"initialize_from": None}) - subset = SequifierConfig.model_validate(subset.model_dump()) - native = resolve_sequifier_config(subset, part_metadata) - for dataset in native.dataset_training.values(): - previous = interfaces[dataset.model_interface] - dataset.interface.policy = previous.policy - dataset.interface.prediction_distributions = ( - previous.prediction_distributions - ) - resolved = native + resolved = resolve_sequifier_config(subset, part_metadata, execution=execution) + else: + # The public resolved training schema remains the serialization boundary. + resolved = execution.training_config( + device=config.run.execution.device, + project_root=str(root), + model_name=config.run.model_name, + seed=config.run.reproducibility.seed, + ) resolved.ownership = config.model_dump(mode="json") resolved.source_validation_fractions = _source_fractions(config) resolved.normalization_by_interface = { @@ -537,30 +508,11 @@ def resolve_training_run(authored: SequifierConfig): resolved._prepared_inputs = prepared_inputs resolved.authored = config resolved.initialize_from = config.initialize_from - resolved.model_path = str( - ( - root - / (config.run.outputs.model_path or f"models/{config.run.model_name}.pt") - ).resolve() - ) - resolved.report_path = str( - ( - root - / ( - config.run.outputs.report_path - or f"outputs/training/{config.run.model_name}/report.json" - ) - ).resolve() - ) - resolved.phase_directory = str( - ( - root - / ( - config.run.outputs.phase_directory - or f"outputs/training/{config.run.model_name}/phases" - ) - ).resolve() - ) + from sequifier.io.output_paths import resolve_training_output_paths + + for name in ("model_path", "report_path", "phase_directory"): + setattr(resolved, name, getattr(config.run.outputs, name)) + resolve_training_output_paths(resolved) resolved.sequence_sources = sequences resolved.preference_sources = preferences if any(phase.algorithm != "supervised" for phase in config.training_plan.phases): @@ -604,7 +556,7 @@ def policy_run_config( ): """Select resolved inputs for one native policy lifecycle.""" from sequifier.artifacts.model_artifact import read_execution_artifact - from sequifier.artifacts.model_config import resolved_config_from_model_config + from sequifier.artifacts.model_config import model_execution_from_model_config from sequifier.artifacts.model_export import model_execution_config from sequifier.config.components import ResumeConfig from sequifier.rl.attachment import apply_binding, resolve_binding @@ -618,7 +570,7 @@ def policy_run_config( execution = config initialization = incoming if incoming is not None else config.initialize_from if initialization: - execution, _ = resolved_config_from_model_config( + execution, _ = model_execution_from_model_config( read_execution_artifact(initialization).model_config.values, device=config.device, interface_name=name, diff --git a/src/sequifier/model/execution.py b/src/sequifier/model/execution.py index 63ce92a6..b8553421 100644 --- a/src/sequifier/model/execution.py +++ b/src/sequifier/model/execution.py @@ -1,21 +1,65 @@ """Model execution facts, independent of dataset ownership and learning phases.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any -@dataclass(frozen=True) +@dataclass class ModelExecution: model: Any interfaces: dict runtime_settings: Any + device: str = "cpu" + project_root: str = "." + model_name: str = "loaded-model" + seed: int = 0 + embedding_layer_names: list[str] = field( + default_factory=lambda: ["backbone.final_norm"] + ) @classmethod def from_config(cls, config): + if isinstance(config, cls): + return config interfaces = {} for dataset in config.dataset_training.values(): interfaces.setdefault(dataset.model_interface, dataset.interface) - return cls(config.model, interfaces, config.runtime_settings) + execution = cls(config.model, interfaces, config.runtime_settings) + for name in ( + "device", + "project_root", + "model_name", + "seed", + "embedding_layer_names", + ): + setattr(execution, name, getattr(config, name, getattr(execution, name))) + return execution + + def interface(self, name): + return self.interfaces[name] + + def validate(self): + from sequifier.config.train_config import interface_build_view + from sequifier.model.decoders import resolve_decoding_plan + from sequifier.model.ingestion_compiler import resolve_ingestion_plan + + for interface in self.interfaces.values(): + view = interface_build_view(self, interface) + resolve_ingestion_plan(view) + resolve_decoding_plan(view) + return self + + def dataset_interface(self, name, spec, metadata, storage_layout, runtime_settings): + """Resolve a storage view while retaining model-owned task parameters.""" + from sequifier.config.train_config import _resolve_interface + + interface = _resolve_interface( + name, spec, metadata, runtime_settings, storage_layout + ) + previous = self.interface(name) + interface.policy = previous.policy + interface.prediction_distributions = dict(previous.prediction_distributions) + return interface def dataset(self, name): """Construct the resolved loss and interface view needed by execution.""" @@ -34,8 +78,8 @@ def dataset(self, name): class_share_log_columns=[], ) - def training_config(self, *, device="cpu", **options): - """Construct the resolved execution configuration.""" + def training_config(self, *, device=None, **options): + """Adapt execution to the public, serialized training schema at its boundary.""" from sequifier.config.train_config import ResolvedSequifierConfig return ResolvedSequifierConfig( @@ -43,16 +87,16 @@ def training_config(self, *, device="cpu", **options): model=self.model, dataset_training={name: self.dataset(name) for name in self.interfaces}, **{ - "project_root": ".", - "model_name": "loaded-model", - "seed": 0, - "device": device, + "project_root": self.project_root, + "model_name": self.model_name, + "seed": self.seed, + "device": self.device if device is None else device, "training_plan": [], "export_generative_model": True, "export_embedding_model": False, "export_onnx": False, "export_pt": False, - "embedding_layer_names": ["backbone.final_norm"], + "embedding_layer_names": list(self.embedding_layer_names), **options, }, ) @@ -68,13 +112,6 @@ def retain_interfaces(self, config): key = name while key in config.dataset_training: key += "_unused" - selected = type(self)( - self.model, - {name: self.interfaces[name]}, - config.runtime_settings, - ) - config.dataset_training[key] = ( - selected.training_config() - .dataset_training[name] - .model_copy(update={"name": key}) + config.dataset_training[key] = self.dataset(name).model_copy( + update={"name": key} ) diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index 6e158f8b..c69da68e 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -81,13 +81,14 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): or session.steps["actor"] != session.transaction ): raise ValueError("DPO sampler position disagrees with optimizer transaction") - if session.should_evaluate_initial(): - initial = evaluate_dpo(actor, reference, data.validation, phase) - session.record_evaluation( - initial["dpo_loss"], - initial=True, - **{f"validation_{key}": val for key, val in initial.items()}, + + def evaluate(**record): + session.evaluate_validation( + lambda: evaluate_dpo(actor, reference, data.validation, phase), **record ) + + if session.should_evaluate_initial(): + evaluate(initial=True) for update in range(session.transaction + 1, phase.iterations + 1): if session.stopping.triggered: break @@ -115,17 +116,12 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): ): (sign * coefficient * chunk.sum()).backward() session.step() - session.transaction = update - final = update == phase.iterations - session.record_training(final=final, training_dpo_loss=loss_total) - if session.should_evaluate(final): - held_out = evaluate_dpo(actor, reference, data.validation, phase) - session.record_evaluation( - held_out["dpo_loss"], - training_dpo_loss=loss_total, - **{f"validation_{key}": value for key, value in held_out.items()}, - ) - session.checkpoint(final, batch_order=batches.state_dict()) + session.complete_update( + final=update == phase.iterations, + training={"training_dpo_loss": loss_total}, + evaluate=evaluate, + checkpoint={"batch_order": batches.state_dict()}, + ) return session.finish( {"optimizer_steps": session.steps["actor"]}, {"data": data.contract, "reference_weight_hash": context.reference_identity}, diff --git a/src/sequifier/rl/group_collection.py b/src/sequifier/rl/group_collection.py new file mode 100644 index 00000000..ce13a901 --- /dev/null +++ b/src/sequifier/rl/group_collection.py @@ -0,0 +1,190 @@ +"""Collection budgets, trajectory ownership and sealing shared by GRPO schedulers.""" + +from dataclasses import dataclass, field, replace + +import torch + +from sequifier.fingerprints import fingerprint +from sequifier.io.image_feature_cache import IMAGE_CACHE_PREFIX +from sequifier.rl.collection import PolicyCursor, stream_seed +from sequifier.rl.environment import OnlineStepResult, validate_start_token +from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer +from sequifier.rl.history import _json_payload_bytes + + +class GroupCollectionError(RuntimeError): + def __init__(self, message, executed_actions): + super().__init__( + f"{message}; discarded {executed_actions} uncommitted environment actions" + ) + self.executed_actions = executed_actions + + +@dataclass +class GroupTrajectory: + group: int + sample: int + token: object + cursor: PolicyCursor + frames: list = field(default_factory=list) + actions: list = field(default_factory=list) + rewards: list = field(default_factory=list) + old_scores: list = field(default_factory=list) + reference_scores: list = field(default_factory=list) + + def append( + self, frame, decision, reference_score, result: OnlineStepResult, reward_scale + ): + reward = float(result.reward) * reward_scale + if not torch.isfinite( + torch.tensor( + [reward, reference_score, decision.log_probability], dtype=torch.float32 + ) + ).all(): + raise ValueError("non-finite GRPO reward or fixed action score") + self.frames.append(frame) + self.actions.append(decision.action) + self.rewards.append(reward) + self.old_scores.append(decision.log_probability) + self.reference_scores.append(reference_score) + + def seal(self, generation, result: OnlineStepResult): + score = float(torch.tensor(self.rewards, dtype=torch.float32).sum()) + if not torch.isfinite(torch.tensor(score)): + raise ValueError("non-finite FP32 GRPO trajectory score") + return GRPOTrajectory( + self.group, + self.sample, + self.token, + self.cursor.history.current.episode_id, + tuple(self.frames), + self.cursor.actor.config.context_length, + tuple(self.actions), + tuple(self.rewards), + tuple(self.old_scores), + tuple(self.reference_scores), + "terminated" + if result.terminated + else "truncated" + if result.truncated + else "max_steps", + score, + 0.0, + generation, + ) + + +class GroupCollection: + def __init__(self, context, adapter, count, generation, env_steps, restart_stream): + self.context, self.adapter = context, adapter + self.phase, self.generation = context.phase, generation + bound = count * self.phase.group_size * self.phase.max_steps + if ( + self.phase.max_env_steps is not None + and env_steps + bound > self.phase.max_env_steps + ): + raise ValueError( + "complete GRPO groups exceed max_env_steps (conservative group_size * max_steps bound)" + ) + if ( + bound * (2048 + 128 * len(context.actor.interface.input_columns)) + > self.phase.max_buffer_bytes + ): + raise ValueError("complete GRPO groups exceed the minimum buffer estimate") + self.seed = stream_seed( + context.run.seed, f"grpo-phase-{context.index}-restart-{restart_stream}", 0 + ) + self.used, self.executed = 0, 0 + self.initial = {} + + def __enter__(self): + return self + + def __exit__(self, kind, error, traceback): + if error is not None and isinstance(error, Exception): + raise GroupCollectionError(str(error), self.executed) from error + + def account(self, size): + self.used += size + if self.used > self.phase.max_buffer_bytes: + raise ValueError("GRPO collection exceeded max_buffer_bytes") + + def start(self, group): + token = validate_start_token( + self.adapter.create_group_start( + stream_seed(self.seed, "group-start", group) + ) + ) + self.account(_json_payload_bytes(token) * self.phase.group_size + 1024) + return token + + def trajectory(self, group, sample, token, *, deferred=False): + cursor = PolicyCursor( + self.context.actor, + self.adapter, + self.context.run.units, + stream_seed(self.seed, f"group-{group}-actions", sample), + defer_device_transfer=deferred, + ) + return GroupTrajectory(group, sample, token, cursor) + + def environment_seed(self, group, sample): + return stream_seed(self.seed, f"group-{group}-environment", sample) + + def validate_initial(self, trajectory): + cursor = trajectory.cursor + inputs = cursor.inputs + static = set(cursor.actor.config.static_columns) + # reset() has one valid position. Include the full shape while avoiding + # expansion of context_length copies of padded image pixels. + identity = fingerprint( + { + "features": { + key: { + "shape": list(value.shape), + "values": (value if key in static else value[:, -1:]).tolist(), + } + for key, value in inputs.inputs.items() + }, + "metadata": { + key: value.tolist() + for key, value in inputs.metadata.items() + if not key.startswith(IMAGE_CACHE_PREFIX) + }, + "units": dict(cursor.units), + } + ) + if ( + trajectory.group in self.initial + and self.initial[trajectory.group] != identity + ): + raise ValueError( + "GRPO group members have different canonical initial histories or units" + ) + self.initial[trajectory.group] = identity + + def seal_group(self, trajectories): + if len(trajectories) != self.phase.group_size: + raise ValueError("GRPO cannot seal an incomplete group") + trajectories = sorted(trajectories, key=lambda value: value.sample_index) + scores = torch.tensor([t.score for t in trajectories], dtype=torch.float32) + mean, spread = scores.mean(), scores.std(unbiased=False) + if not torch.isfinite(mean) or not torch.isfinite(spread): + raise ValueError( + "non-finite FP32 GRPO group mean or population standard deviation" + ) + advantages = (scores - mean) / (spread + self.phase.advantage_epsilon) + if not torch.isfinite(advantages).all(): + raise ValueError("non-finite GRPO group advantages") + return tuple( + replace(t, advantage=float(a)) for t, a in zip(trajectories, advantages) + ) + + def finish(self, groups): + return SealedGroupBuffer( + groups, + self.generation, + self.phase.group_size, + self.context.reference_identity, + self.used, + ), self.executed diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 1f976334..034bc33e 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -23,7 +23,6 @@ ) from sequifier.rl.grpo_collector import collect_groups from sequifier.rl.grpo_update import update_grpo -from sequifier.rl.ppo_evaluation import evaluate_policy from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal from sequifier.trajectory.archives import collection_archive @@ -101,25 +100,8 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): ) resources.callback(workers.close) - def evaluate(): - result = evaluate_policy(actor, phase.evaluation, forbidden_adapter=adapter) - score = float( - torch.tensor( - result["score_mean"] * phase.reward_scale, dtype=torch.float32 - ) - ) - if not torch.isfinite(torch.tensor(score)): - raise ValueError("GRPO evaluation score exceeds finite FP32 range") - session.observe( - score, - **{key: value for key, value in result.items() if key != "score_mean"}, - ) - evaluations.append( - {**result, "score_mean": score, "transaction": session.transaction} - ) - if session.should_evaluate_initial(): - evaluate() + session.evaluate_online(adapter, evaluations, initial=True) while groups < phase.total_groups and not session.stopping.triggered: count = min(phase.groups_per_iteration, phase.total_groups - groups) collection_start = perf_counter() @@ -160,7 +142,7 @@ def evaluate(): ) evaluation_start = perf_counter() if session.should_evaluate(final): - evaluate() + session.evaluate_online(adapter, evaluations) evaluation_seconds = perf_counter() - evaluation_start scores = [t.score for t in buffer.trajectories] record.update( diff --git a/src/sequifier/rl/grpo_collector.py b/src/sequifier/rl/grpo_collector.py index 7b51ef53..805525f1 100644 --- a/src/sequifier/rl/grpo_collector.py +++ b/src/sequifier/rl/grpo_collector.py @@ -1,25 +1,14 @@ """Sequential same-start groups with independent action/environment RNG streams.""" -import json -from dataclasses import replace - import torch -from sequifier.fingerprints import fingerprint from sequifier.io.actions import action_batch -from sequifier.io.image_feature_cache import IMAGE_CACHE_PREFIX -from sequifier.rl.collection import PolicyCursor, stream_seed from sequifier.rl.environment import require_group_start, validate_start_token -from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer -from sequifier.rl.history import frame_storage_bytes - - -class GroupCollectionError(RuntimeError): - def __init__(self, message, executed_actions): - super().__init__( - f"{message}; discarded {executed_actions} uncommitted environment actions" - ) - self.executed_actions = executed_actions +from sequifier.rl.group_collection import ( # noqa: F401 + GroupCollection, + GroupCollectionError, +) +from sequifier.rl.history import _json_payload_bytes, frame_storage_bytes @torch.no_grad() @@ -49,76 +38,31 @@ def collect_groups( ) actor, reference, phase = context.actor.eval(), context.reference, context.phase require_group_start(adapter) - bound = count * phase.group_size * phase.max_steps - if phase.max_env_steps is not None and env_steps + bound > phase.max_env_steps: - raise ValueError( - "GRPO cannot fit the requested complete groups within max_env_steps (conservative group_size * max_steps bound)" - ) - estimate = bound * (2048 + 128 * len(actor.interface.input_columns)) - if estimate > phase.max_buffer_bytes: - raise ValueError( - f"GRPO complete groups require approximately {estimate} buffer bytes" - ) - groups, executed, used = [], 0, 0 - seed = stream_seed( - context.run.seed, f"grpo-phase-{context.index}-restart-{restart_stream}", 0 - ) - try: + with GroupCollection( + context, adapter, count, generation, env_steps, restart_stream + ) as collection: + groups = [] for group_index in range(start_group, start_group + count): - token = validate_start_token( - adapter.create_group_start( - stream_seed(seed, "group-start", group_index) - ) - ) - used += len(json.dumps(token).encode()) * phase.group_size + 1024 - trajectories, initial_key = [], None + token = collection.start(group_index) + trajectories = [] for sample in range(phase.group_size): - # All samples restore the same opaque condition, then independently seed future randomness. - cursor = PolicyCursor( - actor, - adapter, - context.run.units, - stream_seed(seed, f"group-{group_index}-actions", sample), - ) - episode_id = f"group-{group_index}-sample-{sample}" + state = collection.trajectory(group_index, sample, token) + cursor = state.cursor cursor.reset( adapter.reset_group( validate_start_token(token), - stream_seed(seed, f"group-{group_index}-environment", sample), + collection.environment_seed(group_index, sample), ), - episode_id, + f"group-{group_index}-sample-{sample}", ) - inputs = cursor.inputs - key = fingerprint( - { - "features": {k: v.tolist() for k, v in inputs.inputs.items()}, - "metadata": { - k: v.tolist() - for k, v in inputs.metadata.items() - if not k.startswith(IMAGE_CACHE_PREFIX) - }, - "units": dict(cursor.units), - } - ) - if initial_key is not None and key != initial_key: - raise ValueError( - "GRPO samples have different canonical initial histories or units" - ) - initial_key = key - frames, actions, rewards, old_scores, ref_scores = [], [], [], [], [] - reason = "max_steps" - for step in range(phase.max_steps): + collection.validate_initial(state) + result = None + for _ in range(phase.max_steps): frame = cursor.history.current - used += frame_storage_bytes(frame) + 1024 - if used > phase.max_buffer_bytes: - raise ValueError("GRPO group exceeded max_buffer_bytes") + collection.account(frame_storage_bytes(frame) + 1024) inputs = cursor.inputs decision = cursor.decide(compute_entropy=False) - used += 4 * len( - json.dumps(decision.action, allow_nan=False).encode() - ) - if used > phase.max_buffer_bytes: - raise ValueError("GRPO actions exceeded max_buffer_bytes") + collection.account(4 * _json_payload_bytes(decision.action)) reference_score = float( reference.evaluate_actions( inputs, @@ -127,62 +71,15 @@ def collect_groups( ), )[0] ) - executed += 1 - step_result = cursor.step(decision) - reward = float(step_result.reward) * phase.reward_scale - if not torch.isfinite( - torch.tensor( - [reward, reference_score, decision.log_probability], - dtype=torch.float32, - ) - ).all(): - raise ValueError("non-finite GRPO reward or fixed action score") - frames.append(frame) - actions.append(decision.action) - rewards.append(reward) - old_scores.append(decision.log_probability) - ref_scores.append(reference_score) - if step_result.terminated or step_result.truncated: - reason = "terminated" if step_result.terminated else "truncated" - break - score = float(torch.tensor(rewards, dtype=torch.float32).sum()) - if not torch.isfinite(torch.tensor(score)): - raise ValueError("non-finite GRPO trajectory score") - trajectories.append( - GRPOTrajectory( - group_index, - sample, - token, - episode_id, - tuple(frames), - actor.config.context_length, - tuple(actions), - tuple(rewards), - tuple(old_scores), - tuple(ref_scores), - reason, - score, - 0.0, - generation, + collection.executed += 1 + result = cursor.step(decision) + state.append( + frame, decision, reference_score, result, phase.reward_scale ) - ) - scores = torch.tensor([t.score for t in trajectories], dtype=torch.float32) - mean, spread = scores.mean(), scores.std(unbiased=False) - if not torch.isfinite(mean) or not torch.isfinite(spread): - raise ValueError( - "non-finite FP32 GRPO group mean or population standard deviation" - ) - advantages = (scores - mean) / (spread + phase.advantage_epsilon) - if not torch.isfinite(advantages).all(): - raise ValueError("non-finite GRPO group advantages") - groups.append( - tuple( - replace(t, advantage=float(a)) - for t, a in zip(trajectories, advantages) - ) - ) - return SealedGroupBuffer( - groups, generation, phase.group_size, context.reference_identity, used - ), executed - except Exception as exc: - raise GroupCollectionError(str(exc), executed) from exc + if result.terminated or result.truncated: + break + if result is None: + raise ValueError("GRPO requires a positive trajectory horizon") + trajectories.append(state.seal(generation, result)) + groups.append(collection.seal_group(trajectories)) + return collection.finish(groups) diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index e26340b3..f022bfa7 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -83,22 +83,24 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): raise ValueError("IQL checkpoint exceeds phase update budget") if any(steps != session.transaction for steps in session.steps.values()): raise ValueError("IQL optimizer clocks disagree with committed updates") - if session.should_evaluate_initial(): - initial = _evaluate( - actor, - value, - q1, - q2, - data[1], - phase.evaluation.batch_size, - phase.gamma, - actor.device, - ) - session.record_evaluation( - initial["action_nll"], - initial=True, - **{f"validation_{key}": val for key, val in initial.items()}, + + def evaluate(**record): + session.evaluate_validation( + lambda: _evaluate( + actor, + value, + q1, + q2, + data[1], + phase.evaluation.batch_size, + phase.gamma, + actor.device, + ), + **record, ) + + if session.should_evaluate_initial(): + evaluate(initial=True) for update in range(session.transaction + 1, phase.iterations + 1): if session.stopping.triggered: break @@ -147,38 +149,21 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): session.step() _polyak(targets[0], q1, phase.target_update_rate) _polyak(targets[1], q2, phase.target_update_rate) - session.transaction = update - final = update == phase.iterations - session.record_training( - final=final, - **{f"{role}_loss": float(loss.detach()) for role, loss in losses.items()}, - ) - if session.should_evaluate(final): - held_out = _evaluate( - actor, - value, - q1, - q2, - data[1], - phase.evaluation.batch_size, - phase.gamma, - actor.device, - ) - session.record_evaluation( - held_out["action_nll"], - **{ - f"{role}_loss": float(loss.detach()) - for role, loss in losses.items() - }, - **{f"validation_{key}": val for key, val in held_out.items()}, - advantage_mean=float(advantage.mean()), - weight_mean=float(weight.mean()), - weight_cap_fraction=float( + session.complete_update( + final=update == phase.iterations, + training={ + f"{role}_loss": float(loss.detach()) for role, loss in losses.items() + }, + evaluate=evaluate, + evaluation_record=lambda: { + "advantage_mean": float(advantage.mean()), + "weight_mean": float(weight.mean()), + "weight_cap_fraction": float( (weight >= phase.max_advantage_weight).float().mean() ), - ) - session.checkpoint(final) + }, + ) return session.finish( - {f"{role}_optimizer_steps": steps for role, steps in session.steps.items()}, + session.optimizer_clocks(), {"data": context.data.contract}, ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index d0b58adb..6ecf01eb 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -66,19 +66,13 @@ def _initialize_actor(config): source.execution_config.embedding_layer_names = config.embedding_layer_names source.units = dict(config.units) return source - else: - from sequifier.artifacts.model_config import resolved_config_from_model_config - - execution, _ = resolved_config_from_model_config( - config.execution_model, - device=config.device, - interface_name=config.policy_interface, - ) - return build_actor( - config, - config.policy, - execution.interface(config.policy_interface).selected_columns_statistics, - ) + return build_actor( + config, + config.policy, + config.execution_model["interfaces"][config.policy_interface].get( + "selected_columns_statistics", {} + ), + ) def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: diff --git a/src/sequifier/rl/parallel_grpo_collector.py b/src/sequifier/rl/parallel_grpo_collector.py index 8bda75f7..3ff73262 100644 --- a/src/sequifier/rl/parallel_grpo_collector.py +++ b/src/sequifier/rl/parallel_grpo_collector.py @@ -2,63 +2,18 @@ import math from collections import deque -from dataclasses import dataclass, field, replace import torch -from sequifier.fingerprints import fingerprint from sequifier.io.actions import action_record, decode_action from sequifier.io.batch import collate_model_inputs -from sequifier.io.image_feature_cache import IMAGE_CACHE_PREFIX -from sequifier.rl.collection import PolicyCursor, PolicyDecision, stream_seed +from sequifier.rl.collection import PolicyDecision from sequifier.rl.environment import validate_start_token -from sequifier.rl.grpo_buffer import GRPOTrajectory, SealedGroupBuffer -from sequifier.rl.grpo_collector import GroupCollectionError +from sequifier.rl.group_collection import GroupCollection from sequifier.rl.history import frame_storage_bytes from sequifier.rl.simulator_workers import require_parallel_group_start -@dataclass -class _Trajectory: - group: int - sample: int - token: object - cursor: PolicyCursor - frames: list = field(default_factory=list) - actions: list = field(default_factory=list) - rewards: list = field(default_factory=list) - old_scores: list = field(default_factory=list) - reference_scores: list = field(default_factory=list) - - def seal(self, generation, result): - score = float(torch.tensor(self.rewards, dtype=torch.float32).sum()) - if not math.isfinite(score): - raise ValueError("non-finite FP32 GRPO trajectory score") - reason = ( - "terminated" - if result.terminated - else "truncated" - if result.truncated - else "max_steps" - ) - return GRPOTrajectory( - group_index=self.group, - sample_index=self.sample, - start_token=self.token, - episode_id=self.cursor.history.current.episode_id, - frames=tuple(self.frames), - context_length=self.cursor.actor.config.context_length, - actions=tuple(self.actions), - rewards=tuple(self.rewards), - old_log_probabilities=tuple(self.old_scores), - reference_log_probabilities=tuple(self.reference_scores), - cutoff_reason=reason, - score=score, - advantage=0.0, - generation=generation, - ) - - @torch.no_grad() def _decide(actor, reference, cursors): inputs = collate_model_inputs([cursor.inputs for cursor in cursors]).to( @@ -92,31 +47,6 @@ def _decide(actor, reference, cursors): return decisions -def _initial_history_identity(cursor): - # reset() has exactly one valid temporal position; all earlier positions - # are canonical zero padding. Compare its encoded values and full shapes - # without expanding context_length copies of padded camera pixels. - inputs = cursor.inputs - static = set(cursor.actor.config.static_columns) - return fingerprint( - { - "features": { - key: { - "shape": list(value.shape), - "values": (value if key in static else value[:, -1:]).tolist(), - } - for key, value in inputs.inputs.items() - }, - "metadata": { - key: value.tolist() - for key, value in inputs.metadata.items() - if not key.startswith(IMAGE_CACHE_PREFIX) - }, - "units": dict(cursor.units), - } - ) - - @torch.no_grad() def collect_parallel_groups( context, @@ -134,34 +64,15 @@ def collect_parallel_groups( context.phase, ) require_parallel_group_start(adapter) - bound = count * phase.group_size * phase.max_steps - if phase.max_env_steps is not None and env_steps + bound > phase.max_env_steps: - raise ValueError("complete GRPO groups exceed max_env_steps") - if ( - bound * (2048 + 128 * len(actor.interface.input_columns)) - > phase.max_buffer_bytes - ): - raise ValueError("complete GRPO groups exceed the minimum buffer estimate") - seed = stream_seed( - context.run.seed, f"grpo-phase-{context.index}-restart-{restart_stream}", 0 - ) - used, executed = 0, 0 - pending, active, completed, initial = deque(), {}, {}, {} - - def account(size): - nonlocal used - used += size - if used > phase.max_buffer_bytes: - raise ValueError("GRPO collection exceeded max_buffer_bytes") - - try: + pending, active, completed = deque(), {}, {} + with GroupCollection( + context, adapter, count, generation, env_steps, restart_stream + ) as collection: from sequifier.rl.history import _json_payload_bytes + account = collection.account for group in range(start_group, start_group + count): - token = validate_start_token( - adapter.create_group_start(stream_seed(seed, "group-start", group)) - ) - account(_json_payload_bytes(token) * phase.group_size + 1024) + token = collection.start(group) completed[group] = [] pending.extend((group, sample, token) for sample in range(phase.group_size)) while pending or active: @@ -170,19 +81,14 @@ def account(size): if slot in active or not pending: continue group, sample, token = pending.popleft() - cursor = PolicyCursor( - actor, - adapter, - context.run.units, - stream_seed(seed, f"group-{group}-actions", sample), - defer_device_transfer=True, + active[slot] = collection.trajectory( + group, sample, token, deferred=True ) - active[slot] = _Trajectory(group, sample, token, cursor) workers.send( slot, "reset_group", validate_start_token(token), - stream_seed(seed, f"group-{group}-environment", sample), + collection.environment_seed(group, sample), ) resetting.append(slot) for slot in resetting: @@ -192,12 +98,7 @@ def account(size): f"group-{state.group}-sample-{state.sample}", ) frame = state.cursor.history.current - identity = _initial_history_identity(state.cursor) - if state.group in initial and initial[state.group] != identity: - raise ValueError( - "GRPO group members have different initial histories" - ) - initial[state.group] = identity + collection.validate_initial(state) # Include all live history before any environment step is issued. account(frame_storage_bytes(frame) + 1024) account( @@ -220,21 +121,13 @@ def account(size): for slot, (decision, _) in decisions.items(): # Count attempted actions conservatively before sending: a failed # send must never hide a potentially executed simulator action. - executed += 1 + collection.executed += 1 workers.send(slot, "step", decision.domain_action) for slot, (decision, ref) in decisions.items(): state = active[slot] - state.frames.append(state.cursor.history.current) + frame = state.cursor.history.current result = state.cursor.accept_step(decision, workers.receive(slot)) - reward = float(result.reward) * phase.reward_scale - if not math.isfinite(reward) or not torch.isfinite( - torch.tensor(reward, dtype=torch.float32) - ): - raise ValueError("GRPO reward exceeds finite FP32 range") - state.actions.append(decision.action) - state.rewards.append(reward) - state.old_scores.append(decision.log_probability) - state.reference_scores.append(ref) + state.append(frame, decision, ref, result, phase.reward_scale) # Account the successor retained by a live cursor, even if the # episode ends or the collector's finite horizon is reached. if result.observation is not None: @@ -246,24 +139,6 @@ def account(size): ): completed[state.group].append(state.seal(generation, result)) del active[slot] - groups = [] - for trajectories in completed.values(): - if len(trajectories) != phase.group_size: - raise ValueError("GRPO cannot seal an incomplete group") - trajectories.sort(key=lambda value: value.sample_index) - scores = torch.tensor([t.score for t in trajectories], dtype=torch.float32) - mean, spread = scores.mean(), scores.std(unbiased=False) - advantages = (scores - mean) / (spread + phase.advantage_epsilon) - if not torch.isfinite(advantages).all(): - raise ValueError("non-finite GRPO group advantages") - groups.append( - tuple( - replace(t, advantage=float(a)) - for t, a in zip(trajectories, advantages) - ) - ) - return SealedGroupBuffer( - groups, generation, phase.group_size, context.reference_identity, used - ), executed - except Exception as exc: - raise GroupCollectionError(str(exc), executed) from exc + return collection.finish( + [collection.seal_group(group) for group in completed.values()] + ) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 91a42b6a..e1bde758 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -13,6 +13,7 @@ from sequifier.io.batch import SequifierInputs from sequifier.io.policy_window import normalize_policy_features from sequifier.io.preprocessing import normalization_statistics +from sequifier.model.execution import ModelExecution from sequifier.model.execution_schema import ExecutionSchema from sequifier.model.prediction_distribution import PredictionScales from sequifier.rl.distributions import CategoricalActionDistribution @@ -23,28 +24,20 @@ def policy_execution_config( - config: ResolvedPolicyBinding, - model, - device: str = "cpu", - *, - runtime_settings=None, - statistics=None, - execution_model=None, - **run_options, + config, device="cpu", *, execution_model=None, **run_options ): - """Resolve policy alignment directly into the ordinary execution contract.""" + """Select policy alignment over canonical model-owned execution facts.""" + from sequifier.artifacts.model_config import model_execution_from_model_config + from sequifier.rl.attachment import apply_binding + if execution_model is None: raise ValueError("joint policies require a resolved model execution contract") - from sequifier.artifacts.model_config import resolved_config_from_model_config - - execution, _ = resolved_config_from_model_config( + execution, _ = model_execution_from_model_config( execution_model, device=device, interface_name=config.interface_name ) - from sequifier.rl.attachment import apply_binding - apply_binding(execution, config.interface_name, config) for key, value in run_options.items(): - if key in type(execution).model_fields: + if key in ModelExecution.__dataclass_fields__: setattr(execution, key, value) return execution @@ -53,7 +46,7 @@ class Policy(nn.Module): """A convenience view; the network owns all trainable weights and execution.""" config: ResolvedPolicyBinding - execution_config: ResolvedSequifierConfig + execution_config: ResolvedSequifierConfig | ModelExecution interface_name: str network: ComposableTransformerNetwork units: dict[str, str] diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index af849134..b81d7778 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -11,7 +11,6 @@ ) from sequifier.rl.models import HistoryCritic from sequifier.rl.ppo_collector import PPOCollector -from sequifier.rl.ppo_evaluation import evaluate_policy from sequifier.rl.ppo_update import update_ppo from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal from sequifier.rl.snapshots import snapshots_equal @@ -65,13 +64,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): ) session.restore_random() if session.should_evaluate_initial(): - initial = evaluate_policy( - actor, phase.evaluation, forbidden_adapter=adapter - ) - session.observe(initial["selection_value"], **initial) - evaluations.append( - {**initial, "transaction": session.transaction, "initial": True} - ) + session.evaluate_online(adapter, evaluations, initial=True) while env_steps < phase.total_env_steps and not session.stopping.triggered: horizon = min(phase.rollout_env_steps, phase.total_env_steps - env_steps) buffer, episodes = collector.collect(horizon, session.transaction) @@ -86,17 +79,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): session.transaction += 1 final = env_steps == phase.total_env_steps if session.should_evaluate(final): - result = evaluate_policy( - actor, phase.evaluation, forbidden_adapter=adapter - ) - session.observe(result["selection_value"], **result) - evaluations.append( - { - **result, - "transaction": session.transaction, - "env_steps": env_steps, - } - ) + session.evaluate_online(adapter, evaluations, env_steps=env_steps) session.record_training( final=final, **{ @@ -132,10 +115,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): return session.finish( { "env_steps": env_steps, - **{ - f"{role}_optimizer_steps": steps - for role, steps in session.steps.items() - }, + **session.optimizer_clocks(), }, { "adapter": identity, diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index dfe94238..d9c19210 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -96,9 +96,7 @@ def validate_declared_policy(declared, fitted, field): def build_actor(config, fitted, statistics): execution = policy_execution_config( fitted, - config.model, config.device, - statistics=statistics, execution_model=config.execution_model, project_root=config.project_root, model_name=config.model_name, diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 31f4ff73..2ce9f316 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -282,17 +282,8 @@ def phase_reports(self, checkpoint_path): def write_reports(self, checkpoint_path): """Reports are derived views; deleting one does not invalidate recovery.""" - for result in self.results: - from sequifier.artifacts.reports import phase_report - - write_json_atomic( - result.provenance["report_path"], - phase_report( - model_path=result.provenance["actor_path"], - checkpoint_path=checkpoint_path, - **result.as_dict(), - ), - ) + for result, report in zip(self.results, self.phase_reports(checkpoint_path)): + write_json_atomic(result.provenance["report_path"], report) def validate_finite_state(value, field): diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 546efbad..7cdbbd40 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -432,6 +432,64 @@ def record_training(self, *, final=False, **record): if self.context.run._metrics_service is not None: self.context.run._metrics_service.record(self, record, final=final) + def evaluate_validation(self, evaluator, **record): + values = evaluator() + key = self.selection.monitor.removeprefix("validation_") + self.record_evaluation( + values[key], + **record, + **{f"validation_{name}": value for name, value in values.items()}, + ) + + def complete_update( + self, + *, + final, + training, + evaluate, + checkpoint=None, + evaluation_record: dict | Callable[[], dict] | None = None, + ): + """Commit an offline update after all participating optimizers have stepped.""" + self.transaction += 1 + self.record_training(final=final, **training) + if self.should_evaluate(final): + extra = ( + evaluation_record() + if callable(evaluation_record) + else evaluation_record + ) + evaluate(**training, **(extra or {})) + self.checkpoint(final, **(checkpoint or {})) + + def optimizer_clocks(self): + return {f"{role}_optimizer_steps": steps for role, steps in self.steps.items()} + + def evaluate_online(self, adapter, evaluations, *, initial=False, **progress): + from sequifier.rl.ppo_evaluation import evaluate_policy + + phase = self.context.phase + result = evaluate_policy( + self.context.actor, phase.evaluation, forbidden_adapter=adapter + ) + if phase.algorithm == "grpo": + score = float( + torch.tensor( + result["score_mean"] * phase.reward_scale, dtype=torch.float32 + ) + ) + if not math.isfinite(score): + raise ValueError("GRPO evaluation score exceeds finite FP32 range") + self.observe( + score, **{k: v for k, v in result.items() if k != "score_mean"} + ) + result["score_mean"] = score + else: + self.observe(result["selection_value"], **result) + if initial: + progress["initial"] = True + evaluations.append({**result, "transaction": self.transaction, **progress}) + def optimization_state(self, *, include_steps=False): return { role: { diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index a0b9b3e7..47f23bb3 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -52,6 +52,16 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin raise ValueError("imitation checkpoint epoch/transaction clocks disagree") if batches.epoch == phase.epochs and batches.order is not None: raise ValueError("completed imitation checkpoint has a live sampler order") + + def evaluate(**record): + score = evaluate_imitation(actor, data.validation, phase.evaluation.batch_size) + session.record_evaluation( + score, + epochs_completed=batches.epoch, + **record, + **{phase.evaluation.monitor.metric: score}, + ) + if session.should_evaluate_initial(): session.record_evaluation( evaluate_imitation(actor, data.validation, phase.evaluation.batch_size), @@ -71,20 +81,12 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin loss.backward() total += float(loss.detach()) session.step() - session.transaction += 1 - final = batches.epoch == phase.epochs - session.record_training(final=final, training_action_nll=total) - if session.should_evaluate(final): - score = evaluate_imitation( - actor, data.validation, phase.evaluation.batch_size - ) - session.record_evaluation( - score, - epochs_completed=batches.epoch, - training_action_nll=total, - **{phase.evaluation.monitor.metric: score}, - ) - session.checkpoint(final, batch_order=batches.state_dict()) + session.complete_update( + final=batches.epoch == phase.epochs, + training={"training_action_nll": total}, + evaluate=evaluate, + checkpoint={"batch_order": batches.state_dict()}, + ) return session.finish( {"optimizer_steps": session.steps["actor"], "epochs": batches.epoch}, {"data": data.contract}, diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 580e637f..37a59ba2 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -23,7 +23,7 @@ ) from sequifier.artifacts.phase import PhaseArtifacts, verify_artifact from sequifier.artifacts.reports import phase_report, publish_run_report, run_report -from sequifier.config.train_config import SequifierConfig, resolve_sequifier_config +from sequifier.config.train_config import resolve_sequifier_config from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save, write_json_atomic from sequifier.trajectory.sources import file_checksum @@ -38,82 +38,80 @@ class LearningRunResult: def _supervised_config(config, phase, index, incoming, resume): - from sequifier.artifacts.model_config import resolved_config_from_model_config + from sequifier.artifacts.model_config import ( + extend_prediction_distributions, + model_execution_from_model_config, + validate_execution_attachment, + ) + from sequifier.artifacts.model_export import model_execution_config + from sequifier.config.components import ResumeConfig from sequifier.config.metadata import load_dataset_metadata + from sequifier.config.train_config import OwnedModelSpec + from sequifier.model.execution import ModelExecution - authored = config.authored - # The coordinator stores resolved phases; the native builder owns source expansion. - phase_spec = authored.training_plan.phases[index].model_copy(deep=True) + authored = config.authored.model_copy(deep=True) + phase_spec = authored.training_plan.phases[index] phase_spec.optimization = phase.optimization - model = config.model - source = None - forward_attention = None - if incoming is not None: - source, _ = resolved_config_from_model_config( - read_execution_artifact(incoming).model_config.values, device=config.device + training = config.runtime_settings.model_copy(deep=True) + if training.distributed: + raise ValueError("mixed learning plans currently require local phase execution") + artifact = read_execution_artifact(incoming) if incoming is not None else None + source = config + if artifact is not None: + source, _ = model_execution_from_model_config( + artifact.model_config.values, device=config.device ) - from sequifier.artifacts.model_config import extend_prediction_distributions - extend_prediction_distributions(source, config.model) - model = source.model - forward_attention = source.runtime_settings.forward_attention or ( + training.forward_attention = source.runtime_settings.forward_attention or ( "bidirectional" if source.runtime_settings.training_objective == "bert" else "causal" ) - training = config.runtime_settings.model_copy(deep=True) - if training.distributed: - raise ValueError("mixed learning plans currently require local phase execution") - if forward_attention is not None: - training.forward_attention = forward_attention - from sequifier.config.components import ResumeConfig - + execution = ModelExecution.from_config(source) native = PhaseArtifacts.for_phase( config.phase_directory, index, phase.name ).model.parent - training.resume = ResumeConfig( + run = authored.run + run.project_root = config.project_root + run.model_name = f"{config.model_name}-phase-{index:03d}-{phase.name}" + run.execution.device = config.device + run.reproducibility.seed = config.seed + index + run.recovery.resume = ResumeConfig( policy="if_exists" if resume else "never", checkpoint_path=str(native / "run.pt"), ) - values = authored.model_dump(mode="python") - run = values["run"] - run["project_root"] = config.project_root - run["model_name"] = f"{config.model_name}-phase-{index:03d}-{phase.name}" - run["execution"]["device"] = config.device - run["reproducibility"]["seed"] = config.seed + index - run["recovery"]["resume"] = training.resume.model_dump() - run["export"]["pt"] = True - run["logging"].pop("log_interval_iterations", None) - run["outputs"] = { - "model_path": str(native / "native" / "model.pt"), - "report_path": str(native / "native" / "report.json"), - "phase_directory": str(native / "native" / "phases"), - } - model_values = model.model_dump() - model_values["execution"] = { - key: getattr(training, key) - for key in ( - "context_length", - "training_objective", - "target_offset", - "forward_attention", - ) - } - for name, interface in authored.model.interfaces.items(): - if name in model_values["interfaces"]: - model_values["interfaces"][name]["input_contract"] = ( - interface.input_contract.model_dump() + run.export.pt = True + run.logging.log_interval_iterations = 10 + for name, path in { + "model_path": native / "native/model.pt", + "report_path": native / "native/report.json", + "phase_directory": native / "native/phases", + }.items(): + setattr(run.outputs, name, str(path)) + authored.model = OwnedModelSpec.from_execution( + execution, + authored.model, + settings={ + key: getattr(training, key) + for key in ( + "context_length", + "training_objective", + "target_offset", + "forward_attention", ) - values["model"] = model_values + }, + ) names = {ref.dataset for ref in [*phase.sources, *phase.evaluation_sources]} - values["dataset_training"] = { + authored.dataset_training = { name: value - for name, value in values["dataset_training"].items() + for name, value in authored.dataset_training.items() if name in names } - values["training_plan"] = {"phases": [phase_spec.model_dump()]} - values["initialize_from"] = None - authored = SequifierConfig.model_validate(values) + authored.training_plan = authored.training_plan.model_copy( + update={"phases": [phase_spec]} + ) + authored.initialize_from = None + authored.runtime_settings = authored._build_runtime_settings() metadata = { f"{name}.{part_name}": load_dataset_metadata( str(Path(config.project_root) / part.metadata_config_path) @@ -121,34 +119,14 @@ def _supervised_config(config, phase, index, incoming, resume): for name, dataset in authored.dataset_training.items() for part_name, part in dataset.parts.items() } - resolved = resolve_sequifier_config(authored, metadata) - resolved.model_path = run["outputs"]["model_path"] - resolved.report_path = run["outputs"]["report_path"] - resolved.phase_directory = run["outputs"]["phase_directory"] + resolved = resolve_sequifier_config(authored, metadata, execution=execution) resolved.ownership = config.ownership resolved.source_validation_fractions = config.source_validation_fractions resolved.normalization_by_interface = config.normalization_by_interface - if source is None: - for dataset in resolved.dataset_training.values(): - dataset.interface.policy = config.policy_bindings.get( - dataset.model_interface - ) - if source is not None: - # Preserve policy bindings and their model-owned scale modules on return - # to supervised learning. Dataset resolution owns the data/window views. - for dataset in resolved.dataset_training.values(): - previous = source.interface(dataset.model_interface) - dataset.interface.policy = previous.policy - from sequifier.model.execution import ModelExecution - - ModelExecution.from_config(source).retain_interfaces(resolved) - if incoming is not None: - from sequifier.artifacts.model_config import validate_execution_attachment - from sequifier.artifacts.model_export import model_execution_config - + if artifact is not None: + execution.retain_interfaces(resolved) validate_execution_attachment( - read_execution_artifact(incoming).model_config.values, - model_execution_config(resolved), + artifact.model_config.values, model_execution_config(resolved) ) return resolved diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 1ee79f1d..135f0008 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -777,11 +777,9 @@ def validate_phase_learning(config, device): dataset = config.dataset_training[name].model_copy(deep=True) for key, value in learning.model_dump().items(): setattr(dataset, key, value) - targets = set(dataset.interface.target_columns) - if set(dataset.criterion) != targets: - raise ValueError( - f"{phase.name}: {name} criterion keys must equal targets" - ) + learning.validate_targets( + dataset.interface.target_columns, f"{phase.name}: {name}" + ) for column in dataset.class_share_log_columns: if dataset.interface.target_column_types.get(column) != "categorical": raise ValueError( From 08fc8af6274204251a3b3fc81fed44e3f9c4730e Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Tue, 17 Nov 2026 22:12:41 +0100 Subject: [PATCH 47/71] export dtypes to onnx artifact --- src/sequifier/export/onnx.py | 4 ++-- src/sequifier/export/service.py | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/sequifier/export/onnx.py b/src/sequifier/export/onnx.py index 212e4b97..c6d281ea 100644 --- a/src/sequifier/export/onnx.py +++ b/src/sequifier/export/onnx.py @@ -308,7 +308,7 @@ def export( state = manager.capture_local() try: # All graph lowering and dtype conversion happen on a disposable copy. - model = copy.deepcopy(model).cpu().float() + model = copy.deepcopy(model).cpu() for module in model.modules(): if isinstance(module, SelfAttention): module._sequifier_export_attention = True @@ -449,7 +449,7 @@ def export( expected.detach().numpy(), actual, rtol=2e-4, atol=2e-5 ): raise ValueError( - "ONNX artifact differs from its FP32 export reference" + "ONNX artifact differs from its export reference" ) published = [] try: diff --git a/src/sequifier/export/service.py b/src/sequifier/export/service.py index e55f7bee..912a98d0 100644 --- a/src/sequifier/export/service.py +++ b/src/sequifier/export/service.py @@ -2,7 +2,6 @@ from __future__ import annotations -import copy import json from dataclasses import dataclass, field from pathlib import Path @@ -43,7 +42,6 @@ def export( def _onnx_execution_config(config, interface_name): execution = model_execution_config(config) execution["interfaces"] = {interface_name: execution["interfaces"][interface_name]} - execution["layer_type_dtypes"] = None return execution @@ -114,11 +112,6 @@ def _export( ) if self.config.export_onnx: onnx_network = export_network - if any( - parameter.dtype in (torch.float16, torch.bfloat16, torch.float64) - for parameter in export_network.parameters() - ): - onnx_network = copy.deepcopy(export_network).float() count = len(self.config.dataset_training) for dataset_name, dataset in self.config.dataset_training.items(): if self.config.export_generative_model: From 34c7f451017d3093411f85602917e5be58b29997 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 10:14:51 +0100 Subject: [PATCH 48/71] initialization from earlier phases for support networks --- documentation/policy-learning.md | 36 ++++++ documentation/supervised-policy-learning.md | 28 +++- src/sequifier/config/policy_plan.py | 14 ++ src/sequifier/config/train_config.py | 37 +++++- src/sequifier/config/training_resolution.py | 105 +++++++++++++-- src/sequifier/model/factory.py | 8 +- src/sequifier/model/history_transfer.py | 136 ++++++++++++++++++++ src/sequifier/rl/iql.py | 8 +- src/sequifier/rl/learner.py | 92 +++++++++++-- src/sequifier/rl/models.py | 81 ++++++++++-- src/sequifier/rl/ppo.py | 12 +- src/sequifier/rl/preparation.py | 28 ++-- src/sequifier/rl/references.py | 19 ++- src/sequifier/rl/run_config.py | 3 +- src/sequifier/training/coordinator.py | 39 ++++-- 15 files changed, 585 insertions(+), 61 deletions(-) create mode 100644 src/sequifier/model/history_transfer.py diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 1bceecb1..714ae99c 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -123,6 +123,42 @@ Exact GRPO recovery requires reproducible group starts. Evaluation factories mus create independent adapters. The [control adapter](../src/sequifier/rl/control.py) is a complete local example. +## Initialize PPO and IQL from a pretrained history encoder + +Set `history_initialization: phase_start_actor` on a PPO or IQL phase with +`optimizer_state: reset`. PPO initializes its value encoder from the incoming +actor. IQL initializes the value and both Q encoders from that same actor. Each +network owns an independent copy of the transformer and input encoders. The +scalar heads and the two Q action encoders start fresh, and each target Q starts +as a copy of its corresponding critic. + +With `initialize_from`, the first phase receives the pretrained actor. Later +phases use the actor selected at the preceding handoff. The default +`history_initialization: random` uses fresh value and Q encoders. A phase with +`optimizer_state: continue` restores the preceding networks and optimizer state. + +The transferred history parameters follow the phase’s usual trainability rules. +An optimizer plan can fine-tune the backbone at a lower rate than the heads. +For example, put the following under a PPO or IQL phase's `optimization` and +use the same grouping for `actor` or IQL's `critic` role as needed. + +```yaml +value: + optimizer: + groups: + - id: backbone + select: {module: backbone} + optimizer: {name: Adam} + learning_rate: 0.00003 + - id: remaining + select: otherwise + optimizer: {name: Adam} + learning_rate: 0.0003 +``` + +See [supervised models and policy learning](supervised-policy-learning.md) for +initialization from artifacts whose decoder predicts different targets. + ## PPO and GRPO PPO seals one rollout generation, computes boundary-aware FP32 GAE, then optimizes diff --git a/documentation/supervised-policy-learning.md b/documentation/supervised-policy-learning.md index a06fcb84..bf96d32f 100644 --- a/documentation/supervised-policy-learning.md +++ b/documentation/supervised-policy-learning.md @@ -21,8 +21,9 @@ policy_bindings: fixed: 0.5 ``` -Add datasets/phases as in the [policy guide](policy-learning.md). A separately -declared model must agree with the artifact. Omitted categorical `values` select +Add datasets/phases as in the [policy guide](policy-learning.md). With the default +`actor_initialization: attach`, a separately declared model must agree with the +artifact. Omitted categorical `values` select the whole decoder vocabulary; an explicit subset is reported as an adaptation. Real `scale` explicitly selects `model` or `{fixed: ...}` in physical units. Learned scales belong to the interface's `prediction_distributions`, where @@ -32,6 +33,29 @@ Bounds change the deterministic prediction and count as adaptation. Samples reta transform latents; external endpoints need declared atoms. `active_when` references earlier action fields; inactive fields use `inactive_value` with zero likelihood cost. +### Initialize a fresh action decoder + +When the artifact predicts different targets, use these run-level fields with an +explicit downstream `model`, `policy_bindings`, and a policy first phase. + +```yaml +initialize_from: models/observation-pretrain.pt +actor_initialization: history_only +``` + +Use the same named interface and compatible history inputs, preprocessing, +context, attention, precision and backbone architecture. Declare the downstream +action targets and an untied decoder. The artifact supplies the history encoder +and backbone weights, and the downstream model supplies the newly initialized +decoder and action distribution parameters. Set `history_initialization: +phase_start_actor` on PPO or IQL phases to also initialize their value or Q history +encoders. See the [policy guide](policy-learning.md#initialize-ppo-and-iql-from-a-pretrained-history-encoder). + +Check that every input is available at decision time. Pretraining with future +context can change how well the representation works on histories available to +the policy. Compare actor-only and actor-plus-auxiliary initialization under the +same downstream data, evaluation seeds and training budgets. + Ordinary branches factor independently; autoregressive branches retain target order and score recorded prefixes. Selecting a later autoregressive action requires all preceding decoder targets. Composite branches and tied weights are retained. diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index b609cebd..3f68a64c 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -85,11 +85,23 @@ def optimization_configs(self): @model_validator(mode="after") def validate_optimization(self): + if ( + getattr(self, "history_initialization", None) == "phase_start_actor" + and self.optimizer_state == "continue" + ): + raise ValueError("history_initialization requires optimizer_state: reset") self.trainability.validate_roles(self.optimization_roles) for config in self.optimization_configs().values(): config.validate_budget(self.optimizer_step_budget()) return self + @model_serializer(mode="wrap") + def serialize_history_initialization(self, handler): + values = handler(self) + if values.get("history_initialization") == "random": + values.pop("history_initialization") + return values + @field_validator("name") @classmethod def phase_name(cls, value): @@ -120,6 +132,7 @@ def optimizer_step_budget(self, training_size=None): class IQLPhaseConfig(OfflinePhaseConfig): optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value", "critic") algorithm: Literal["iql"] + history_initialization: Literal["random", "phase_start_actor"] = "random" evaluation: OfflineEvaluationConfig = Field( default_factory=lambda: OfflineEvaluationConfig(cadence=Cadence(iterations=100)) ) @@ -201,6 +214,7 @@ def serial_evaluation(self): class PPOPhaseConfig(PhaseConfig): optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value") algorithm: Literal["ppo"] + history_initialization: Literal["random", "phase_start_actor"] = "random" name: str = Field(default="online_ppo", min_length=1) environment: OnlineEnvironmentConfig evaluation: OnlineEvaluationConfig diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 384e60f5..14cb2e64 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -15,7 +15,14 @@ from types import SimpleNamespace from typing import Annotated, Any, ClassVar, Literal, Optional -from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator +from pydantic import ( + ConfigDict, + Field, + PrivateAttr, + field_validator, + model_serializer, + model_validator, +) # Reusable schemas are independent of the canonical root configuration. from sequifier.config.components import ( @@ -924,6 +931,7 @@ class ResolvedSequifierConfig(ConfigModel): dataset_training: dict[str, ResolvedDatasetTrainingSpec] authored: SequifierConfig | None = Field(default=None, exclude=True) initialize_from: str | None = None + actor_initialization: Literal["attach", "history_only"] = "attach" model_path: str = "" report_path: str = "" phase_directory: str = "" @@ -935,6 +943,13 @@ class ResolvedSequifierConfig(ConfigModel): curriculum_training: bool = False curriculum_column: Optional[str] = None + @model_serializer(mode="wrap") + def serialize_actor_initialization(self, handler): + values = handler(self) + if self.actor_initialization == "attach": + values.pop("actor_initialization", None) + return values + @property def evaluation_sources(self) -> list[ResolvedTrainingSource]: """All supervised evaluation dependencies, for loader and metric setup.""" @@ -2184,10 +2199,18 @@ class SequifierConfig(ConfigModel): run: RunSettings model: OwnedModelSpec | None = None initialize_from: str | None = None + actor_initialization: Literal["attach", "history_only"] = "attach" dataset_training: dict[str, DatasetTrainingSpecModel] = Field(default_factory=dict) policy_bindings: dict[str, PolicyBinding] = Field(default_factory=dict) training_plan: TrainingPlanModel + @model_serializer(mode="wrap") + def serialize_actor_initialization(self, handler): + values = handler(self) + if self.actor_initialization == "attach": + values.pop("actor_initialization", None) + return values + @model_validator(mode="before") @classmethod def apply_defaults(cls, values): @@ -2242,6 +2265,18 @@ def apply_defaults(cls, values): @model_validator(mode="after") def validate_contract(self): + if self.actor_initialization == "history_only" and ( + not self.initialize_from + or self.model is None + or self.model.backbone is None + or not self.model.interfaces + or not self.policy_bindings + or self.training_plan.phases[0].algorithm == "supervised" + ): + raise ValueError( + "actor_initialization: history_only requires initialize_from, " + "a complete downstream model, policy_bindings and a policy first phase" + ) if ( self.model is None or self.model.backbone is None ) and self.initialize_from is None: diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 53290ff7..05712672 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -190,6 +190,46 @@ def _inferred_metadata(config, name, binding): ) +def _history_initialized_metadata(config, name, binding, source): + """Resolve new targets while sourcing observation preprocessing from the artifact.""" + metadata = _inferred_metadata(config, name, binding) + static = source.static_conditioning + inputs = set(source.input_columns + (static.columns if static else [])) + targets = set(config.model.interfaces[name].target_columns) + for key in ( + "column_data_types", + "n_classes", + "id_maps", + "selected_columns_statistics", + ): + values = getattr(metadata, key) + original = getattr(source, key) + for column in inputs: + if column in targets and column in values and column in original: + if ( + key != "selected_columns_statistics" + and values[column] != original[column] + ): + raise ValueError( + f"history transfer differs at {name}.{key}.{column}" + ) + values.pop(column, None) + if column in original: + values[column] = original[column] + return metadata.model_copy( + update={ + "static_columns": static.columns if static else [], + "static_features_path": source.static_features_path, + "image_sources": source.image_sources, + "preconditioning_image": source.preconditioning_image, + "special_token_ids": source.special_token_ids, + "depth_layouts": source.depth_layouts, + "tensor_payload_version": source.tensor_payload_version, + "normalize_real_columns": source.normalize_real_columns, + } + ) + + def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): """Fit missing raw codecs/moments on the same episode-safe training split.""" from types import SimpleNamespace @@ -296,6 +336,7 @@ def resolve_training_run(authored: SequifierConfig): from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.artifacts.model_config import model_execution_from_model_config from sequifier.model.execution import ModelExecution + from sequifier.model.history_transfer import validate_history_transfer from sequifier.rl.attachment import apply_binding, resolve_binding from sequifier.rl.prepared_inputs import PreparedTrainingInputs @@ -305,17 +346,19 @@ def resolve_training_run(authored: SequifierConfig): config.run.project_root = str(root) training = config.runtime_settings source = None + history_only = config.actor_initialization == "history_only" if config.initialize_from: config.initialize_from = str((root / config.initialize_from).resolve()) source, _ = model_execution_from_model_config( read_execution_artifact(config.initialize_from).model_config.values, device=config.run.execution.device, ) - if config.model is not None: + if config.model is not None and not history_only: from sequifier.artifacts.model_config import extend_prediction_distributions extend_prediction_distributions(source, config.model) - config.model = OwnedModelSpec.from_execution(source, config.model) + if not history_only: + config.model = OwnedModelSpec.from_execution(source, config.model) for field in ( "context_length", "training_objective", @@ -323,6 +366,12 @@ def resolve_training_run(authored: SequifierConfig): "forward_attention", ): actual = getattr(source.runtime_settings, field) + if ( + history_only + and field in {"training_objective", "target_offset"} + and field in training.model_fields_set + ): + continue if ( field in training.model_fields_set and getattr(training, field) != actual @@ -331,6 +380,14 @@ def resolve_training_run(authored: SequifierConfig): f"model.execution.{field} differs from initialize_from" ) setattr(training, field, actual) + if history_only: + if ( + "layer_type_dtypes" in config.run.execution.precision.model_fields_set + and training.layer_type_dtypes + != source.runtime_settings.layer_type_dtypes + ): + raise ValueError("history transfer differs at layer_type_dtypes") + training.layer_type_dtypes = source.runtime_settings.layer_type_dtypes assert config.model is not None assert config.model.backbone is not None for dataset in config.dataset_training.values(): @@ -397,8 +454,19 @@ def resolve_training_run(authored: SequifierConfig): ) metadata[dataset.model_interface] = value interfaces = {} + history_interface = ( + phase_interface(config, config.training_plan.phases[0]) + if history_only + else None + ) + if history_only: + assert source is not None + if history_interface not in source.interfaces: + raise ValueError( + f"history transfer requires interface {history_interface!r}" + ) for name, interface in config.model.interfaces.items(): - if source is not None: + if source is not None and not history_only: interfaces[name] = source.interface(name).model_copy(deep=True) if name in metadata: candidate = _resolve_interface( @@ -420,13 +488,19 @@ def resolve_training_run(authored: SequifierConfig): ) else: value = metadata.get(name) + binding = config.policy_bindings.get(name) + if name == history_interface and value is None: + assert source is not None + if binding is None: + raise ValueError(f"{name}: history_only requires a policy binding") + value = _history_initialized_metadata( + config, name, binding, source.interface(name) + ) if value is None: - binding = config.policy_bindings.get(name) if binding is None: raise ValueError(f"{name}: model interface requires metadata") value = _inferred_metadata(config, name, binding) - binding = config.policy_bindings.get(name) - if binding is not None: + if binding is not None and name != history_interface: value = _fit_input_metadata( config, name, binding, value, root, prepared_inputs ) @@ -434,6 +508,8 @@ def resolve_training_run(authored: SequifierConfig): name, interface, value, training, value.storage_layout ) execution = ModelExecution(config.model.architecture_model(), interfaces, training) + if history_only: + validate_history_transfer(source, execution, history_interface) for name, binding in config.policy_bindings.items(): if name not in interfaces: raise ValueError(f"policy_bindings: unknown interface {name!r}") @@ -508,6 +584,7 @@ def resolve_training_run(authored: SequifierConfig): resolved._prepared_inputs = prepared_inputs resolved.authored = config resolved.initialize_from = config.initialize_from + resolved.actor_initialization = config.actor_initialization from sequifier.io.output_paths import resolve_training_output_paths for name in ("model_path", "report_path", "phase_directory"): @@ -552,7 +629,13 @@ def resolve_training_run(authored: SequifierConfig): def policy_run_config( - config, phases, *, incoming=None, index=0, destination=None, resuming=None + config, + phases, + *, + incoming=None, + index=0, + destination=None, + resuming=None, ): """Select resolved inputs for one native policy lifecycle.""" from sequifier.artifacts.model_artifact import read_execution_artifact @@ -568,8 +651,11 @@ def policy_run_config( raise ValueError("a native policy lifecycle requires one interface") name = next(iter(names)) execution = config - initialization = incoming if incoming is not None else config.initialize_from - if initialization: + initialization = ( + incoming if index or incoming is not None else config.initialize_from + ) + actor_initialization = config.actor_initialization if index == 0 else "attach" + if initialization and actor_initialization == "attach": execution, _ = model_execution_from_model_config( read_execution_artifact(initialization).model_config.values, device=config.device, @@ -610,6 +696,7 @@ def policy_run_config( execution_model=model_execution_config(execution), policy_interface=name, initialize_from=initialization, + actor_initialization=actor_initialization, normalization=config.normalization_by_interface.get(name), units=binding.units, sources=config.sequence_sources, diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index c17de510..032af723 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -257,6 +257,7 @@ def _build_composable_network( *, device: torch.device, initialize: bool, + restored_history_interface: str | None, logger: Any, ) -> BuiltModel: from sequifier.config.train_config import interface_build_view @@ -285,13 +286,14 @@ def _build_composable_network( route_initialization_components: dict[str, tuple[nn.Module, Any]] = {} for name, interface in resolved_interfaces.items(): view = interface_build_view(config, interface) + load_pretrained = initialize and name != restored_history_interface built_ingestion = compile_feature_ingestion( hparams=view, direct_real_dtype_provider=lambda: backbone.layers[ 0 ].ff.get_first_layer_dtype(), max_concat_tensors=config.runtime_settings.max_concat_tensors, - load_pretrained=initialize, + load_pretrained=load_pretrained, ) from sequifier.model.ingestions import EmbeddingFeatureIngestion @@ -366,7 +368,7 @@ def _build_composable_network( interface.preconditioning_image_encoder, backbone.model_dim, len(interface.preconditioning_image.views), - initialize, + load_pretrained, ) prefix_projection = ( nn.Linear(2 * backbone.model_dim, backbone.model_dim) @@ -504,6 +506,7 @@ def build_transformer_network( *, device: torch.device, initialize: bool = True, + restored_history_interface: str | None = None, logger: Any | None = None, ) -> BuiltModel: """Build one shared backbone and every distinct named model interface.""" @@ -513,5 +516,6 @@ def build_transformer_network( config, device=device, initialize=initialize, + restored_history_interface=restored_history_interface, logger=resolved_logger, ) diff --git a/src/sequifier/model/history_transfer.py b/src/sequifier/model/history_transfer.py new file mode 100644 index 00000000..62fbad20 --- /dev/null +++ b/src/sequifier/model/history_transfer.py @@ -0,0 +1,136 @@ +"""Compatibility and weight copying for the model's decision-history path.""" + +from sequifier.model.execution import ModelExecution +from sequifier.model.execution_schema import ExecutionSchema + +_HISTORY_COMPONENTS = ( + "ingestion", + "ingestion_adapter", + "static_condition_encoder", + "preconditioning_image_encoder", + "prefix_projection", +) + + +def _ingestion_contract(ingestion): + values = ingestion.model_dump(exclude={"initialization", "initialization_seed"}) + if ingestion.type == "composite": + values["branches"] = [ + (name, _ingestion_contract(branch)) + for name, branch in ingestion.branches.items() + ] + return values + + +def _history_contract(config, interface_name): + execution = ModelExecution.from_config(config) + if interface_name not in execution.interfaces: + raise ValueError(f"history transfer requires interface {interface_name!r}") + interface = execution.interface(interface_name) + settings = execution.runtime_settings + static = interface.static_conditioning + columns = interface.input_columns + (static.columns if static else []) + media_fields = { + "path", + "signature", + "time_index_path", + "time_index_signature", + "video_index_paths", + } + return { + "backbone.architecture": execution.model.backbone.architecture, + "layer_type_dtypes": settings.layer_type_dtypes, + "context_length": settings.context_length, + "forward_attention": settings.forward_attention + or ("bidirectional" if settings.training_objective == "bert" else "causal"), + "inputs": ExecutionSchema.from_interface( + interface, settings.context_length + ).inputs, + "input_columns": interface.input_columns, + "ingestion": _ingestion_contract(interface.ingestion), + "static_conditioning": static, + "feature_layout": interface.feature_layout, + "depth_layouts": interface.depth_layouts.compatibility_signature(columns), + "image_sources": { + name: item.model_dump(exclude=media_fields) + for name, item in interface.image_sources.items() + }, + "preconditioning_image": interface.preconditioning_image.compatibility_signature() + if interface.preconditioning_image is not None + else None, + "preconditioning_image_encoder": interface.preconditioning_image_encoder, + "normalize_real_columns": interface.normalize_real_columns, + "special_token_ids": interface.special_token_ids, + "input_metadata": { + column: { + key: getattr(interface, key).get(column) + for key in ( + "column_data_types", + "n_classes", + "id_maps", + "categorical_hashing", + "selected_columns_statistics", + ) + } + for column in columns + }, + } + + +def validate_history_transfer(source, destination, interface_name): + expected = _history_contract(source, interface_name) + actual = _history_contract(destination, interface_name) + for field, value in expected.items(): + if value != actual[field]: + if field == "input_metadata": + field = next( + f"{field}.{column}.{key}" + for column, metadata in value.items() + for key, entry in metadata.items() + if entry != actual[field][column][key] + ) + raise ValueError(f"history transfer differs at {interface_name}.{field}") + + +def history_components(network, interface_name): + route = network.resolve_interface(interface_name) + return { + "backbone": network.backbone, + **{ + f"interfaces.{interface_name}.{name}": component + for name in _HISTORY_COMPONENTS + if (component := getattr(route, name)) is not None + }, + } + + +def history_parameter_ids(network, interface_name): + return { + id(parameter) + for module in history_components(network, interface_name).values() + for parameter in module.parameters() + } + + +def copy_history_weights(source, destination, interface_name): + """Copy parameters and persistent buffers without adopting source storage or flags.""" + source_modules = history_components(source, interface_name) + destination_modules = history_components(destination, interface_name) + if source_modules.keys() != destination_modules.keys(): + raise ValueError(f"history transfer components differ at {interface_name}") + weights = {} + for path, module in destination_modules.items(): + incoming = source_modules[path].state_dict() + expected = module.state_dict() + if incoming.keys() != expected.keys(): + raise ValueError(f"history transfer tensor keys differ at {path}") + for name, tensor in expected.items(): + if ( + tensor.shape != incoming[name].shape + or tensor.dtype != incoming[name].dtype + ): + raise ValueError(f"history transfer tensor differs at {path}.{name}") + weights[path] = incoming + for path, module in destination_modules.items(): + module.load_state_dict(weights[path], strict=True) + return list(weights) diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index f022bfa7..257f7391 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -8,7 +8,7 @@ from sequifier.config.policy_plan import IQLPhaseConfig from sequifier.io.iql_dataset import IQLBatch from sequifier.io.tensors import collate_tensor_records -from sequifier.rl.models import make_iql_models +from sequifier.rl.models import critic_history_options, make_iql_models from sequifier.rl.offline_data import OfflineData from sequifier.rl.run_state import PhaseContext, PhaseSession @@ -71,6 +71,12 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): actor.execution_config, actor.interface_name, actor.config.actions, + **critic_history_options( + actor, + context.run.phases, + context.index, + restore=context.resume_state is not None, + ), ) for module in (value, q1, q2, *targets): module.eval() diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 6ecf01eb..503a0177 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -8,7 +8,12 @@ from sequifier.artifacts.phase import PhaseArtifacts from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism, configure_logger -from sequifier.rl.checkpoint import load_policy, phase_actor_path, save_policy +from sequifier.rl.checkpoint import ( + load_policy, + phase_actor_path, + save_policy, + validate_policy_weights, +) from sequifier.rl.preparation import ( build_actor, preflight_environments, @@ -39,6 +44,8 @@ def _plan_contract(config): values = config.model_dump(mode="json") values.pop("resume") values.pop("checkpoint_path", None) + if config.actor_initialization == "attach": + values.pop("actor_initialization") for phase in values["phases"]: for role in phase["optimization"].values(): if role["scheduler"]["name"] == "OneCycleLR": @@ -47,7 +54,63 @@ def _plan_contract(config): return values +def _new_actor(config): + return build_actor( + config, + config.policy, + config.execution_model["interfaces"][config.policy_interface].get( + "selected_columns_statistics", {} + ), + ) + + +def _initialize_history_actor(config): + from sequifier.artifacts.model_artifact import load_execution_model + from sequifier.model.history_transfer import ( + copy_history_weights, + history_parameter_ids, + validate_history_transfer, + ) + from sequifier.rl.models import validate_independent_networks + from sequifier.trajectory.sources import file_checksum + + if not config.initialize_from: + raise ValueError("history_only requires initialize_from") + actor = _new_actor(config) + name = actor.interface_name + decoder = actor.network.resolve_interface(name).decoder + history_ids = history_parameter_ids(actor.network, name) + if any(id(parameter) in history_ids for parameter in decoder.parameters()): + raise ValueError(f"{name}: history_only requires an untied action decoder") + checksum = file_checksum(config.initialize_from) + random = capture_rng_state(actor.device) + try: + source, execution, _ = load_execution_model( + config.initialize_from, device=config.device, interface_name=name + ) + finally: + restore_rng_state(actor.device, random) + require_equal( + checksum, file_checksum(config.initialize_from), "initialize_from.sha256" + ) + validate_history_transfer(execution, actor.execution_config, name) + validate_independent_networks(source, actor.network) + components = copy_history_weights(source, actor.network, name) + validate_policy_weights(actor) + actor.transfer_report = { + "mode": "history_only", + "interface": name, + "source_sha256": checksum, + "source_preprocessing": "preserved", + "transferred_components": components, + "fresh_components": [f"interfaces.{name}.decoder"], + } + return actor + + def _initialize_actor(config): + if config.actor_initialization == "history_only": + return _initialize_history_actor(config) if config.initialize_from: source = load_policy( config.initialize_from, @@ -66,13 +129,7 @@ def _initialize_actor(config): source.execution_config.embedding_layer_names = config.embedding_layer_names source.units = dict(config.units) return source - return build_actor( - config, - config.policy, - config.execution_model["interfaces"][config.policy_interface].get( - "selected_columns_statistics", {} - ), - ) + return _new_actor(config) def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: @@ -177,6 +234,24 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: ) validate_output_paths(config, inputs) incoming_weight_hash = weight_fingerprint(actor.network.state_dict()) + initialization = {} + if getattr(phase, "history_initialization", None) == "phase_start_actor": + from sequifier.model.history_transfer import history_components + + initialization["history_initialization"] = { + "source": "phase_start_actor", + "source_weight_hash": incoming_weight_hash, + "transferred_components": list( + history_components(actor.network, actor.interface_name) + ), + "fresh_components": {"value": ["head"]}, + } + if phase.algorithm == "iql": + initialization["history_initialization"]["fresh_components"].update( + {name: ["action_embedding", "head"] for name in ("q1", "q2")} + ) + if phase_index == 0 and config.actor_initialization == "history_only": + initialization["actor_initialization"] = recovery.initialization actor.eval() reference, reference_identity = references.resolve(actor) @@ -214,6 +289,7 @@ def commit(state): "resolved_phase": phase.model_dump(mode="json"), "representation_fingerprint": representation.identity, "incoming_weight_hash": incoming_weight_hash, + **initialization, **PhaseArtifacts(actor_path).describe(), }, ) diff --git a/src/sequifier/rl/models.py b/src/sequifier/rl/models.py index f11c3d52..880d3ab2 100644 --- a/src/sequifier/rl/models.py +++ b/src/sequifier/rl/models.py @@ -7,7 +7,38 @@ from sequifier.io.batch import SequifierInputs from sequifier.io.policy_window import normalize_policy_features +from sequifier.model.dtypes import cast_floating_to_module_dtype from sequifier.model.factory import build_transformer_network +from sequifier.model.history_transfer import copy_history_weights, history_parameter_ids + + +def history_transfer_enabled(phases, index): + """Continue with the same parameter eligibility as the last fresh phase.""" + while index > 0 and phases[index].optimizer_state == "continue": + index -= 1 + return getattr(phases[index], "history_initialization", None) == "phase_start_actor" + + +def critic_history_options(actor, phases, index, *, restore=False): + enabled = history_transfer_enabled(phases, index) + fresh = phases[index].optimizer_state == "reset" and not restore + return { + "history_source": actor.network if enabled and fresh else None, + "preserve_history_trainability": enabled, + } + + +def validate_independent_networks(*modules): + seen = set() + for module in modules: + storage = { + (tensor.device, tensor.untyped_storage().data_ptr()) + for tensor in (*module.parameters(), *module.buffers()) + if tensor.numel() + } + if seen & storage: + raise ValueError("policy networks share parameter or buffer storage") + seen.update(storage) class ActionEncoder(nn.Module): @@ -56,18 +87,33 @@ def forward(self, actions): class HistoryCritic(nn.Module): def __init__( - self, execution_config, interface_name: str, action_count: int | dict | None + self, + execution_config, + interface_name: str, + action_count: int | dict | None, + *, + history_source=None, + preserve_history_trainability=False, ): super().__init__() + restored_history = preserve_history_trainability or history_source is not None self.network = build_transformer_network( - execution_config, device=torch.device(execution_config.device) + execution_config, + device=torch.device(execution_config.device), + restored_history_interface=interface_name if restored_history else None, ).network self.interface_name = interface_name self.interface = execution_config.interface(interface_name) + history_ids = ( + history_parameter_ids(self.network, interface_name) + if restored_history + else set() + ) for parameter in self.network.resolve_interface( interface_name ).decoder.parameters(): - parameter.requires_grad_(False) + if id(parameter) not in history_ids: + parameter.requires_grad_(False) width = self.network.model_dim self.action_embedding = ( ActionEncoder( @@ -88,6 +134,17 @@ def __init__( nn.Linear(width, 1), ) self.to(execution_config.device) + if history_source is not None: + copy_history_weights(history_source, self.network, interface_name) + + @classmethod + def ppo_value(cls, actor, **history): + value = cls( + deepcopy(actor.execution_config), actor.interface_name, None, **history + ) + if history.get("history_source") is not None: + validate_independent_networks(actor.network, value) + return value def forward( self, @@ -117,13 +174,19 @@ def prepared_value(self, features, metadata, action=None): ): raise ValueError("critic actions must be int64 [batch]") state = torch.cat((state, self.action_embedding(action)), dim=-1) - return self.head(state).squeeze(-1).float() + return ( + self.head(cast_floating_to_module_dtype(state, self.head[0])) + .squeeze(-1) + .float() + ) -def make_iql_models(execution_config, interface_name: str, action_count: int | dict): - value = HistoryCritic(execution_config, interface_name, None) - q1 = HistoryCritic(execution_config, interface_name, action_count) - q2 = HistoryCritic(execution_config, interface_name, action_count) +def make_iql_models( + execution_config, interface_name: str, action_count: int | dict, **history +): + value = HistoryCritic(execution_config, interface_name, None, **history) + q1 = HistoryCritic(execution_config, interface_name, action_count, **history) + q2 = HistoryCritic(execution_config, interface_name, action_count, **history) targets = ( deepcopy(q1).requires_grad_(False), deepcopy(q2).requires_grad_(False), @@ -133,4 +196,6 @@ def make_iql_models(execution_config, interface_name: str, action_count: int | d {id(parameter) for module in modules for parameter in module.parameters()} ) != sum(1 for module in modules for _ in module.parameters()): raise ValueError("IQL networks share parameters") + if (source := history.get("history_source")) is not None: + validate_independent_networks(source, *modules) return value, q1, q2, targets diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index b81d7778..763387eb 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -9,7 +9,7 @@ load_online_adapter, require_exact_snapshot, ) -from sequifier.rl.models import HistoryCritic +from sequifier.rl.models import HistoryCritic, critic_history_options from sequifier.rl.ppo_collector import PPOCollector from sequifier.rl.ppo_update import update_ppo from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal @@ -18,8 +18,14 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): actor, phase = context.actor.eval(), context.phase - value = HistoryCritic( - deepcopy(actor.execution_config), actor.interface_name, None + value = HistoryCritic.ppo_value( + actor, + **critic_history_options( + actor, + context.run.phases, + context.index, + restore=context.resume_state is not None, + ), ).eval() session = PhaseSession(context, modules={"value": value}) state = session.state diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index d9c19210..2be85ac2 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -105,7 +105,13 @@ def build_actor(config, fitted, statistics): execution.embedding_layer_names = config.embedding_layer_names actor = Policy.from_network( build_transformer_network( - execution, device=torch.device(config.device) + execution, + device=torch.device(config.device), + restored_history_interface=( + config.policy_interface + if config.actor_initialization == "history_only" + else None + ), ).network, execution, config.policy_interface, @@ -167,9 +173,11 @@ def preflight_environments(config): def preflight_optimization(config, actor, data): """Resolve constructors and authored scheduler budgets before creating outputs.""" - from copy import deepcopy - - from sequifier.rl.models import HistoryCritic, make_iql_models + from sequifier.rl.models import ( + HistoryCritic, + critic_history_options, + make_iql_models, + ) baseline = { name: parameter.requires_grad @@ -177,19 +185,21 @@ def preflight_optimization(config, actor, data): } budgets, previous_masks = [], None try: - for phase, part in zip(config.phases, data): + for index, (phase, part) in enumerate(zip(config.phases, data)): for name, parameter in actor.network.named_parameters(): parameter.requires_grad_(baseline[name]) modules = {"actor": actor.network} + history = critic_history_options(actor, config.phases, index, restore=True) if phase.algorithm == "iql": value, q1, q2, _ = make_iql_models( - actor.execution_config, actor.interface_name, actor.config.actions + actor.execution_config, + actor.interface_name, + actor.config.actions, + **history, ) modules.update(value=value, critic=torch.nn.ModuleList([q1, q2])) elif phase.algorithm == "ppo": - modules["value"] = HistoryCritic( - deepcopy(actor.execution_config), actor.interface_name, None - ) + modules["value"] = HistoryCritic.ppo_value(actor, **history) masks = {} for role, module in modules.items(): frozen = phase.trainability.policy(role).resolve(module) diff --git a/src/sequifier/rl/references.py b/src/sequifier/rl/references.py index 737c441c..8a49eb61 100644 --- a/src/sequifier/rl/references.py +++ b/src/sequifier/rl/references.py @@ -17,12 +17,19 @@ def __init__(self, config, recovery, *, resuming): and config.initialize_from and Path(config.initialize_from).is_file() ): - _, identity = self._artifact(config.initialize_from) - require_equal( - recovery.initialization["weight_hash"], - identity, - "resume.initialize_from.weights", - ) + if config.actor_initialization == "history_only": + require_equal( + recovery.initialization["transfer"]["source_sha256"], + file_checksum(config.initialize_from), + "resume.initialize_from.sha256", + ) + else: + _, identity = self._artifact(config.initialize_from) + require_equal( + recovery.initialization["weight_hash"], + identity, + "resume.initialize_from.weights", + ) for index, phase in enumerate(config.phases): reference = getattr(phase, "reference", None) if reference is None or reference.source != "artifact": diff --git a/src/sequifier/rl/run_config.py b/src/sequifier/rl/run_config.py index fc1cffb9..9e77618e 100644 --- a/src/sequifier/rl/run_config.py +++ b/src/sequifier/rl/run_config.py @@ -1,6 +1,6 @@ """Resolved inputs for the native policy runner; never an authored config.""" -from typing import Any +from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, PrivateAttr @@ -27,6 +27,7 @@ class ResolvedPolicyRun(BaseModel): execution_model: dict policy_interface: str initialize_from: str | None = None + actor_initialization: Literal["attach", "history_only"] = "attach" normalization: FrozenNormalizationConfig | None = None sources: dict[str, SequenceSourceConfig] preferences: dict[str, str] diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 37a59ba2..66e992b8 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -75,6 +75,7 @@ def _supervised_config(config, phase, index, incoming, resume): run.project_root = config.project_root run.model_name = f"{config.model_name}-phase-{index:03d}-{phase.name}" run.execution.device = config.device + run.execution.precision.layer_type_dtypes = training.layer_type_dtypes run.reproducibility.seed = config.seed + index run.recovery.resume = ResumeConfig( policy="if_exists" if resume else "never", @@ -111,6 +112,7 @@ def _supervised_config(config, phase, index, incoming, resume): update={"phases": [phase_spec]} ) authored.initialize_from = None + authored.actor_initialization = "attach" authored.runtime_settings = authored._build_runtime_settings() metadata = { f"{name}.{part_name}": load_dataset_metadata( @@ -249,6 +251,16 @@ def run_learning_plan(config) -> LearningRunResult: state["incoming_sha256"], "committed incoming model artifact changed", ) + initialization = {"path": config.initialize_from} + if config.actor_initialization == "history_only" and state["completed"]: + initialization = state["completed"][0]["report"]["provenance"][ + "actor_initialization" + ] + verify_artifact( + config.initialize_from, + initialization["transfer"]["source_sha256"], + "history initialization artifact changed", + ) # Validate model/source/environment contracts for the complete plan first. from sequifier.rl.preparation import ( preflight_environments, @@ -261,10 +273,14 @@ def run_learning_plan(config) -> LearningRunResult: dependencies = {} protected_inputs = set() for index, phase in enumerate(phases): + if index == 0: + phase_incoming = config.initialize_from + elif index <= len(state["completed"]): + phase_incoming = state["completed"][index - 1]["model_path"] + else: + phase_incoming = None if phase.algorithm == "supervised": - resolved = _supervised_config( - config, phase, index, state["incoming"], False - ) + resolved = _supervised_config(config, phase, index, phase_incoming, False) from sequifier.io.output_paths import supervised_input_paths protected_inputs.update(supervised_input_paths(resolved)) @@ -276,14 +292,13 @@ def run_learning_plan(config) -> LearningRunResult: destination = PhaseArtifacts.for_phase( config.phase_directory, index, phase.name, "model.pt" ).model - if index == 0: - phase_incoming = config.initialize_from - elif index <= len(state["completed"]): - phase_incoming = state["completed"][index - 1]["model_path"] - else: - phase_incoming = state["incoming"] run = _policy_config( - config, phase, index, phase_incoming, destination, False + config, + phase, + index, + phase_incoming, + destination, + False, ) inputs = prepared_inputs.resolve(run) inputs.revalidate(inputs.identity, f"phases.{phase.name}.source_identity") @@ -366,6 +381,8 @@ def save_boundary(): ) artifacts.write_receipt(phase_identity, report) write_json_atomic(artifacts.report, report) + if index == 0 and config.actor_initialization == "history_only": + initialization = report["provenance"]["actor_initialization"] state["stopped"] = report.get("stop_scope") == "end_plan" state["completed"].append( { @@ -392,7 +409,7 @@ def save_boundary(): run_id=state["run_id"], session_id=session_id, source_provenance=dependencies, - initialization={"path": config.initialize_from}, + initialization=initialization, resume=resume.model_dump(mode="json"), exports=[config.model_path], status="complete", From b9b19b5338a3efcacc2f2b9a57a3e04150267838 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 10:39:39 +0100 Subject: [PATCH 49/71] single plan coordinator --- src/sequifier/artifacts/phase.py | 15 +- src/sequifier/config/train_config.py | 4 + src/sequifier/config/training_resolution.py | 49 +- src/sequifier/rl/checkpoint.py | 31 +- src/sequifier/rl/learner.py | 143 ++-- src/sequifier/rl/metrics.py | 7 +- src/sequifier/rl/run_checkpoint.py | 22 +- src/sequifier/rl/run_state.py | 33 +- src/sequifier/runtime/builder.py | 20 +- src/sequifier/runtime/plan.py | 76 ++ src/sequifier/runtime/stopping.py | 9 +- src/sequifier/train.py | 37 +- src/sequifier/training/coordinator.py | 347 ++++---- src/sequifier/training/engine.py | 847 ++++++++++---------- src/sequifier/training/phase_runners.py | 201 +++++ src/sequifier/training/state.py | 7 +- 16 files changed, 1073 insertions(+), 775 deletions(-) create mode 100644 src/sequifier/runtime/plan.py create mode 100644 src/sequifier/training/phase_runners.py diff --git a/src/sequifier/artifacts/phase.py b/src/sequifier/artifacts/phase.py index 99bc7f84..a9bb6ab4 100644 --- a/src/sequifier/artifacts/phase.py +++ b/src/sequifier/artifacts/phase.py @@ -28,27 +28,38 @@ def receipt(self): def describe(self): return {"actor_path": str(self.model), "report_path": str(self.report)} + def publish_model(self, artifact): + from sequifier.artifacts.model_artifact import save_model_artifact + + save_model_artifact(artifact, self.model) + + def publish_report(self, report): + write_json_atomic(self.report, report) + def record(self, report): return { "sha256": file_checksum(self.model), "result_fingerprint": fingerprint(report), } - def read_receipt(self, identity): + def read_receipt(self, identity, *, run_id=None): if not self.receipt.is_file(): return None receipt = json.loads(self.receipt.read_text()) + if run_id is not None and receipt.get("run_id") != run_id: + return None if receipt["identity"] != identity or receipt["sha256"] != file_checksum( self.model ): raise ValueError("completed phase artifact or receipt changed") return receipt - def write_receipt(self, identity, report): + def write_receipt(self, identity, report, *, run_id=None): write_json_atomic( self.receipt, { "identity": identity, + "run_id": run_id, "sha256": file_checksum(self.model), "report": report, }, diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 14cb2e64..673c661f 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -913,6 +913,10 @@ class ResolvedSequifierConfig(ConfigModel): """Runtime configuration after all dataset parts and interfaces resolve.""" _prepared_inputs: Any = PrivateAttr(default=None) + # Mixed plans keep prepared loader views alongside their model execution view. + _supervised_datasets: dict[str, ResolvedDatasetTrainingSpec] = PrivateAttr( + default_factory=dict + ) ownership: dict[str, Any] = Field(default_factory=dict) source_validation_fractions: dict[str, float] = Field(default_factory=dict) diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 05712672..01a7b4c5 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -16,7 +16,6 @@ TrainingRuntimeSettings, _resolve_interface, resolve_sequifier_config, - resolve_supervised_phases, ) from sequifier.special_tokens import SPECIAL_TOKEN_IDS @@ -558,17 +557,29 @@ def resolve_training_run(authored: SequifierConfig): and previous.model_interface != current.model_interface ): raise ValueError("optimizer continuation cannot change model_interface") - if all(phase.algorithm == "supervised" for phase in config.training_plan.phases): - subset = _supervised_subset(config).model_copy(update={"initialize_from": None}) - resolved = resolve_sequifier_config(subset, part_metadata, execution=execution) - else: - # The public resolved training schema remains the serialization boundary. + supervised = None + if any(phase.algorithm == "supervised" for phase in config.training_plan.phases): + subset = _supervised_subset(config).model_copy( + update={"initialize_from": None, "actor_initialization": "attach"} + ) + supervised = resolve_sequifier_config( + subset, part_metadata, execution=execution + ) + policy_phases = any( + phase.algorithm != "supervised" for phase in config.training_plan.phases + ) + if policy_phases: resolved = execution.training_config( device=config.run.execution.device, project_root=str(root), model_name=config.run.model_name, seed=config.run.reproducibility.seed, ) + if supervised is not None: + resolved._supervised_datasets = supervised.dataset_training + else: + assert supervised is not None + resolved = supervised resolved.ownership = config.model_dump(mode="json") resolved.source_validation_fractions = _source_fractions(config) resolved.normalization_by_interface = { @@ -592,26 +603,16 @@ def resolve_training_run(authored: SequifierConfig): resolve_training_output_paths(resolved) resolved.sequence_sources = sequences resolved.preference_sources = preferences - if any(phase.algorithm != "supervised" for phase in config.training_plan.phases): - supervised = { - phase.name: phase - for phase in resolve_supervised_phases( - [ - phase - for phase in config.training_plan.phases - if phase.algorithm == "supervised" - ] - ) - } + if policy_phases: + supervised_by_name = ( + {phase.name: phase for phase in supervised.training_plan} + if supervised is not None + else {} + ) resolved.training_plan = [ - supervised.get(phase.name, phase) for phase in config.training_plan.phases + supervised_by_name.get(phase.name, phase) + for phase in config.training_plan.phases ] - resolved.validate_next_occurrence_phases( - { - name: dataset.model_interface - for name, dataset in config.dataset_training.items() - } - ) for index, phase in enumerate(resolved.training_plan): if phase.algorithm == "supervised": phase.resolved_seed = config.run.reproducibility.seed + index diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 6c39a835..78fc4fda 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -30,21 +30,24 @@ def validate_policy_weights(policy: Policy) -> None: def save_policy( policy: Policy, path: str | Path, *, metadata: dict | None = None ) -> None: + save_model_artifact(policy_model_artifact(policy, metadata=metadata), path) + + +def policy_model_artifact( + policy: Policy, *, metadata: dict | None = None +) -> ModelArtifact: validate_policy_weights(policy) - save_model_artifact( - build_model_artifact( - policy.network, - policy.execution_config, - provenance={ - **(metadata or {}), - "transfer": getattr(policy, "transfer_report", None), - }, - state_dict={ - name: value.detach().cpu() - for name, value in policy.network.state_dict().items() - }, - ), - path, + return build_model_artifact( + policy.network, + policy.execution_config, + provenance={ + **(metadata or {}), + "transfer": getattr(policy, "transfer_report", None), + }, + state_dict={ + name: value.detach().cpu() + for name, value in policy.network.state_dict().items() + }, ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 503a0177..0abdb7b1 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -1,7 +1,10 @@ -"""Sequential policy coordination over prepared data and validated recovery state.""" +"""Native policy phases over prepared data and validated recovery state.""" + +from __future__ import annotations from dataclasses import replace from pathlib import Path +from typing import TYPE_CHECKING import torch @@ -37,8 +40,13 @@ require_equal, weight_fingerprint, ) +from sequifier.runtime.plan import PhaseOutcome, RunIdentity, execute_phase, run_phases from sequifier.runtime.random_state import capture_rng_state, restore_rng_state +if TYPE_CHECKING: + from sequifier.config.train_config import ResolvedSequifierConfig, SequifierConfig + from sequifier.training.coordinator import LearningRunResult + def _plan_contract(config): values = config.model_dump(mode="json") @@ -132,27 +140,38 @@ def _initialize_actor(config): return _new_actor(config) -def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: +def run_policy_training( + config: ResolvedPolicyRun | ResolvedSequifierConfig | SequifierConfig, +) -> PolicyRunResult | LearningRunResult: """Validate the whole run, execute phase-local lifecycles, and export its final actor.""" - if not isinstance(config, ResolvedPolicyRun): - from sequifier.config.train_config import ( - ResolvedSequifierConfig, - SequifierConfig, - ) - from sequifier.config.training_resolution import ( - policy_run_config, - resolve_training_run, - ) + if isinstance(config, ResolvedPolicyRun): + return _run_policy(config) - if isinstance(config, SequifierConfig): - config = resolve_training_run(config) - if not isinstance(config, ResolvedSequifierConfig) or any( - phase.algorithm == "supervised" for phase in config.training_plan - ): - raise TypeError( - "policy training requires a resolved policy-only training plan" - ) - config = policy_run_config(config, config.training_plan) + from sequifier.config.train_config import ResolvedSequifierConfig, SequifierConfig + from sequifier.config.training_resolution import resolve_training_run + from sequifier.training.coordinator import run_policy_plan + + if isinstance(config, SequifierConfig): + config = resolve_training_run(config) + if not isinstance(config, ResolvedSequifierConfig) or any( + phase.algorithm == "supervised" for phase in config.training_plan + ): + raise TypeError("policy training requires a resolved policy-only training plan") + return run_policy_plan(config) + + +def run_policy_phase( + config: ResolvedPolicyRun, identity: RunIdentity, *, legacy_recovery: bool = False +) -> PolicyRunResult: + """Run one prepared policy phase for a portable handoff.""" + if len(config.phases) != 1: + raise ValueError("a portable policy phase requires one resolved phase") + return _run_policy( + config, phase_only=True, identity=identity, legacy_recovery=legacy_recovery + ) + + +def _run_policy(config, *, phase_only=False, identity=None, legacy_recovery=False): torch.set_float32_matmul_precision("highest") configure_determinism(config.seed, config.enforce_determinism) plan = _plan_contract(config) @@ -162,7 +181,15 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: validate_output_paths(config, inputs) configure_logger(config.project_root, config.model_name, 0) checkpoint = Path(config.checkpoint_path) - recovery = RunCheckpoint.load(checkpoint, config, plan, data_identity) + recovery = RunCheckpoint.load( + checkpoint, + config, + plan, + data_identity, + run_id=identity.run_id + if identity is not None and not legacy_recovery + else None, + ) resuming = recovery is not None adapter_contract = preflight_environments(config) if recovery is not None: @@ -214,7 +241,9 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: ) from sequifier.rl.metrics import PolicyMetrics - config._metrics_service = PolicyMetrics(config, recovery.run_id) + identity = identity or RunIdentity.start(recovery.run_id) + recovery.run_id = identity.run_id + config._metrics_service = PolicyMetrics(config, identity) references = PolicyReferences(config, recovery, resuming=resuming) saved_random = capture_rng_state(actor.device) step_budgets = preflight_optimization(config, actor, data) @@ -224,10 +253,9 @@ def run_policy_training(config: ResolvedPolicyRun) -> PolicyRunResult: # Persist the complete fitted representation before the first update. recovery.save(checkpoint, actor.device) recovery.write_reports(checkpoint) - while recovery.phase_index < len(config.phases) and not recovery.stopped: - phase_index = recovery.phase_index + + def run_phase(phase_index, phase): current_state = recovery.phase_state - phase = config.phases[phase_index] inputs.revalidate( data_identity, f"phases.{phase.name}.source_identity", @@ -276,9 +304,9 @@ def commit(state): commit, step_budgets[phase_index], ) - actor, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) + selected, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) representation.validate_actor( - actor, config.units, f"phases.{phase.name}.selected_actor" + selected, config.units, f"phases.{phase.name}.selected_actor" ) actor_path = phase_actor_path(config.phase_directory, phase_index, phase.name) result = replace( @@ -293,32 +321,49 @@ def commit(state): **PhaseArtifacts(actor_path).describe(), }, ) + return PhaseOutcome((selected, result), result.stop_scope) + + def commit_phase(index, phase, outcome): + nonlocal actor + actor, result = outcome.result recovery.continuation = getattr(actor, "_phase_continuation", None) recovery.finish_phase(actor, result) recovery.save(checkpoint, actor.device) recovery.write_reports(checkpoint) - from sequifier.artifacts.reports import publish_run_report, run_report - - report = run_report( - model_path=config.model_path, - checkpoint_path=checkpoint, - phases=recovery.phase_reports(checkpoint), - run_id=recovery.run_id, - session_id=config._metrics_service.session_id, - status="complete", - completion_reason="early_stopping" if recovery.stopped else "completed", - exports=[config.model_path], - plan=plan, - plan_fingerprint=fingerprint(plan), - representation_fingerprint=representation.identity, - source_provenance=data_identity, - reference_provenance=recovery.reference_contract, - initialization=recovery.initialization, - resume=config.resume.model_dump(mode="json"), - ) - save_policy(actor.eval(), config.model_path, metadata=report) - publish_run_report(report, config.report_path) - recovery.save(checkpoint, actor.device) + + if phase_only: + if not recovery.results: + execute_phase(0, config.phases[0], run_phase, commit_phase) + else: + completion_reason = run_phases( + config.phases, + run_phase, + commit_phase, + start=recovery.phase_index, + stopped=recovery.stopped, + ) + from sequifier.artifacts.reports import publish_run_report, run_report + + report = run_report( + model_path=config.model_path, + checkpoint_path=checkpoint, + phases=recovery.phase_reports(checkpoint), + run_id=recovery.run_id, + session_id=config._metrics_service.session_id, + status="complete", + completion_reason=completion_reason, + exports=[config.model_path], + plan=plan, + plan_fingerprint=fingerprint(plan), + representation_fingerprint=representation.identity, + source_provenance=data_identity, + reference_provenance=recovery.reference_contract, + initialization=recovery.initialization, + resume=config.resume.model_dump(mode="json"), + ) + save_policy(actor.eval(), config.model_path, metadata=report) + publish_run_report(report, config.report_path) + recovery.save(checkpoint, actor.device) return PolicyRunResult( actor, tuple(recovery.results), diff --git a/src/sequifier/rl/metrics.py b/src/sequifier/rl/metrics.py index 0ee42907..79c95fa5 100644 --- a/src/sequifier/rl/metrics.py +++ b/src/sequifier/rl/metrics.py @@ -2,21 +2,20 @@ import math import time -import uuid from sequifier.training.metrics import StructuredMetricWriters class PolicyMetrics: - def __init__(self, config, run_id): + def __init__(self, config, identity): self.writer = StructuredMetricWriters( config.project_root, config.model_name, 0, dataset_name=config.policy_interface, ) - self.run_id = run_id - self.session_id = uuid.uuid4().hex + self.run_id = identity.run_id + self.session_id = identity.session_id self.started = time.monotonic() self.pending = None diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 2ce9f316..6cd4807d 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -1,7 +1,6 @@ """Policy recovery with artifact-backed, authoritative committed phase boundaries.""" import math -import uuid from dataclasses import asdict, dataclass, field, fields from pathlib import Path @@ -18,8 +17,8 @@ ) from sequifier.artifacts.phase import PhaseArtifacts from sequifier.fingerprints import fingerprint -from sequifier.io.atomic import atomic_save, write_json_atomic -from sequifier.rl.checkpoint import phase_actor_path, save_policy +from sequifier.io.atomic import atomic_save +from sequifier.rl.checkpoint import phase_actor_path, policy_model_artifact from sequifier.rl.references import validate_reference_state from sequifier.rl.run_state import ( PHASE_STATE_VERSION, @@ -29,6 +28,7 @@ require_equal, weight_fingerprint, ) +from sequifier.runtime.plan import new_run_id from sequifier.runtime.random_state import capture_rng_state from sequifier.trajectory.sources import file_checksum @@ -67,7 +67,7 @@ class RunCheckpoint: phase_state: dict | None = None continuation: dict | None = None stopped: bool = False - run_id: str = field(default_factory=lambda: uuid.uuid4().hex) + run_id: str = field(default_factory=new_run_id) @property def results(self): @@ -78,7 +78,7 @@ def phase_index(self): return len(self.completed) @classmethod - def load(cls, path, config, plan, data_identity): + def load(cls, path, config, plan, data_identity, *, run_id=None): path = Path(path) if config.resume.policy == "never": return None @@ -97,6 +97,8 @@ def load(cls, path, config, plan, data_identity): raise ValueError( "policy resume requires a version-7 policy run checkpoint; initialize from its model artifact to start a new run" ) + if run_id is not None and payload.get("run_id") != run_id: + return None try: saved_plan = payload["plan"] require_equal(plan, saved_plan, "resume.plan") @@ -256,13 +258,11 @@ def finish_phase(self, actor, result): weight_fingerprint(actor.network.state_dict()), "handoff.selected_actor", ) - path = result.provenance["actor_path"] - save_policy(actor, path, metadata=result.as_dict()) + artifacts = PhaseArtifacts(Path(result.provenance["actor_path"])) + artifacts.publish_model(policy_model_artifact(actor, metadata=result.as_dict())) weights = clone_weights(actor.network) self.completed.append( - CompletedPhase( - result, **PhaseArtifacts(Path(path)).record(result.as_dict()) - ) + CompletedPhase(result, **artifacts.record(result.as_dict())) ) self.stopped = result.stop_scope == "end_plan" self.incoming_actor = weights @@ -283,7 +283,7 @@ def phase_reports(self, checkpoint_path): def write_reports(self, checkpoint_path): """Reports are derived views; deleting one does not invalidate recovery.""" for result, report in zip(self.results, self.phase_reports(checkpoint_path)): - write_json_atomic(result.provenance["report_path"], report) + PhaseArtifacts(Path(result.provenance["actor_path"])).publish_report(report) def validate_finite_state(value, field): diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 7cdbbd40..b6fc974b 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -12,6 +12,7 @@ from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_plan import PhaseConfig from sequifier.rl.policy import Policy +from sequifier.runtime.plan import metric_improved, select_handoff, stop_scope from sequifier.runtime.random_state import capture_rng_state, restore_rng_state from sequifier.runtime.stopping import StoppingState @@ -164,11 +165,7 @@ def observe(self, actor, metric, transaction): self.last_metric = metric if metric is None: return - if self.best_metric is None or ( - metric < self.best_metric - if self.direction == "min" - else metric > self.best_metric - ): + if metric_improved(metric, self.best_metric, self.direction): self.best_metric = float(metric) self.best_weights = clone_weights(actor.network) self.best_transaction = transaction @@ -184,19 +181,21 @@ def state_dict(self): ) } - def finish(self, context, transaction, clocks, metrics, provenance): + def finish(self, context, transaction: int, clocks, metrics, provenance): actor = context.actor.eval() + if self.phase.handoff == "best" and self.best_weights is None: + raise ValueError( + f"phases.{self.phase.name}: handoff best requires a finite {self.monitor}" + ) + selected, metric = select_handoff( + self.phase.handoff, + (transaction, self.last_metric), + (self.best_transaction, self.best_metric), + ) + if selected is None: + raise ValueError(f"{self.phase.name}: no selected transaction") if self.phase.handoff == "best": - if self.best_weights is None: - raise ValueError( - f"phases.{self.phase.name}: handoff best requires a finite {self.monitor}" - ) actor.network.load_state_dict(self.best_weights, strict=True) - if self.best_transaction is None: - raise ValueError("best actor has no selected transaction") - selected, metric = self.best_transaction, self.best_metric - else: - selected, metric = transaction, self.last_metric result = PolicyPhaseResult( context.index, self.phase.name, @@ -603,7 +602,5 @@ def finish(self, clocks, provenance): return actor, replace( result, - stop_scope=self.context.phase.stopping.on_trigger - if self.stopping.triggered - else None, + stop_scope=stop_scope(self.context.phase.stopping, self.stopping.triggered), ) diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 4366dcb7..e619cd26 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -2,7 +2,6 @@ from __future__ import annotations -import uuid from collections.abc import Callable from dataclasses import dataclass from pathlib import Path @@ -24,6 +23,7 @@ semantic_optimizer_groups, ) from sequifier.runtime.context import ExecutionEnvironment, RunContext +from sequifier.runtime.plan import RunIdentity from sequifier.runtime.precision import PrecisionPolicy from sequifier.runtime.random_state import RandomStateManager from sequifier.training.checkpoint_service import CheckpointRestorer, CheckpointService @@ -168,6 +168,8 @@ def build( integrations: IntegrationManager, *, initial_artifact=None, + identity: RunIdentity | None = None, + legacy_recovery: bool = False, ) -> TrainingRun: from sequifier.artifacts.run_checkpoint import ( checkpoint_path as output_checkpoint_path, @@ -195,12 +197,25 @@ def build( restorer = CheckpointRestorer() checkpoint_path = select_run_checkpoint(config) loaded = restorer.load(checkpoint_path) if checkpoint_path is not None else None + if ( + loaded is not None + and identity is not None + and not legacy_recovery + and loaded.checkpoint.run_state["run_id"] != identity.run_id + ): + loaded = None if loaded is not None: restorer.validate_for_restore(loaded, config) state = RunState.from_state_dict(loaded.checkpoint.run_state) - state.session_id = uuid.uuid4().hex else: state = RunState() + if identity is None: + identity = strategy.gather_objects( + RunIdentity.start(state.run_id) if strategy.rank == 0 else None + )[0] + if not isinstance(identity, RunIdentity): + raise RuntimeError("Rank zero did not provide a run identity") + state.run_id, state.session_id = identity.run_id, identity.session_id validate_phase_learning(config, execution.device) @@ -371,6 +386,7 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: self._construct_loaders(run) if loaded is not None: restorer.restore_runtime(loaded, run) + run.state.run_id = context.run_id run.state.session_id = context.session_id self._warm_up(run) if loaded is not None: diff --git a/src/sequifier/runtime/plan.py b/src/sequifier/runtime/plan.py new file mode 100644 index 00000000..bc860324 --- /dev/null +++ b/src/sequifier/runtime/plan.py @@ -0,0 +1,76 @@ +"""Common phase ordering, commit boundaries, and plan stopping.""" + +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Generic, TypeVar +from uuid import uuid4 + +Phase = TypeVar("Phase") +Result = TypeVar("Result") + + +def new_run_id() -> str: + return uuid4().hex + + +@dataclass(frozen=True) +class RunIdentity: + run_id: str + session_id: str + + @classmethod + def start(cls, run_id: str | None = None): + return cls(run_id or new_run_id(), new_run_id()) + + +@dataclass(frozen=True) +class PhaseOutcome(Generic[Result]): + result: Result + stop_scope: str | None = None + completion_reason: str | None = None + + +def stop_scope(stopping, triggered: bool) -> str | None: + return stopping.on_trigger if triggered and stopping is not None else None + + +def metric_improved(value, best, direction, min_delta=0) -> bool: + if best is None: + return True + return value < best - min_delta if direction == "min" else value > best + min_delta + + +def select_handoff(handoff: str, last: Result, best: Result | None) -> Result: + return best if handoff == "best" and best is not None else last + + +def execute_phase( + index: int, + phase: Phase, + run_phase: Callable[[int, Phase], PhaseOutcome[Result]], + commit_phase: Callable[[int, Phase, PhaseOutcome[Result]], None], +) -> PhaseOutcome[Result]: + outcome = run_phase(index, phase) + commit_phase(index, phase, outcome) + return outcome + + +def run_phases( + phases: Sequence[Phase], + run_phase: Callable[[int, Phase], PhaseOutcome[Result]], + commit_phase: Callable[[int, Phase, PhaseOutcome[Result]], None], + *, + start: int = 0, + stopped: bool = False, + completion_reason: str = "completed", +) -> str: + """Commit each native result before acting on its stop or advancing.""" + if stopped: + return "early_stopping" + for index in range(start, len(phases)): + outcome = execute_phase(index, phases[index], run_phase, commit_phase) + if outcome.stop_scope == "end_plan": + return "early_stopping" + if outcome.completion_reason is not None: + return outcome.completion_reason + return completion_reason diff --git a/src/sequifier/runtime/stopping.py b/src/sequifier/runtime/stopping.py index 8324d36e..b6701a96 100644 --- a/src/sequifier/runtime/stopping.py +++ b/src/sequifier/runtime/stopping.py @@ -3,6 +3,8 @@ import math from dataclasses import dataclass +from sequifier.runtime.plan import metric_improved + @dataclass class StoppingState: @@ -17,12 +19,7 @@ def observe(self, config, value, direction, observation): if self.last_observation is not None and observation <= self.last_observation: return self.triggered self.last_observation = observation - improved = self.best is None or ( - value < self.best - config.min_delta - if direction == "min" - else value > self.best + config.min_delta - ) - if improved: + if metric_improved(value, self.best, direction, config.min_delta): self.best = float(value) self.without_improvement = 0 else: diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 00f8c502..5f2f2384 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -34,6 +34,7 @@ from sequifier.model.tracing import CaptureRequest # noqa: E402 from sequifier.runtime.builder import RunBuilder # noqa: E402 from sequifier.runtime.context import ExecutionEnvironment # noqa: E402 +from sequifier.training.coordinator import coordinate_training # noqa: E402 from sequifier.training.engine import TrainingEngine # noqa: E402 from sequifier.typechecking import beartype # noqa: E402 @@ -179,29 +180,23 @@ def run_training( ) -> Any: """Launch canonical service-composed training locally or across workers.""" - if isinstance(config, TrainModel): - from sequifier.io.output_paths import resolve_training_output_paths + return coordinate_training( + config, + launch_supervised=launch_supervised, + integration_specs=integration_specs, + integration_instances=integration_instances, + semantic_optimizer_grouping=semantic_optimizer_grouping, + ) - resolve_training_output_paths(config) - if isinstance(config, TrainModel) and any( - phase.algorithm != "supervised" for phase in config.training_plan - ): - if integration_specs or integration_instances or semantic_optimizer_grouping: - raise ValueError( - "policy phases do not support custom integrations or parameter grouping" - ) - from sequifier.config.training_resolution import policy_run_config - from sequifier.rl.learner import run_policy_training - from sequifier.training.coordinator import run_learning_plan - if ( - all(phase.algorithm != "supervised" for phase in config.training_plan) - and len({phase.model_interface for phase in config.training_plan}) == 1 - ): - return run_policy_training(policy_run_config(config, config.training_plan)) - return run_learning_plan(config) - if not isinstance(config, TrainModel): - raise TypeError("Training requires a canonical resolved config.") +def launch_supervised( + config: TrainModel, + *, + integration_specs: tuple[IntegrationSpec, ...], + integration_instances: tuple[Any, ...], + semantic_optimizer_grouping: bool, +) -> None: + """Launch the native supervised runtime in its execution environment.""" spec = config.runtime_settings if spec.distributed and integration_instances: raise ValueError( diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 66e992b8..bb309114 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -5,7 +5,6 @@ Completing a phase publishes its selected artifact before advancing the envelope. """ -import uuid from dataclasses import dataclass from pathlib import Path from typing import Any @@ -17,15 +16,22 @@ recovery_payload, ) from sequifier.artifacts.model_artifact import ( - build_model_artifact, + ModelArtifact, read_execution_artifact, save_model_artifact, ) from sequifier.artifacts.phase import PhaseArtifacts, verify_artifact from sequifier.artifacts.reports import phase_report, publish_run_report, run_report -from sequifier.config.train_config import resolve_sequifier_config from sequifier.fingerprints import fingerprint -from sequifier.io.atomic import atomic_save, write_json_atomic +from sequifier.io.atomic import atomic_save +from sequifier.runtime.plan import PhaseOutcome, RunIdentity, new_run_id, run_phases +from sequifier.training.phase_runners import ( + PortablePhaseRequest, + policy_phase_config, + run_portable_policy_phase, + run_supervised_phase, + supervised_phase_config, +) from sequifier.trajectory.sources import file_checksum @@ -37,183 +43,67 @@ class LearningRunResult: phases: tuple[dict, ...] -def _supervised_config(config, phase, index, incoming, resume): - from sequifier.artifacts.model_config import ( - extend_prediction_distributions, - model_execution_from_model_config, - validate_execution_attachment, - ) - from sequifier.artifacts.model_export import model_execution_config - from sequifier.config.components import ResumeConfig - from sequifier.config.metadata import load_dataset_metadata - from sequifier.config.train_config import OwnedModelSpec - from sequifier.model.execution import ModelExecution - - authored = config.authored.model_copy(deep=True) - phase_spec = authored.training_plan.phases[index] - phase_spec.optimization = phase.optimization - training = config.runtime_settings.model_copy(deep=True) - if training.distributed: - raise ValueError("mixed learning plans currently require local phase execution") - artifact = read_execution_artifact(incoming) if incoming is not None else None - source = config - if artifact is not None: - source, _ = model_execution_from_model_config( - artifact.model_config.values, device=config.device - ) - extend_prediction_distributions(source, config.model) - training.forward_attention = source.runtime_settings.forward_attention or ( - "bidirectional" - if source.runtime_settings.training_objective == "bert" - else "causal" - ) - execution = ModelExecution.from_config(source) - native = PhaseArtifacts.for_phase( - config.phase_directory, index, phase.name - ).model.parent - run = authored.run - run.project_root = config.project_root - run.model_name = f"{config.model_name}-phase-{index:03d}-{phase.name}" - run.execution.device = config.device - run.execution.precision.layer_type_dtypes = training.layer_type_dtypes - run.reproducibility.seed = config.seed + index - run.recovery.resume = ResumeConfig( - policy="if_exists" if resume else "never", - checkpoint_path=str(native / "run.pt"), - ) - run.export.pt = True - run.logging.log_interval_iterations = 10 - for name, path in { - "model_path": native / "native/model.pt", - "report_path": native / "native/report.json", - "phase_directory": native / "native/phases", - }.items(): - setattr(run.outputs, name, str(path)) - authored.model = OwnedModelSpec.from_execution( - execution, - authored.model, - settings={ - key: getattr(training, key) - for key in ( - "context_length", - "training_objective", - "target_offset", - "forward_attention", - ) - }, - ) - names = {ref.dataset for ref in [*phase.sources, *phase.evaluation_sources]} - authored.dataset_training = { - name: value - for name, value in authored.dataset_training.items() - if name in names - } - authored.training_plan = authored.training_plan.model_copy( - update={"phases": [phase_spec]} - ) - authored.initialize_from = None - authored.actor_initialization = "attach" - authored.runtime_settings = authored._build_runtime_settings() - metadata = { - f"{name}.{part_name}": load_dataset_metadata( - str(Path(config.project_root) / part.metadata_config_path) - ) - for name, dataset in authored.dataset_training.items() - for part_name, part in dataset.parts.items() - } - resolved = resolve_sequifier_config(authored, metadata, execution=execution) - resolved.ownership = config.ownership - resolved.source_validation_fractions = config.source_validation_fractions - resolved.normalization_by_interface = config.normalization_by_interface - if artifact is not None: - execution.retain_interfaces(resolved) - validate_execution_attachment( - artifact.model_config.values, model_execution_config(resolved) - ) - return resolved - +@dataclass(frozen=True) +class PortablePhaseCommit: + artifacts: PhaseArtifacts + fingerprint: str + model: ModelArtifact | None + report: dict -def _run_supervised(config, phase, index, incoming, destination, resume): - from sequifier.helpers import configure_determinism, configure_logger - from sequifier.integration import IntegrationManager - from sequifier.runtime.builder import RunBuilder - from sequifier.runtime.context import ExecutionEnvironment - from sequifier.training.engine import TrainingEngine - resolved = _supervised_config(config, phase, index, incoming, resume) - from sequifier.artifacts.run_checkpoint import checkpoint_path - from sequifier.io.output_paths import ( - supervised_input_paths, - validate_training_destinations, - ) +def coordinate_training( + config, + *, + launch_supervised, + integration_specs, + integration_instances, + semantic_optimizer_grouping, +): + """Choose native execution and recovery services for the resolved plan.""" + from sequifier.config.train_config import ResolvedSequifierConfig + from sequifier.io.output_paths import resolve_training_output_paths - validate_training_destinations( - resolved, checkpoint_path(resolved), supervised_input_paths(resolved) - ) - configure_logger(config.project_root, resolved.model_name, 0) - configure_determinism( - config.seed + index, resolved.runtime_settings.enforce_determinism - ) - environment = ExecutionEnvironment( - rank=0, - local_rank=0, - world_size=1, - device=torch.device(config.device), - distributed=False, - ) - integrations = IntegrationManager(rank=0, world_size=1, distributed=False) - run = RunBuilder().build( - resolved, environment, integrations, initial_artifact=incoming - ) - try: - result = TrainingEngine().run(run) - if phase.handoff == "best" and run.state.best_model_state_dict: - run.network.load_state_dict(run.state.best_model_state_dict, strict=True) - save_model_artifact(build_model_artifact(run.network, resolved), destination) - return { - **run.state.phase_reports[-1], - "completion_reason": result.completion_reason, - "resolved_seed": config.seed + index, - "stop_scope": phase.stopping.on_trigger - if run.state.stopping_state.get("triggered") - else None, - "optimizer_steps": run.state.optimizer_step, - "handoff": phase.handoff, - } - finally: - run.distributed.finalize() + if not isinstance(config, ResolvedSequifierConfig): + raise TypeError("Training requires a canonical resolved config.") + resolve_training_output_paths(config) + phases = config.training_plan + if all(phase.algorithm == "supervised" for phase in phases): + return launch_supervised( + config, + integration_specs=integration_specs, + integration_instances=integration_instances, + semantic_optimizer_grouping=semantic_optimizer_grouping, + ) + if integration_specs or integration_instances or semantic_optimizer_grouping: + raise ValueError( + "policy phases do not support custom integrations or parameter grouping" + ) + if all(phase.algorithm != "supervised" for phase in phases): + return run_policy_plan(config) + return run_learning_plan(config) -def _policy_config(config, phase, index, incoming, destination, resume): +def run_policy_plan(config): + """Use native policy recovery when the plan shares an actor representation.""" from sequifier.config.training_resolution import policy_run_config - - return policy_run_config( - config, - [phase], - incoming=incoming, - index=index, - destination=destination, - resuming=resume, - ) - - -def _run_policy(config, phase, index, incoming, destination, resume): from sequifier.rl.learner import run_policy_training - run = _policy_config(config, phase, index, incoming, destination, resume) - return run_policy_training(run).phases[0].as_dict() + if len({phase.model_interface for phase in config.training_plan}) == 1: + return run_policy_training(policy_run_config(config, config.training_plan)) + return run_learning_plan(config) def run_learning_plan(config) -> LearningRunResult: phases = config.training_plan + if any(phase.optimizer_state == "continue" for phase in phases): + raise ValueError("portable phase handoffs cannot continue optimizer state") resume = config.runtime_settings.resume from sequifier.config.components import ResumeConfig resume = resume or ResumeConfig() - checkpoint = Path(config.project_root) / ( - resume.checkpoint_path - or f"checkpoints/runs/{config.model_name}/{config.model_name}-latest.pt" - ) + from sequifier.artifacts.run_checkpoint import checkpoint_path + + checkpoint = checkpoint_path(config) plan = config.model_dump(mode="json") plan["runtime_settings"].pop("resume", None) plan.get("ownership", {}).get("run", {}).get("recovery", {}).pop("resume", None) @@ -228,7 +118,7 @@ def run_learning_plan(config) -> LearningRunResult: if config.initialize_from else None, "active": False, - "run_id": uuid.uuid4().hex, + "run_id": new_run_id(), } if resume.policy != "never" and checkpoint.is_file(): state = recovery_payload( @@ -242,8 +132,8 @@ def run_learning_plan(config) -> LearningRunResult: raise ValueError("learning run checkpoint differs from the requested plan") elif resume.policy == "required": raise FileNotFoundError(checkpoint) - session_id = uuid.uuid4().hex - state.setdefault("run_id", uuid.uuid4().hex) + identity_scope = RunIdentity.start(state.get("run_id")) + state["run_id"] = identity_scope.run_id if not 0 <= len(state["completed"]) <= len(phases): raise ValueError("learning run checkpoint exceeds the phase plan") verify_artifact( @@ -280,7 +170,9 @@ def run_learning_plan(config) -> LearningRunResult: else: phase_incoming = None if phase.algorithm == "supervised": - resolved = _supervised_config(config, phase, index, phase_incoming, False) + resolved = supervised_phase_config( + config, phase, index, phase_incoming, False + ) from sequifier.io.output_paths import supervised_input_paths protected_inputs.update(supervised_input_paths(resolved)) @@ -292,7 +184,7 @@ def run_learning_plan(config) -> LearningRunResult: destination = PhaseArtifacts.for_phase( config.phase_directory, index, phase.name, "model.pt" ).model - run = _policy_config( + run = policy_phase_config( config, phase, index, @@ -328,23 +220,18 @@ def save_boundary(): ) atomic_save(checkpoint, checkpoint_envelope("learning", state, model)) - for index in range(len(state["completed"]), len(phases)): - if state.get("stopped"): - break - phase = phases[index] + def run_phase(index, phase): incoming = state["incoming"] verify_artifact( incoming, state["incoming_sha256"], "incoming model artifact changed since the committed boundary", ) - artifacts = PhaseArtifacts.for_phase( - config.phase_directory, index, phase.name, "model.pt" - ) - destination = artifacts.model + artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) resuming = state["active"] and resume.policy != "never" + legacy_recovery = resuming and "active_run_id" not in state if not state["active"]: - state["active"] = True + state.update(active=True, active_run_id=identity_scope.run_id) save_boundary() phase_identity = fingerprint( { @@ -352,52 +239,100 @@ def save_boundary(): "incoming": state["incoming_sha256"], } ) - receipt = artifacts.read_receipt(phase_identity) if resuming else None + receipt = ( + artifacts.read_receipt(phase_identity, run_id=state.get("active_run_id")) + if resuming + else None + ) + result = None if receipt is not None: report = receipt["report"] else: - runner = _run_supervised if phase.algorithm == "supervised" else _run_policy - report = runner(config, phase, index, incoming, destination, resuming) + request = PortablePhaseRequest( + index, incoming, artifacts, resuming, legacy_recovery, identity_scope + ) + runner = ( + run_supervised_phase + if phase.algorithm == "supervised" + else run_portable_policy_phase + ) + result = runner(config, phase, request) + details = { + key: value + for key, value in result.report.items() + if key + not in { + "index", + "name", + "algorithm", + "model_path", + "checkpoint_path", + "artifact_type", + "format_version", + } + } + if phase.algorithm != "supervised": + details["provenance"] = { + **details["provenance"], + **artifacts.describe(), + } report = phase_report( index=index, name=phase.name, algorithm=phase.algorithm, - model_path=destination, - checkpoint_path=destination.with_name("run.pt"), - **{ - key: value - for key, value in report.items() - if key - not in { - "index", - "name", - "algorithm", - "model_path", - "checkpoint_path", - "artifact_type", - "format_version", - } - }, + model_path=artifacts.model, + checkpoint_path=artifacts.model.with_name("run.pt"), + **details, ) - artifacts.write_receipt(phase_identity, report) - write_json_atomic(artifacts.report, report) + return PhaseOutcome( + PortablePhaseCommit( + artifacts, phase_identity, result.artifact if result else None, report + ), + report.get("stop_scope"), + ) + + def commit_phase(index, phase, outcome): + nonlocal initialization + commit = outcome.result + artifacts, report = commit.artifacts, commit.report + if commit.model is not None: + artifacts.publish_model(commit.model) + artifacts.write_receipt( + commit.fingerprint, report, run_id=state.get("active_run_id") + ) + artifacts.publish_report(report) if index == 0 and config.actor_initialization == "history_only": initialization = report["provenance"]["actor_initialization"] - state["stopped"] = report.get("stop_scope") == "end_plan" + state["stopped"] = outcome.stop_scope == "end_plan" + checksum = file_checksum(artifacts.model) state["completed"].append( { "name": phase.name, "algorithm": phase.algorithm, - "model_path": str(destination), + "model_path": str(artifacts.model), "report": report, } ) state.update( - incoming=str(destination), - incoming_sha256=file_checksum(destination), + incoming=str(artifacts.model), + incoming_sha256=checksum, active=False, ) + state.pop("active_run_id", None) save_boundary() + + for index, item in enumerate(state["completed"]): + artifacts = PhaseArtifacts.for_phase( + config.phase_directory, index, phases[index].name + ) + artifacts.publish_report(item["report"]) + completion_reason = run_phases( + phases, + run_phase, + commit_phase, + start=len(state["completed"]), + stopped=state.get("stopped", False), + ) final = read_execution_artifact(state["incoming"]) save_model_artifact(final, config.model_path) publish_run_report( @@ -407,13 +342,13 @@ def save_boundary(): phases=[item["report"] for item in state["completed"]], plan_fingerprint=identity, run_id=state["run_id"], - session_id=session_id, + session_id=identity_scope.session_id, source_provenance=dependencies, initialization=initialization, resume=resume.model_dump(mode="json"), exports=[config.model_path], status="complete", - completion_reason="early_stopping" if state.get("stopped") else "completed", + completion_reason=completion_reason, ), config.report_path, ) diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 1bdcc077..c9525407 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import os import time from contextlib import nullcontext @@ -30,8 +31,15 @@ StepIdentity, ValidationCompleted, ) -from sequifier.io.atomic import write_json_atomic from sequifier.logging_paths import model_log_directory +from sequifier.runtime.plan import ( + PhaseOutcome, + execute_phase, + metric_improved, + run_phases, + select_handoff, + stop_scope, +) from sequifier.runtime.random_state import RandomState from sequifier.runtime.stopping import StoppingState from sequifier.training.checkpoint_service import CheckpointRequest @@ -184,15 +192,11 @@ def _update_monitor( "inf" ): run.state.best_validation_loss = float("-inf") - import math - if not math.isfinite(value): return value, False run.state.last_validation_metric = value - improved = ( - value < run.state.best_validation_loss - if monitor.direction == "min" - else value > run.state.best_validation_loss + improved = metric_improved( + value, run.state.best_validation_loss, monitor.direction ) if improved: run.state.best_validation_loss = value @@ -255,21 +259,14 @@ def _publish_phase(self, run, index, phase): last = run.distributed.capture_model_state(run.network) if run.distributed.rank != 0: return - selected = ( - (run.state.best_model_state_dict or last) - if phase.handoff == "best" - else last - ) + selected = select_handoff(phase.handoff, last, run.state.best_model_state_dict) artifacts = PhaseArtifacts.for_phase( run.config.phase_directory, index, phase.name ) - save_model_artifact( - build_model_artifact(run.network, run.config, state_dict=selected), - artifacts.model, + artifacts.publish_model( + build_model_artifact(run.network, run.config, state_dict=selected) ) metric = run.state.best_validation_loss - import math - report = phase_report( index=index, name=phase.name, @@ -294,16 +291,29 @@ def _publish_phase(self, run, index, phase): "epochs": run.state.phase_epoch, "optimizer_steps": run.state.optimizer_step, }, - stop_scope=phase.stopping.on_trigger - if phase.stopping and run.state.stopping_state.get("triggered") - else None, + stop_scope=stop_scope( + phase.stopping, run.state.stopping_state.get("triggered") + ), ) - write_json_atomic(artifacts.report, report) + artifacts.publish_report(report) run.state.phase_reports = [ r for r in run.state.phase_reports if r["index"] != index ] + [report] def run(self, run: Any) -> RunResult: + return self._run(run, phase_only=False) + + def run_phase(self, run: Any) -> dict: + """Run one prepared supervised phase for a portable handoff.""" + if len(run.config.training_plan) != 1: + raise ValueError("a portable supervised phase requires one resolved phase") + result = self._run(run, phase_only=True) + return { + **run.state.phase_reports[-1], + "completion_reason": result.completion_reason, + } + + def _run(self, run: Any, *, phase_only: bool) -> RunResult: config = run.config state = run.state time_checkpoint_intervals_enabled = bool( @@ -406,424 +416,425 @@ def flush_metrics( ) current_batch = sum(state.iterator_positions.values()) current_batches_total = 1 - try: - # A checkpoint may have been committed immediately before phase - # publication. Rebuild that derived record before advancing. - if ( - state.phase_epoch_complete - and ( - state.plan_stopped - or state.phase_epoch - >= config.training_plan[state.phase_index].epochs - or state.stopping_state.get("triggered") - ) - and not any( - r["index"] == state.phase_index for r in state.phase_reports + + def run_phase(phase_index, phase): + nonlocal access, active_dataset, accumulation_count, last_identity + nonlocal boundary, boundary_identity, batch_in_progress, batch_committed + nonlocal last_snapshot, last_snapshot_step, last_latest + nonlocal current_epoch, current_batch, current_batches_total + nonlocal stop_requested, completion_reason + run.metrics.set_phase(phase) + if phase_index != state.phase_index: + previous = config.training_plan[phase_index - 1] + if previous.handoff == "best": + selected = run.distributed.gather_objects( + state.best_model_state_dict + )[0] + if selected is None: + raise ValueError( + f"{previous.name}: best handoff has no finite evaluation" + ) + run.distributed.restore_model_state(run.network, selected) + if not compatible_supervised_phases( + config, phase_index - 1, phase_index + ): + state.best_validation_loss = float("inf") + state.best_model_state_dict = None + state.epochs_without_improvement = 0 + state.stopping_state = {} + state.last_validation_metric = None + last_snapshot = time.monotonic() + last_snapshot_step = state.global_batch_step + state.last_snapshot_batch_step = last_snapshot_step + phase_stopping = StoppingState(**state.stopping_state) + phase_stop = phase_stopping.triggered + if run.optimization_phase_index != phase_index: + from sequifier.training.runtime import configure_phase + + configure_phase( + config, + run.network, + run.datasets, + phase_index, + run.distributed.device, ) + if ( + phase.evaluation + and phase.evaluation.before_phase + and phase_index not in state.initial_evaluated_phases ): - self._publish_phase( - run, state.phase_index, config.training_plan[state.phase_index] + self._evaluate( + run, + self._evaluation_context( + run, + kind="initial", + phase_index=phase_index, + phase_epoch=0, + epoch=state.epoch, + batch=0, + batches_total=1, + ), ) - for phase_index, phase in enumerate(config.training_plan): - run.metrics.set_phase(phase) - if state.plan_stopped: - completion_reason = "early_stopping" - break - if phase_index < state.phase_index: - continue - if phase_index != state.phase_index: - previous = config.training_plan[phase_index - 1] - if previous.handoff == "best": - selected = run.distributed.gather_objects( - state.best_model_state_dict - )[0] - if selected is None: - raise ValueError( - f"{previous.name}: best handoff has no finite evaluation" - ) - run.distributed.restore_model_state(run.network, selected) - if not compatible_supervised_phases( - config, phase_index - 1, phase_index - ): - state.best_validation_loss = float("inf") - state.best_model_state_dict = None - state.epochs_without_improvement = 0 - state.stopping_state = {} - state.last_validation_metric = None - last_snapshot = time.monotonic() - last_snapshot_step = state.global_batch_step - state.last_snapshot_batch_step = last_snapshot_step - phase_stopping = StoppingState(**state.stopping_state) - phase_stop = phase_stopping.triggered - if phase_stop: - continue + state.initial_evaluated_phases.append(phase_index) + for dataset in run.datasets.values(): + for part in dataset.parts.values(): + if part.factory.batch_size != phase.batch_size: + part.factory.loaders.clear() + part.factory.batch_size = phase.batch_size + sources = [ + build_source_runtime(source, run.datasets) for source in phase.sources + ] + scheduler = SourceScheduler( + phase, + sources, + seed=config.seed + phase_index, + phase_index=phase_index, + ) + if phase_index == state.phase_index and state.source_scheduler_state: + scheduler.load_state_dict(state.source_scheduler_state) + start_epoch = ( + state.phase_epoch + (1 if state.phase_epoch_complete else 0) + if phase_index == state.phase_index + else 1 + ) + for phase_epoch in range(max(1, start_epoch), phase.epochs + 1): + resuming = ( + state.phase_index == phase_index + and state.phase_epoch == phase_epoch + and not state.phase_epoch_complete + and bool(state.source_scheduler_state) + ) + current_epoch = state.epoch if resuming else state.epoch + 1 + state.phase_index = phase_index + state.phase_epoch = phase_epoch + state.phase_epoch_complete = False if run.optimization_phase_index != phase_index: - from sequifier.training.runtime import configure_phase - - configure_phase( - config, - run.network, - run.datasets, - phase_index, - run.distributed.device, + run.start_phase(phase_index) + state.optimizer_step = run.optimization.optimizer_step + state.epoch = current_epoch + state.batch = 0 + state.accumulation_index = 0 + state.iterator_positions = {} + state.source_scheduler_state = scheduler.state_dict() + access = run.optimization.access(run.network) + run.optimization.optimizer.zero_grad(set_to_none=True) + boundary = capture_runtime_boundary(run) + boundary_identity = last_identity + run.callable_network.train() + current_batches_total = sum(source.num_batches() for source in sources) + current_batch = ( + sum(state.iterator_positions.values()) if resuming else 0 + ) + for runtime_batch in scheduler.iter_epoch(phase_epoch): + batch_in_progress = True + batch_committed = False + self._check_pruning(run) + current_batch += 1 + dataset = run.datasets.resolve(runtime_batch.dataset) + if active_dataset is not None and active_dataset != dataset.name: + flush() + active_dataset = dataset.name + started = time.perf_counter() + prepared = run.loss.prepare_batch( + runtime_batch.batch, dataset, run.distributed.device ) - if ( - phase.evaluation - and phase.evaluation.before_phase - and phase_index not in state.initial_evaluated_phases - ): - self._evaluate( - run, - self._evaluation_context( - run, - kind="initial", - phase_index=phase_index, - phase_epoch=0, - epoch=state.epoch, - batch=0, - batches_total=1, - ), + dataset.metrics.training_batches += 1 + identity = self._identity( + run, current_epoch, current_batch, accumulation_count ) - state.initial_evaluated_phases.append(phase_index) - for dataset in run.datasets.values(): - for part in dataset.parts.values(): - if part.factory.batch_size != phase.batch_size: - part.factory.loaders.clear() - part.factory.batch_size = phase.batch_size - sources = [ - build_source_runtime(source, run.datasets) - for source in phase.sources - ] - scheduler = SourceScheduler( - phase, - sources, - seed=config.seed + phase_index, - phase_index=phase_index, - ) - if phase_index == state.phase_index and state.source_scheduler_state: - scheduler.load_state_dict(state.source_scheduler_state) - start_epoch = ( - state.phase_epoch + (1 if state.phase_epoch_complete else 0) - if phase_index == state.phase_index - else 1 - ) - for phase_epoch in range(max(1, start_epoch), phase.epochs + 1): - resuming = ( - state.phase_index == phase_index - and state.phase_epoch == phase_epoch - and not state.phase_epoch_complete - and bool(state.source_scheduler_state) + prepared_event = BatchPrepared( + access=access, + identity=identity, + inputs=prepared.features, + targets=prepared.targets, + metadata=prepared.metadata, ) - current_epoch = state.epoch if resuming else state.epoch + 1 - state.phase_index = phase_index - state.phase_epoch = phase_epoch - state.phase_epoch_complete = False - if run.optimization_phase_index != phase_index: - run.start_phase(phase_index) - state.optimizer_step = run.optimization.optimizer_step - state.epoch = current_epoch - state.batch = 0 - state.accumulation_index = 0 - state.iterator_positions = {} - state.source_scheduler_state = scheduler.state_dict() - access = run.optimization.access(run.network) - run.optimization.optimizer.zero_grad(set_to_none=True) - boundary = capture_runtime_boundary(run) - boundary_identity = last_identity - run.callable_network.train() - current_batches_total = sum( - source.num_batches() for source in sources + if run.integrations.enabled: + run.integrations.emit(prepared_event) + snapshot_interval_batches = ( + phase.checkpoints.snapshot_interval_batches ) - current_batch = ( - sum(state.iterator_positions.values()) if resuming else 0 + batch_snapshot_due = bool( + snapshot_interval_batches is not None + and identity.global_batch_step - last_snapshot_step + >= snapshot_interval_batches ) - for runtime_batch in scheduler.iter_epoch(phase_epoch): - batch_in_progress = True - batch_committed = False - self._check_pruning(run) - current_batch += 1 - dataset = run.datasets.resolve(runtime_batch.dataset) - if ( - active_dataset is not None - and active_dataset != dataset.name - ): - flush() - active_dataset = dataset.name - started = time.perf_counter() - prepared = run.loss.prepare_batch( - runtime_batch.batch, dataset, run.distributed.device - ) - dataset.metrics.training_batches += 1 - identity = self._identity( - run, current_epoch, current_batch, accumulation_count - ) - prepared_event = BatchPrepared( - access=access, - identity=identity, - inputs=prepared.features, - targets=prepared.targets, - metadata=prepared.metadata, - ) - if run.integrations.enabled: - run.integrations.emit(prepared_event) - snapshot_interval_batches = ( - phase.checkpoints.snapshot_interval_batches - ) - batch_snapshot_due = bool( - snapshot_interval_batches is not None - and identity.global_batch_step - last_snapshot_step - >= snapshot_interval_batches - ) - gradient_sync_due = bool( - accumulation_count + 1 - >= run.optimization.gradient_policy.accumulation_batches - or current_batch >= current_batches_total - or batch_snapshot_due - ) - gradient_sync_context = ( - run.callable_network.no_sync() - if defer_ddp_gradient_sync and not gradient_sync_due - else nullcontext() - ) - with gradient_sync_context: - with run.precision.autocast(): - trace = run.integrations.forward_trace(prepared_event) - output = run.callable_network( - prepared.features, - prepared.metadata, - teacher_targets=prepared.loss_targets, - teacher_valid_mask=prepared.loss_valid_mask, - interface_name=dataset.interface_name, - trace=trace, - ) - if run.integrations.enabled: - run.integrations.emit( - ForwardCompleted( - access=access, - identity=identity, - outputs=output.logits, - captures={} - if trace is None - else dict(trace.captures), - ) + gradient_sync_due = bool( + accumulation_count + 1 + >= run.optimization.gradient_policy.accumulation_batches + or current_batch >= current_batches_total + or batch_snapshot_due + ) + gradient_sync_context = ( + run.callable_network.no_sync() + if defer_ddp_gradient_sync and not gradient_sync_due + else nullcontext() + ) + with gradient_sync_context: + with run.precision.autocast(): + trace = run.integrations.forward_trace(prepared_event) + output = run.callable_network( + prepared.features, + prepared.metadata, + teacher_targets=prepared.loss_targets, + teacher_valid_mask=prepared.loss_valid_mask, + interface_name=dataset.interface_name, + trace=trace, + ) + if run.integrations.enabled: + run.integrations.emit( + ForwardCompleted( + access=access, + identity=identity, + outputs=output.logits, + captures={} + if trace is None + else dict(trace.captures), ) - loss = run.loss.calculate( - output, prepared, dataset, run.network ) - if run.integrations.enabled: - run.integrations.emit( - LossComputed( - access=access, - identity=identity, - loss=loss.backward_loss.detach(), - backward_loss=loss.backward_loss, - ) - ) - run.optimization.accumulate( - loss.backward_loss, - identity, - run.integrations, - run.network, + loss = run.loss.calculate( + output, prepared, dataset, run.network ) - accumulation_count += 1 - last_identity = identity - state.epoch = current_epoch - state.batch = current_batch - state.global_batch_step = identity.global_batch_step - state.accumulation_index = accumulation_count - 1 - state.source_scheduler_state = scheduler.state_dict() - position_key = f"{runtime_batch.dataset}.{runtime_batch.part}" - state.iterator_positions[position_key] = ( - state.iterator_positions.get(position_key, 0) + 1 + if run.integrations.enabled: + run.integrations.emit( + LossComputed( + access=access, + identity=identity, + loss=loss.backward_loss.detach(), + backward_loss=loss.backward_loss, + ) + ) + run.optimization.accumulate( + loss.backward_loss, + identity, + run.integrations, + run.network, ) - dataset.parts[runtime_batch.part].iterator_positions[ - "training" - ] = state.iterator_positions[position_key] - batch_committed = True - batch_in_progress = False - window = metric_windows.setdefault( - dataset.name, - { - "sums": { - name: value.new_zeros(()) - for name, value in loss.accounting_sums.items() - }, - "count": loss.accounting_count.new_zeros(()).to( - next(iter(loss.accounting_sums.values())).dtype - ), - "batches": 0, - "seconds": 0.0, - "part": runtime_batch.part, + accumulation_count += 1 + last_identity = identity + state.epoch = current_epoch + state.batch = current_batch + state.global_batch_step = identity.global_batch_step + state.accumulation_index = accumulation_count - 1 + state.source_scheduler_state = scheduler.state_dict() + position_key = f"{runtime_batch.dataset}.{runtime_batch.part}" + state.iterator_positions[position_key] = ( + state.iterator_positions.get(position_key, 0) + 1 + ) + dataset.parts[runtime_batch.part].iterator_positions["training"] = ( + state.iterator_positions[position_key] + ) + batch_committed = True + batch_in_progress = False + window = metric_windows.setdefault( + dataset.name, + { + "sums": { + name: value.new_zeros(()) + for name, value in loss.accounting_sums.items() }, - ) - for name, value in loss.accounting_sums.items(): - window["sums"][name] += value - window["count"] += loss.accounting_count.to( - window["count"].dtype - ) - window["batches"] += 1 - window["seconds"] += time.perf_counter() - started - if window["part"] != runtime_batch.part: - window["part"] = None - if ( - window["batches"] - >= config.runtime_settings.log_interval_batches - ): - flush_metrics( - dataset.name, - current_epoch, - current_batch, - current_batches_total, - ) - if ( - accumulation_count - >= run.optimization.gradient_policy.accumulation_batches - ): - flush() - now = time.monotonic() - snapshot_due = bool( - phase.checkpoints.snapshot_interval_batches is not None - and state.global_batch_step - last_snapshot_step - >= phase.checkpoints.snapshot_interval_batches - ) or bool( - phase.checkpoints.snapshot_interval_minutes is not None - and now - last_snapshot - >= phase.checkpoints.snapshot_interval_minutes * 60 - ) - latest_due = bool( - config.runtime_settings.latest_checkpoint_interval_minutes - is not None - and now - last_latest - >= config.runtime_settings.latest_checkpoint_interval_minutes - * 60 - ) - if ( - run.distributed.world_size > 1 - and time_checkpoint_intervals_enabled - ): - checkpoint_due = torch.tensor( - [ - int(latest_due) if run.distributed.rank == 0 else 0, - int(snapshot_due) - if run.distributed.rank == 0 - else 0, - ], - device=run.distributed.device, - dtype=torch.int32, - ) - torch.distributed.broadcast(checkpoint_due, src=0) - latest_due = bool(checkpoint_due[0].item()) - snapshot_due = bool(checkpoint_due[1].item()) - if latest_due or snapshot_due: - flush() - if latest_due: - run.checkpoints.save( - CheckpointRequest("latest", identity), run - ) - last_latest = time.monotonic() - if snapshot_due: - if ( - phase.evaluation_sources - and phase.evaluation is not None - and phase.evaluation.on_snapshot - ): - self._evaluate( - run, - self._evaluation_context( - run, - kind="interval", - phase_index=phase_index, - phase_epoch=phase_epoch, - epoch=current_epoch, - batch=current_batch, - batches_total=current_batches_total, - ), - ) - run.callable_network.train() - state.last_snapshot_batch_step = state.global_batch_step - run.checkpoints.save( - CheckpointRequest( - f"epoch-{current_epoch}-batch-{current_batch}", - identity, - ), - run, - ) - last_snapshot = time.monotonic() - last_snapshot_step = state.global_batch_step - if stop_requested: - completion_reason = "integration_requested_stop" - break - flush() - for dataset_name in tuple(metric_windows): + "count": loss.accounting_count.new_zeros(()).to( + next(iter(loss.accounting_sums.values())).dtype + ), + "batches": 0, + "seconds": 0.0, + "part": runtime_batch.part, + }, + ) + for name, value in loss.accounting_sums.items(): + window["sums"][name] += value + window["count"] += loss.accounting_count.to(window["count"].dtype) + window["batches"] += 1 + window["seconds"] += time.perf_counter() - started + if window["part"] != runtime_batch.part: + window["part"] = None + if ( + window["batches"] + >= config.runtime_settings.log_interval_batches + ): flush_metrics( - dataset_name, + dataset.name, current_epoch, current_batch, current_batches_total, ) - state.phase_epoch_complete = True - evaluation = ( - self._evaluate( - run, - self._evaluation_context( - run, - kind="epoch_end", - phase_index=phase_index, - phase_epoch=phase_epoch, - epoch=current_epoch, - batch=current_batch, - batches_total=current_batches_total, - ), - ) - if phase.evaluation_sources - and ( - phase_epoch % phase.evaluation_interval_epochs == 0 - or phase_epoch == phase.epochs - ) - else EvaluationResult() + if ( + accumulation_count + >= run.optimization.gradient_policy.accumulation_batches + ): + flush() + now = time.monotonic() + snapshot_due = bool( + phase.checkpoints.snapshot_interval_batches is not None + and state.global_batch_step - last_snapshot_step + >= phase.checkpoints.snapshot_interval_batches + ) or bool( + phase.checkpoints.snapshot_interval_minutes is not None + and now - last_snapshot + >= phase.checkpoints.snapshot_interval_minutes * 60 ) - if run.optimization.scheduler_policy.step_on == "epoch": - monitor = phase.evaluation_monitor - metric = ( - evaluation.sources[monitor.source].total_loss - if monitor is not None - and monitor.source in evaluation.sources - else None - ) - run.optimization.step_scheduler(metric) - state.iterator_positions = {} - state.source_scheduler_state = scheduler.state_dict() - monitored, _ = self._update_monitor(run, evaluation, phase) - phase_stop = phase_stopping.observe( - phase.stopping, - monitored, - phase.evaluation_monitor.direction - if phase.evaluation_monitor - else "min", - phase_epoch, + latest_due = bool( + config.runtime_settings.latest_checkpoint_interval_minutes + is not None + and now - last_latest + >= config.runtime_settings.latest_checkpoint_interval_minutes + * 60 ) - state.stopping_state = asdict(phase_stopping) - if phase_stop: - state.plan_stopped = phase.stopping.on_trigger == "end_plan" - stop_requested = state.plan_stopped - completion_reason = "early_stopping" - boundary = capture_runtime_boundary(run) - boundary_identity = last_identity if ( - phase_stop - or phase_epoch % phase.checkpoint_interval_epochs == 0 - or phase_epoch == phase.epochs + run.distributed.world_size > 1 + and time_checkpoint_intervals_enabled ): + checkpoint_due = torch.tensor( + [ + int(latest_due) if run.distributed.rank == 0 else 0, + int(snapshot_due) if run.distributed.rank == 0 else 0, + ], + device=run.distributed.device, + dtype=torch.int32, + ) + torch.distributed.broadcast(checkpoint_due, src=0) + latest_due = bool(checkpoint_due[0].item()) + snapshot_due = bool(checkpoint_due[1].item()) + if latest_due or snapshot_due: + flush() + if latest_due: + run.checkpoints.save(CheckpointRequest("latest", identity), run) + last_latest = time.monotonic() + if snapshot_due: + if ( + phase.evaluation_sources + and phase.evaluation is not None + and phase.evaluation.on_snapshot + ): + self._evaluate( + run, + self._evaluation_context( + run, + kind="interval", + phase_index=phase_index, + phase_epoch=phase_epoch, + epoch=current_epoch, + batch=current_batch, + batches_total=current_batches_total, + ), + ) + run.callable_network.train() + state.last_snapshot_batch_step = state.global_batch_step run.checkpoints.save( - CheckpointRequest(f"epoch-{current_epoch}", last_identity), + CheckpointRequest( + f"epoch-{current_epoch}-batch-{current_batch}", + identity, + ), run, ) - if stop_requested or phase_stop: + last_snapshot = time.monotonic() + last_snapshot_step = state.global_batch_step + if stop_requested: + completion_reason = "integration_requested_stop" break - self._publish_phase(run, phase_index, phase) - run.checkpoints.save(CheckpointRequest("latest", last_identity), run) - if stop_requested: + flush() + for dataset_name in tuple(metric_windows): + flush_metrics( + dataset_name, + current_epoch, + current_batch, + current_batches_total, + ) + state.phase_epoch_complete = True + evaluation = ( + self._evaluate( + run, + self._evaluation_context( + run, + kind="epoch_end", + phase_index=phase_index, + phase_epoch=phase_epoch, + epoch=current_epoch, + batch=current_batch, + batches_total=current_batches_total, + ), + ) + if phase.evaluation_sources + and ( + phase_epoch % phase.evaluation_interval_epochs == 0 + or phase_epoch == phase.epochs + ) + else EvaluationResult() + ) + if run.optimization.scheduler_policy.step_on == "epoch": + monitor = phase.evaluation_monitor + metric = ( + evaluation.sources[monitor.source].total_loss + if monitor is not None and monitor.source in evaluation.sources + else None + ) + run.optimization.step_scheduler(metric) + state.iterator_positions = {} + state.source_scheduler_state = scheduler.state_dict() + monitored, _ = self._update_monitor(run, evaluation, phase) + phase_stop = phase_stopping.observe( + phase.stopping, + monitored, + phase.evaluation_monitor.direction + if phase.evaluation_monitor + else "min", + phase_epoch, + ) + state.stopping_state = asdict(phase_stopping) + if phase_stop: + state.plan_stopped = ( + stop_scope(phase.stopping, phase_stop) == "end_plan" + ) + stop_requested = state.plan_stopped + completion_reason = "early_stopping" + boundary = capture_runtime_boundary(run) + boundary_identity = last_identity + if ( + phase_stop + or phase_epoch % phase.checkpoint_interval_epochs == 0 + or phase_epoch == phase.epochs + ): + run.checkpoints.save( + CheckpointRequest(f"epoch-{current_epoch}", last_identity), + run, + ) + if stop_requested or phase_stop: break + return PhaseOutcome( + None, + stop_scope(phase.stopping, phase_stopping.triggered), + completion_reason if stop_requested else None, + ) + + def commit_phase(index, phase, outcome): + self._publish_phase(run, index, phase) + run.checkpoints.save(CheckpointRequest("latest", last_identity), run) + + try: + start = state.phase_index + phase = config.training_plan[start] + if state.phase_epoch_complete and ( + state.plan_stopped + or state.phase_epoch >= phase.epochs + or state.stopping_state.get("triggered") + ): + # A native checkpoint can precede publication. Rebuild the + # derived artifact on every rank before moving forward. + self._publish_phase(run, start, phase) + start += 1 + if phase_only and start == 0: + execute_phase(0, phase, run_phase, commit_phase) + elif not phase_only: + plan_reason = run_phases( + config.training_plan, + run_phase, + commit_phase, + start=start, + stopped=state.plan_stopped, + completion_reason="normal_completion", + ) + if plan_reason != "normal_completion": + completion_reason = plan_reason except BaseException: if update_failed: run.optimization.optimizer.zero_grad(set_to_none=True) @@ -842,6 +853,14 @@ def flush_metrics( raise run.checkpoints.save(CheckpointRequest("latest", last_identity), run) + if phase_only: + return RunResult( + "early_stopping" + if state.stopping_state.get("triggered") + else completion_reason, + (), + {}, + ) run.distributed.barrier() last_state = run.distributed.capture_model_state(run.network) best_state = state.best_model_state_dict or last_state @@ -864,10 +883,8 @@ def flush_metrics( publication, ) if run.distributed.rank == 0: - selected = ( - best_state - if config.training_plan[state.phase_index].handoff == "best" - else last_state + selected = select_handoff( + config.training_plan[state.phase_index].handoff, last_state, best_state ) save_model_artifact( build_model_artifact(run.network, config, state_dict=selected), diff --git a/src/sequifier/training/phase_runners.py b/src/sequifier/training/phase_runners.py new file mode 100644 index 00000000..af6ed01e --- /dev/null +++ b/src/sequifier/training/phase_runners.py @@ -0,0 +1,201 @@ +"""Adapt resolved phases to native execution and exact recovery.""" + +from dataclasses import dataclass + +import torch + +from sequifier.artifacts.model_artifact import ModelArtifact, read_execution_artifact +from sequifier.artifacts.phase import PhaseArtifacts +from sequifier.runtime.plan import RunIdentity + + +@dataclass(frozen=True) +class PortablePhaseRequest: + index: int + incoming: str | None + artifacts: PhaseArtifacts + resuming: bool + legacy_recovery: bool + identity: RunIdentity + + +@dataclass(frozen=True) +class PortablePhaseResult: + artifact: ModelArtifact + report: dict + + +def supervised_phase_config(config, phase, index, incoming, resume): + from sequifier.artifacts.model_config import ( + extend_prediction_distributions, + model_execution_from_model_config, + validate_execution_attachment, + ) + from sequifier.artifacts.model_export import model_execution_config + from sequifier.config.components import ResumeConfig + from sequifier.model.execution import ModelExecution + + training = config.runtime_settings.model_copy(deep=True) + training.bert_spec = phase.learning.bert_spec + training.next_occurrence_config = phase.learning.next_occurrence_config + training.window_stride = phase.learning.window_stride + if training.distributed: + raise ValueError("mixed learning plans currently require local phase execution") + native = PhaseArtifacts.for_phase( + config.phase_directory, index, phase.name + ).model.parent + training.resume = ResumeConfig( + policy="if_exists" if resume else "never", + checkpoint_path=str(native / "run.pt"), + ) + training.log_interval_iterations = 10 + names = {ref.dataset for ref in [*phase.sources, *phase.evaluation_sources]} + prepared = config._supervised_datasets or config.dataset_training + datasets = { + name: dataset.model_copy(update=phase.learning.datasets[name].model_dump()) + for name, dataset in prepared.items() + if name in names + } + if datasets.keys() != names or any( + not dataset.parts for dataset in datasets.values() + ): + raise ValueError(f"{phase.name}: supervised phase requires resolved datasets") + + model = config.model + source = ModelExecution.from_config(config) + artifact = read_execution_artifact(incoming) if incoming is not None else None + if artifact is not None: + source, _ = model_execution_from_model_config( + artifact.model_config.values, device=config.device + ) + extend_prediction_distributions(source, config.model) + model = source.model + training.forward_attention = source.runtime_settings.forward_attention or ( + "bidirectional" + if source.runtime_settings.training_objective == "bert" + else "causal" + ) + datasets = { + name: dataset.model_copy( + update={ + "interface": dataset.interface.model_copy( + update={ + "policy": source.interface(dataset.model_interface).policy, + "prediction_distributions": dict( + source.interface( + dataset.model_interface + ).prediction_distributions + ), + } + ) + } + ) + for name, dataset in datasets.items() + } + + resolved = config.model_copy( + update={ + "model": model, + "dataset_training": datasets, + "training_plan": [phase.model_copy(deep=True)], + "active_phase_index": 0, + "runtime_settings": training, + "seed": config.seed + index, + "model_name": f"{config.model_name}-phase-{index:03d}-{phase.name}", + "initialize_from": None, + "actor_initialization": "attach", + "curriculum_training": bool(phase.curriculum), + "curriculum_column": phase.curriculum.column if phase.curriculum else None, + "export_pt": True, + "model_path": str(native / "native/model.pt"), + "report_path": str(native / "native/report.json"), + "phase_directory": str(native / "native/phases"), + } + ) + source.retain_interfaces(resolved) + if artifact is not None: + validate_execution_attachment( + artifact.model_config.values, model_execution_config(resolved) + ) + return resolved + + +def run_supervised_phase( + config, phase, request: PortablePhaseRequest +) -> PortablePhaseResult: + from sequifier.helpers import configure_determinism, configure_logger + from sequifier.integration import IntegrationManager + from sequifier.runtime.builder import RunBuilder + from sequifier.runtime.context import ExecutionEnvironment + from sequifier.training.engine import TrainingEngine + + resolved = supervised_phase_config( + config, phase, request.index, request.incoming, request.resuming + ) + torch.set_float32_matmul_precision( + resolved.runtime_settings.float32_matmul_precision + ) + configure_logger(config.project_root, resolved.model_name, 0) + configure_determinism(resolved.seed, resolved.runtime_settings.enforce_determinism) + environment = ExecutionEnvironment( + rank=0, + local_rank=0, + world_size=1, + device=torch.device(config.device), + distributed=False, + ) + integrations = IntegrationManager(rank=0, world_size=1, distributed=False) + run = RunBuilder().build( + resolved, + environment, + integrations, + initial_artifact=request.incoming, + identity=request.identity, + legacy_recovery=request.legacy_recovery, + ) + try: + report = TrainingEngine().run_phase(run) + return PortablePhaseResult( + read_execution_artifact(report["model_path"]), + { + **report, + "resolved_seed": resolved.seed, + "optimizer_steps": run.state.optimizer_step, + }, + ) + finally: + run.distributed.finalize() + + +def policy_phase_config(config, phase, index, incoming, destination, resume): + from sequifier.config.training_resolution import policy_run_config + + return policy_run_config( + config, + [phase], + incoming=incoming, + index=index, + destination=destination, + resuming=resume, + ) + + +def run_portable_policy_phase( + config, phase, request: PortablePhaseRequest +) -> PortablePhaseResult: + from sequifier.rl.learner import run_policy_phase + + run = policy_phase_config( + config, + phase, + request.index, + request.incoming, + request.artifacts.model, + request.resuming, + ) + result = run_policy_phase( + run, request.identity, legacy_recovery=request.legacy_recovery + ).phases[0] + return PortablePhaseResult( + read_execution_artifact(result.provenance["actor_path"]), result.as_dict() + ) diff --git a/src/sequifier/training/state.py b/src/sequifier/training/state.py index e5ee103e..bed0980b 100644 --- a/src/sequifier/training/state.py +++ b/src/sequifier/training/state.py @@ -2,13 +2,14 @@ from __future__ import annotations -import uuid from copy import deepcopy from dataclasses import asdict, dataclass, field from typing import Any, Mapping from torch import Tensor +from sequifier.runtime.plan import new_run_id + @dataclass(frozen=True) class RunStateSnapshot: @@ -17,8 +18,8 @@ class RunStateSnapshot: @dataclass class RunState: - run_id: str = field(default_factory=lambda: uuid.uuid4().hex) - session_id: str = field(default_factory=lambda: uuid.uuid4().hex) + run_id: str = field(default_factory=new_run_id) + session_id: str = field(default_factory=new_run_id) stopping_state: dict[str, Any] = field(default_factory=dict) phase_reports: list[dict[str, Any]] = field(default_factory=list) last_validation_metric: float | None = None From 02aa9ccef0896cd0d1f664d7a02302ebd8e5e91c Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 11:27:13 +0100 Subject: [PATCH 50/71] clean up model execution --- src/sequifier/api.py | 2 + src/sequifier/artifacts/loading.py | 5 +- src/sequifier/artifacts/model_artifact.py | 19 +- src/sequifier/artifacts/model_config.py | 31 ++- src/sequifier/artifacts/model_export.py | 31 +-- src/sequifier/artifacts/run_checkpoint.py | 21 +- src/sequifier/config/infer_config.py | 44 ++-- src/sequifier/config/inference_common.py | 4 +- .../config/input_contract_resolution.py | 39 ++-- src/sequifier/config/policy_infer_config.py | 2 +- .../config/sequence_import_config.py | 4 +- src/sequifier/config/train_config.py | 197 ++++++++++++------ src/sequifier/config/training_resolution.py | 46 ++-- src/sequifier/evaluation/compatibility.py | 5 +- src/sequifier/evaluation/service.py | 12 +- src/sequifier/export/preflight.py | 41 ++-- src/sequifier/export/service.py | 90 ++++---- src/sequifier/io/output_paths.py | 8 +- src/sequifier/model/execution.py | 124 ++++------- src/sequifier/model/factory.py | 22 +- src/sequifier/rl/checkpoint.py | 30 +-- src/sequifier/rl/policy.py | 17 +- src/sequifier/runtime/builder.py | 12 +- src/sequifier/train.py | 18 +- src/sequifier/training/coordinator.py | 29 ++- src/sequifier/training/loss.py | 12 +- src/sequifier/training/phase_runners.py | 35 +--- src/sequifier/training/runtime.py | 26 ++- 28 files changed, 491 insertions(+), 435 deletions(-) diff --git a/src/sequifier/api.py b/src/sequifier/api.py index 95792331..9443c1b8 100644 --- a/src/sequifier/api.py +++ b/src/sequifier/api.py @@ -12,6 +12,7 @@ canonical_parameter_name, canonicalize_state_dict, ) +from sequifier.model.execution import ModelExecution from sequifier.model.factory import build_transformer_network from sequifier.model.network import ( ComposableTransformerNetwork, @@ -40,6 +41,7 @@ "InterventionBinding", "ModelArtifact", "ModelArtifactMetadata", + "ModelExecution", "ModelExecutionConfig", "ModelInterfaceModule", "ModelOutput", diff --git a/src/sequifier/artifacts/loading.py b/src/sequifier/artifacts/loading.py index 70e65b9a..053b84a7 100644 --- a/src/sequifier/artifacts/loading.py +++ b/src/sequifier/artifacts/loading.py @@ -10,6 +10,7 @@ from torch import nn from sequifier.artifacts.model_artifact import read_artifact, restore_model_artifact +from sequifier.artifacts.model_config import resolved_training_config from sequifier.model.parameter_catalog import ParameterCatalog @@ -43,7 +44,7 @@ def load_model_for_analysis( raise ValueError("Current Sequifier artifacts embed their model configuration.") artifact_path = Path(path).expanduser().resolve() artifact, artifact_type = read_artifact(artifact_path) - network, resolved_config, artifact = restore_model_artifact( + network, execution, artifact = restore_model_artifact( artifact, device=options.device, interface_name=interface_name, @@ -61,7 +62,7 @@ def load_model_for_analysis( network = torch.compile(network) return LoadedModel( network=network, - config=resolved_config, + config=resolved_training_config(execution), parameter_catalog=catalog, artifact_metadata={ "path": str(artifact_path), diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index b2f422bd..7b3b45bb 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -9,13 +9,17 @@ import torch from torch import Tensor -from sequifier.artifacts.model_config import resolved_config_from_model_config +from sequifier.artifacts.model_config import ( + model_execution_from_model_config, + resolved_training_config, +) from sequifier.artifacts.model_export import model_execution_config from sequifier.artifacts.state_dict import ( canonicalize_state_dict, validate_model_state_contract, ) from sequifier.io.atomic import atomic_save +from sequifier.model.execution import ModelExecution from sequifier.model.factory import build_transformer_network MODEL_ARTIFACT_FORMAT_VERSION = 3 @@ -131,9 +135,10 @@ def load_model_artifact( """Load a portable artifact and return network, resolved config, and schema.""" artifact, _ = read_artifact(path, expected_type="sequifier_model") - return restore_model_artifact( + network, execution, artifact = restore_model_artifact( artifact, device=device, interface_name=interface_name ) + return network, resolved_training_config(execution), artifact def restore_model_artifact( @@ -141,9 +146,9 @@ def restore_model_artifact( *, device: str = "cpu", interface_name: str | None = None, -) -> tuple[Any, Any, ModelArtifact]: +) -> tuple[Any, ModelExecution, ModelArtifact]: """Reconstruct execution identically for portable models and run checkpoints.""" - config, _ = resolved_config_from_model_config( + config, _ = model_execution_from_model_config( artifact.model_config.values, device=device, interface_name=interface_name, @@ -193,7 +198,8 @@ def load_execution_model( *, device: str = "cpu", interface_name: str | None = None, -) -> tuple[Any, Any, ModelArtifact]: +) -> tuple[Any, ModelExecution, ModelArtifact]: + """Load a network and its model-owned execution contract.""" return restore_model_artifact( path if isinstance(path, ModelArtifact) else read_execution_artifact(path), device=device, @@ -211,8 +217,9 @@ def load_weights_from_run_checkpoint( device: str = "cpu", interface_name: str | None = None, ) -> tuple[Any, Any, ModelArtifact]: - return restore_model_artifact( + network, execution, artifact = restore_model_artifact( model_artifact_from_run_checkpoint(path), device=device, interface_name=interface_name, ) + return network, resolved_training_config(execution), artifact diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index dfda2991..fd414b33 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -95,7 +95,7 @@ def resolved_config_from_model_config( execution, selected = model_execution_from_model_config( values, device=device, interface_name=interface_name, validate=False ) - return execution.training_config(), selected + return resolved_training_config(execution), selected @beartype @@ -225,13 +225,34 @@ def model_execution_from_model_config( def execution_config(model, interfaces, runtime_settings, *, device="cpu", **options): """Adapt model execution to the existing public resolved training contract.""" - from sequifier.model.execution import ModelExecution - - return ModelExecution(model, interfaces, runtime_settings).training_config( - device=device, **options + return resolved_training_config( + ModelExecution(model, interfaces, runtime_settings, device=device), **options ) +def resolved_training_config( + execution: ModelExecution, **options +) -> ResolvedSequifierConfig: + """Project a model onto the resolved training schema for compatibility callers.""" + values = { + "project_root": execution.project_root, + "model_name": execution.model_name, + "seed": execution.seed, + "device": execution.device, + "runtime_settings": execution.runtime_settings, + "model": execution.model, + "interfaces": execution.interfaces, + "dataset_training": {}, + "training_plan": [], + "export_generative_model": True, + "export_embedding_model": False, + "export_onnx": False, + "export_pt": False, + "embedding_layer_names": list(execution.embedding_layer_names), + } + return ResolvedSequifierConfig(**{**values, **options}) + + def extend_prediction_distributions(execution, declared_model): """Allow explicit new scale parameters while preserving an initialized model.""" exclude = {"interfaces": {"__all__": {"prediction_distributions"}}} diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index c902736c..dcaad2c4 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -1,24 +1,18 @@ from typing import Any +from sequifier.model.execution import ModelExecution +from sequifier.model.execution_schema import ExecutionSchema from sequifier.typechecking import beartype @beartype -def model_execution_config(training_config: Any) -> dict[str, Any]: +def model_execution_config(config: Any) -> dict[str, Any]: """Return only configuration required to reconstruct model execution.""" - from sequifier.model.execution import ModelExecution - from sequifier.model.execution_schema import ExecutionSchema - - if not isinstance(training_config, ModelExecution) and not hasattr( - training_config, "dataset_training" - ): - raise TypeError( - "Model export requires a canonical training config or model execution" - ) - execution = ModelExecution.from_config(training_config) + execution = ModelExecution.from_config(config) + spec = execution.runtime_settings interfaces = {} - for interface in execution.interfaces.values(): + for name, interface in execution.interfaces.items(): values = interface.model_dump( mode="python", exclude={ @@ -30,26 +24,23 @@ def model_execution_config(training_config: Any) -> dict[str, Any]: }, ) values["execution_schema"] = ExecutionSchema.from_interface( - interface, training_config.runtime_settings.context_length + interface, spec.context_length ).to_dict() values["depth_layouts"] = interface.depth_layouts.model_dump(mode="json") - interfaces[interface.name] = values + interfaces[name] = values if interface.policy is not None: - interfaces[interface.name]["policy"] = interface.policy.model_dump( - mode="json" - ) - spec = training_config.runtime_settings + values["policy"] = interface.policy.model_dump(mode="json") return { "training_objective": spec.training_objective, "forward_attention": spec.forward_attention or ("bidirectional" if spec.training_objective == "bert" else "causal"), "context_length": spec.context_length, "target_offset": spec.target_offset, - "backbone": training_config.model.backbone.model_dump( + "backbone": execution.model.backbone.model_dump( mode="python", exclude={"repository"}, ), - "embedding_layer_names": list(training_config.embedding_layer_names), + "embedding_layer_names": list(execution.embedding_layer_names), "layer_type_dtypes": spec.layer_type_dtypes, "interfaces": interfaces, } diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index b4403fa7..fd67e5fe 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -98,6 +98,15 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": or payload.get("initialization_seed_derivation_version") != 1 ): raise ValueError("Unsupported initialization provenance version") + saved_config = payload["training_config"] + training_config = ResolvedSequifierConfig.model_validate(saved_config) + removed_datasets = ( + saved_config["dataset_training"].keys() + - training_config.dataset_training.keys() + if "interfaces" not in saved_config + else set() + ) + loader_parts = payload["loader_state"]["parts"] checkpoint = cls( format_version=int(payload["format_version"]), model=ModelArtifact.from_state_dict(payload["model"]), @@ -106,13 +115,17 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": random_state=DistributedRandomState( states=tuple(payload["random_state"]["states"]) ), - loader_state=LoaderState(parts=dict(payload["loader_state"]["parts"])), + loader_state=LoaderState( + parts={ + name: parts + for name, parts in loader_parts.items() + if parts or name not in removed_datasets + } + ), integration_state=IntegrationState( values=dict(payload["integration_state"]["values"]) ), - training_config=ResolvedSequifierConfig.model_validate( - payload["training_config"] - ), + training_config=training_config, ) checkpoint.validate() return checkpoint diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 29191773..99c87b79 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -132,14 +132,13 @@ def _merge_loaded_execution_values( @beartype def _execution_source( - training: Any, dataset: Any + config: Any, interface: Any ) -> tuple[dict[str, Any], DatasetMetadata]: - interface = dataset.interface - global_spec = training.runtime_settings + global_spec = config.runtime_settings layout = interface.storage_layout return ( { - "model_interface": dataset.model_interface, + "model_interface": interface.name, "input_columns": list(interface.input_columns), "static_columns": ( list(interface.static_conditioning.columns) @@ -449,38 +448,35 @@ def load_inferer_config( if dataset_name is not None: if dataset_name not in datasets: raise ValueError(f"Unknown inference dataset {dataset_name!r}") - dataset = datasets[dataset_name] + interface = training.dataset_interface(dataset_name) + source = f"training config dataset {dataset_name!r}" elif authored_values.get("model_interface") is not None: interface_name = authored_values["model_interface"] - dataset = next( - ( - candidate - for candidate in datasets.values() - if candidate.model_interface == interface_name - ), - None, - ) - if dataset is None: + if interface_name not in training.interfaces: raise ValueError(f"Unknown model interface {interface_name!r}") + interface = training.interface(interface_name) + source = f"training config interface {interface_name!r}" elif len(datasets) == 1: - dataset = next(iter(datasets.values())) + name = next(iter(datasets)) + interface = training.dataset_interface(name) + source = f"training config dataset {name!r}" else: raise ValueError( "A dataset or model_interface selection is required for the " "training config" ) - loaded_values, training_metadata = _execution_source(training, dataset) + loaded_values, training_metadata = _execution_source(training, interface) image_selections.append( ( - image_views(dataset.interface.ingestion), - dataset.interface.preconditioning_image_encoder is not None, + image_views(interface.ingestion), + interface.preconditioning_image_encoder is not None, ) ) - source = f"training config dataset {dataset.name!r}" _merge_loaded_execution_values(authored_values, loaded_values, source) loaded_metadata_sources.append((training_metadata, source, True)) if dataset_name is not None: + dataset = datasets[dataset_name] part_name = authored_values.get("part") if part_name is None: if len(dataset.parts) != 1: @@ -519,8 +515,8 @@ def load_inferer_config( ) training = selection.resolve(str(authored_values.get("device", "cpu"))) interface_name = selection.interface_name - dataset = training.dataset_training[interface_name] - decoder = dataset.interface.decoder + interface = training.interface(interface_name) + decoder = interface.decoder branches = ( decoder.branches.values() if decoder.type == "composite" else (decoder,) ) @@ -530,11 +526,11 @@ def load_inferer_config( if branch.type == "autoregressive_transformer" for target in branch.target_columns ) - loaded_values, artifact_metadata = _execution_source(training, dataset) + loaded_values, artifact_metadata = _execution_source(training, interface) image_selections.append( ( - image_views(dataset.interface.ingestion), - dataset.interface.preconditioning_image_encoder is not None, + image_views(interface.ingestion), + interface.preconditioning_image_encoder is not None, ) ) if artifact_metadata_source is None: diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index 010ce084..1e2ee3b6 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -108,9 +108,9 @@ def policy_bound(self): return self.values["interfaces"][self.interface_name].get("policy") is not None def resolve(self, device): - from sequifier.artifacts.model_config import resolved_config_from_model_config + from sequifier.artifacts.model_config import model_execution_from_model_config - return resolved_config_from_model_config( + return model_execution_from_model_config( self.values, device=device, interface_name=self.interface_name )[0] diff --git a/src/sequifier/config/input_contract_resolution.py b/src/sequifier/config/input_contract_resolution.py index b62eb086..2a76ef48 100644 --- a/src/sequifier/config/input_contract_resolution.py +++ b/src/sequifier/config/input_contract_resolution.py @@ -16,9 +16,7 @@ def validate_input_contracts(config, resolved): for value in resolved.dataset_training.values() if value.model_interface == name ] - if not datasets: - continue - if declaration.metadata_config_path: + if datasets and declaration.metadata_config_path: expected = load_dataset_metadata( str(Path(config.run.project_root) / declaration.metadata_config_path) ) @@ -33,21 +31,20 @@ def validate_input_contracts(config, resolved): if normalization is None: continue resolved.normalization_by_interface[name] = normalization - for dataset in datasets: - interface = dataset.interface - expected = normalization.statistics - real_inputs = interface.real_columns + interface.static_real_columns - if normalization.source == "identity": - expected = {column: {"mean": 0.0, "std": 1.0} for column in real_inputs} - for column, values in expected.items(): - if column not in real_inputs: - raise ValueError( - f"{name}: normalization references unknown/non-real input {column!r}" - ) - actual = interface.selected_columns_statistics.get(column) - if not interface.normalize_real_columns: - actual = {"mean": 0.0, "std": 1.0} - if actual != values: - raise ValueError( - f"model.interfaces.{name}.input_contract.normalization.{column} differs from frozen metadata/artifact" - ) + interface = resolved.interface(name) + expected = normalization.statistics + real_inputs = interface.real_columns + interface.static_real_columns + if normalization.source == "identity": + expected = {column: {"mean": 0.0, "std": 1.0} for column in real_inputs} + for column, values in expected.items(): + if column not in real_inputs: + raise ValueError( + f"{name}: normalization references unknown/non-real input {column!r}" + ) + actual = interface.selected_columns_statistics.get(column) + if not interface.normalize_real_columns: + actual = {"mean": 0.0, "std": 1.0} + if actual != values: + raise ValueError( + f"model.interfaces.{name}.input_contract.normalization.{column} differs from frozen metadata/artifact" + ) diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 6649d104..74213fe4 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -143,7 +143,7 @@ def policy_inference_config( if field.type == "categorical" ): raise ValueError("action labels must have unique probability column names") - contract, metadata = _execution_source(execution, execution.dataset(selected)) + contract, metadata = _execution_source(execution, execution.interface(selected)) for key, actual in contract.items(): if ( key in request.model_fields_set diff --git a/src/sequifier/config/sequence_import_config.py b/src/sequifier/config/sequence_import_config.py index 60a55197..ff5f0778 100644 --- a/src/sequifier/config/sequence_import_config.py +++ b/src/sequifier/config/sequence_import_config.py @@ -45,11 +45,11 @@ def resolved_policy(self) -> ResolvedPolicyBinding: if self.initialize_from: from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.artifacts.model_config import ( - resolved_config_from_model_config, + model_execution_from_model_config, ) from sequifier.rl.attachment import resolve_binding - execution, name = resolved_config_from_model_config( + execution, name = model_execution_from_model_config( read_execution_artifact( Path(self.project_root) / self.initialize_from ).model_config.values, diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 673c661f..e88ef908 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -725,16 +725,9 @@ class ResolvedDatasetPart(ConfigModel): file_order: Literal["shuffled", "name"] = "shuffled" -class ResolvedModelInterface(ConfigModel): - policy: ResolvedPolicyBinding | None = None - prediction_distributions: dict[str, PredictionDistributionConfig] = Field( - default_factory=dict - ) +class ResolvedInterfaceStorage(ConfigModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - depth_layouts: DepthLayoutRegistryModel = Field( - default_factory=DepthLayoutRegistryModel - ) tensor_payload_version: int = 1 split_paths: list[str] = Field(default_factory=list) prediction_aligned_splits: list[int] = Field(default_factory=list) @@ -742,12 +735,29 @@ class ResolvedModelInterface(ConfigModel): window_lengths: Optional[list[int]] = None prediction_length: Optional[int] = None target_offset: Optional[int] = None - name: str - input_columns: list[str] - static_conditioning: StaticConditioningConfig | None = None static_features_path: str | None = None image_sources: dict[str, ImageSourceConfig] = Field(default_factory=dict) preconditioning_image: PreconditioningImageConfig | None = None + storage_layout: StoredWindowLayout + + @classmethod + def from_interface( + cls, interface: ResolvedModelInterface + ) -> ResolvedInterfaceStorage: + return cls.model_validate(interface.model_dump(include=set(cls.model_fields))) + + +class ResolvedModelInterface(ResolvedInterfaceStorage): + policy: ResolvedPolicyBinding | None = None + prediction_distributions: dict[str, PredictionDistributionConfig] = Field( + default_factory=dict + ) + depth_layouts: DepthLayoutRegistryModel = Field( + default_factory=DepthLayoutRegistryModel + ) + name: str + input_columns: list[str] + static_conditioning: StaticConditioningConfig | None = None preconditioning_image_encoder: ImageEncoderConfig | None = None static_categorical_columns: list[str] = Field(default_factory=list) static_real_columns: list[str] = Field(default_factory=list) @@ -774,7 +784,6 @@ class ResolvedModelInterface(ConfigModel): target_decoder_ids: dict[str, list[int]] target_n_classes: dict[str, int] target_global_to_decoder: dict[str, list[int]] - storage_layout: StoredWindowLayout window_view: ModelWindowView @model_validator(mode="after") @@ -795,7 +804,7 @@ class ResolvedDatasetTrainingSpec(ConfigModel): name: str model_interface: str - interface: ResolvedModelInterface + storage: ResolvedInterfaceStorage | None = Field(default=None, exclude=True) parts: dict[str, ResolvedDatasetPart] criterion: dict[str, str] class_weights: Optional[dict[str, list[float]]] = None @@ -913,10 +922,6 @@ class ResolvedSequifierConfig(ConfigModel): """Runtime configuration after all dataset parts and interfaces resolve.""" _prepared_inputs: Any = PrivateAttr(default=None) - # Mixed plans keep prepared loader views alongside their model execution view. - _supervised_datasets: dict[str, ResolvedDatasetTrainingSpec] = PrivateAttr( - default_factory=dict - ) ownership: dict[str, Any] = Field(default_factory=dict) source_validation_fractions: dict[str, float] = Field(default_factory=dict) @@ -932,6 +937,7 @@ class ResolvedSequifierConfig(ConfigModel): seed: int runtime_settings: TrainingRuntimeSettings model: ModelSpecModel + interfaces: dict[str, ResolvedModelInterface] = Field(min_length=1) dataset_training: dict[str, ResolvedDatasetTrainingSpec] authored: SequifierConfig | None = Field(default=None, exclude=True) initialize_from: str | None = None @@ -947,11 +953,40 @@ class ResolvedSequifierConfig(ConfigModel): curriculum_training: bool = False curriculum_column: Optional[str] = None + @model_validator(mode="before") + @classmethod + def read_dataset_interfaces(cls, values): + """Read checkpoints with embedded dataset interface views.""" + if not isinstance(values, dict) or not isinstance( + values.get("dataset_training"), dict + ): + return values + interfaces = dict(values.get("interfaces", {})) + legacy = "interfaces" not in values + datasets = {} + for name, dataset in values["dataset_training"].items(): + if isinstance(dataset, dict) and "interface" in dataset: + dataset = dict(dataset) + interface = ResolvedModelInterface.model_validate( + dataset.pop("interface") + ) + if legacy: + interfaces.setdefault(dataset["model_interface"], interface) + if not dataset["parts"]: + continue + dataset["storage"] = ResolvedInterfaceStorage.from_interface(interface) + datasets[name] = dataset + return {**values, "interfaces": interfaces, "dataset_training": datasets} + @model_serializer(mode="wrap") - def serialize_actor_initialization(self, handler): + def serialize_training_config(self, handler, info): values = handler(self) if self.actor_initialization == "attach": values.pop("actor_initialization", None) + for name, dataset in values.get("dataset_training", {}).items(): + dataset["interface"] = self.dataset_interface(name).model_dump( + mode=info.mode + ) return values @property @@ -973,13 +1008,7 @@ def evaluation_sources(self) -> list[ResolvedTrainingSource]: export_pt: bool export_with_dropout: bool = False - @model_validator(mode="after") - @beartype - def validate_next_occurrence_metadata(self): - self.validate_next_occurrence_phases() - return self - - def validate_next_occurrence_phases(self, dataset_interfaces=None): + def validate_next_occurrence_phases(self): objective = get_objective_class(self.runtime_settings.training_objective) is_next = issubclass(objective, NextOccurrenceObjective) for phase in self.training_plan: @@ -998,12 +1027,8 @@ def validate_next_occurrence_phases(self, dataset_interfaces=None): source.dataset for source in phase.sources + phase.evaluation_sources } for dataset_name in names: - key = ( - dataset_interfaces[dataset_name] - if dataset_interfaces is not None - else dataset_name - ) - interface = self.dataset_training[key].interface + dataset = self.dataset_training[dataset_name] + interface = self.interface(dataset.model_interface) if column not in interface.target_columns: raise ValueError( f"{phase.name}: next_occurrence_config.column_name must be " @@ -1037,11 +1062,20 @@ def validate_model_execution_plans(self): from sequifier.model.execution import ModelExecution + for name, interface in self.interfaces.items(): + if name not in self.model.interfaces or name != interface.name: + raise ValueError(f"Invalid resolved model interface {name!r}") for dataset in self.dataset_training.values(): - if dataset.interface.depth_layouts and any( + if dataset.model_interface not in self.interfaces: + raise ValueError( + f"Dataset {dataset.name!r} references unknown interface " + f"{dataset.model_interface!r}" + ) + if self.interface(dataset.model_interface).depth_layouts and any( part.read_format != "pt" for part in dataset.parts.values() ): raise ValueError("Preprocessed depth inputs require read_format: pt") + self.validate_next_occurrence_phases() ModelExecution.from_config(self).validate() return self @@ -1053,7 +1087,7 @@ def dataset_count(self) -> int: @property @beartype def interface_names(self) -> tuple[str, ...]: - return tuple(self.model.interfaces) + return tuple(self.interfaces) @beartype def dataset(self, name: Optional[str] = None) -> ResolvedDatasetTrainingSpec: @@ -1066,16 +1100,17 @@ def dataset(self, name: Optional[str] = None) -> ResolvedDatasetTrainingSpec: @beartype def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: if name is None: - names = { - dataset.model_interface for dataset in self.dataset_training.values() - } - if len(names) != 1: + if len(self.interfaces) != 1: raise ValueError("A model interface selection is required") - name = next(iter(names)) - for dataset in self.dataset_training.values(): - if dataset.model_interface == name: - return dataset.interface - raise KeyError(name) + name = next(iter(self.interfaces)) + return self.interfaces[name] + + def dataset_interface(self, name: str) -> ResolvedModelInterface: + dataset = self.dataset_training[name] + interface = self.interface(dataset.model_interface) + if dataset.storage is None: + return interface + return interface.model_copy(update=dict(dataset.storage)) @dataclass(frozen=True) @@ -1566,13 +1601,15 @@ def resolve_sequifier_config( evaluated = _evaluated_parts(config) resolved_datasets: dict[str, ResolvedDatasetTrainingSpec] = {} - interface_semantics: dict[str, dict[str, Any]] = {} + interfaces = {} if execution is None else dict(execution.interfaces) + dataset_semantics: dict[str, dict[str, Any]] = {} for dataset_name, dataset_spec in config.dataset_training.items(): learning = config.dataset_learning(dataset_name) if dataset_spec.model_interface is None: raise ValueError(f"Dataset {dataset_name!r} requires a model interface") - interface_spec = config.model.interfaces[dataset_spec.model_interface] + interface_name = dataset_spec.model_interface + interface_spec = config.model.interfaces[interface_name] resolved_parts = {} expected_signature = None expected_form = None @@ -1662,22 +1699,12 @@ def resolve_sequifier_config( if first_training_layout is None: first_training_layout = layout assert first_training_layout is not None - interface = ( - execution.dataset_interface( - dataset_spec.model_interface, - interface_spec, - first_metadata, - first_training_layout, - config.runtime_settings, - ) - if execution is not None - else _resolve_interface( - dataset_spec.model_interface, - interface_spec, - first_metadata, - config.runtime_settings, - first_training_layout, - ) + interface = _resolve_interface( + interface_name, + interface_spec, + first_metadata, + config.runtime_settings, + first_training_layout, ) for part_name, resolved_part in resolved_parts.items(): ref = f"{dataset_name}.{part_name}" @@ -1726,15 +1753,33 @@ def resolve_sequifier_config( f"uses {interface.decoder.prediction_length}." ) semantic_contract = _interface_semantics(interface) - prior_contract = interface_semantics.get(dataset_spec.model_interface) + if execution is not None: + canonical = _interface_semantics(execution.interface(interface_name)) + dataset_fields = { + "storage_layout", + "window_view", + "policy", + "prediction_distributions", + } + _assert_compatible( + { + key: value + for key, value in canonical.items() + if key not in dataset_fields + }, + semantic_contract, + f"Dataset {dataset_name!r} for interface {interface_name!r}", + ) + prior_contract = dataset_semantics.get(interface_name) if prior_contract is not None: _assert_compatible( prior_contract, semantic_contract, - f"Datasets sharing interface {dataset_spec.model_interface!r}", + f"Datasets sharing interface {interface_name!r}", ) else: - interface_semantics[dataset_spec.model_interface] = semantic_contract + dataset_semantics[interface_name] = semantic_contract + interfaces.setdefault(interface_name, interface) for target, criterion in learning.criterion.items(): if ( @@ -1777,8 +1822,8 @@ def resolve_sequifier_config( resolved_datasets[dataset_name] = ResolvedDatasetTrainingSpec( name=dataset_name, - model_interface=dataset_spec.model_interface, - interface=interface, + model_interface=interface_name, + storage=ResolvedInterfaceStorage.from_interface(interface), parts=resolved_parts, criterion=learning.criterion, class_weights=learning.class_weights, @@ -1786,6 +1831,23 @@ def resolve_sequifier_config( class_share_log_columns=learning.class_share_log_columns, ) + for name, spec in config.model.interfaces.items(): + if name in interfaces: + continue + path = spec.input_contract.metadata_config_path + if path is None: + raise ValueError(f"{name}: model interface requires metadata") + interface_metadata = load_dataset_metadata( + normalize_path(path, config.run.project_root) + ) + interfaces[name] = _resolve_interface( + name, + spec, + interface_metadata, + config.runtime_settings, + interface_metadata.storage_layout, + ) + phases = resolve_supervised_phases( config.training_plan.phases, seed=config.run.reproducibility.seed ) @@ -1803,6 +1865,7 @@ def resolve_sequifier_config( seed=config.run.reproducibility.seed, runtime_settings=config.runtime_settings, model=config.model.architecture_model(), + interfaces=interfaces, dataset_training=resolved_datasets, training_plan=phases, curriculum_training=bool(config.training_plan.phases[0].curriculum), @@ -2099,7 +2162,9 @@ def dataset_part_view( dataset = config.dataset_training[dataset_name] part = dataset.parts[part_name] - interface_config = interface_build_view(config, dataset.interface) + interface_config = interface_build_view( + config, config.dataset_interface(dataset_name) + ) interface_values = vars(interface_config).copy() interface_values["static_features_path"] = part.metadata.static_features_path selected_sources = { diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 01a7b4c5..41db2f5c 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -11,6 +11,7 @@ DatasetPartSpecModel, OwnedModelSpec, PreferencePartSpecModel, + ResolvedSequifierConfig, SequencePartSpecModel, SequifierConfig, TrainingRuntimeSettings, @@ -557,29 +558,28 @@ def resolve_training_run(authored: SequifierConfig): and previous.model_interface != current.model_interface ): raise ValueError("optimizer continuation cannot change model_interface") - supervised = None if any(phase.algorithm == "supervised" for phase in config.training_plan.phases): subset = _supervised_subset(config).model_copy( update={"initialize_from": None, "actor_initialization": "attach"} ) - supervised = resolve_sequifier_config( - subset, part_metadata, execution=execution - ) - policy_phases = any( - phase.algorithm != "supervised" for phase in config.training_plan.phases - ) - if policy_phases: - resolved = execution.training_config( + resolved = resolve_sequifier_config(subset, part_metadata, execution=execution) + else: + resolved = ResolvedSequifierConfig( + model=execution.model, + interfaces=execution.interfaces, + runtime_settings=training, + dataset_training={}, + training_plan=[], device=config.run.execution.device, project_root=str(root), model_name=config.run.model_name, seed=config.run.reproducibility.seed, + export_generative_model=config.run.export.generative_model, + export_embedding_model=config.run.export.embedding_model, + embedding_layer_names=config.run.export.embedding_layer_names, + export_onnx=config.run.export.onnx, + export_pt=config.run.export.pt, ) - if supervised is not None: - resolved._supervised_datasets = supervised.dataset_training - else: - assert supervised is not None - resolved = supervised resolved.ownership = config.model_dump(mode="json") resolved.source_validation_fractions = _source_fractions(config) resolved.normalization_by_interface = { @@ -603,16 +603,11 @@ def resolve_training_run(authored: SequifierConfig): resolve_training_output_paths(resolved) resolved.sequence_sources = sequences resolved.preference_sources = preferences - if policy_phases: - supervised_by_name = ( - {phase.name: phase for phase in supervised.training_plan} - if supervised is not None - else {} - ) - resolved.training_plan = [ - supervised_by_name.get(phase.name, phase) - for phase in config.training_plan.phases - ] + supervised_by_name = {phase.name: phase for phase in resolved.training_plan} + resolved.training_plan = [ + supervised_by_name[phase.name] if phase.algorithm == "supervised" else phase + for phase in config.training_plan.phases + ] for index, phase in enumerate(resolved.training_plan): if phase.algorithm == "supervised": phase.resolved_seed = config.run.reproducibility.seed + index @@ -643,6 +638,7 @@ def policy_run_config( from sequifier.artifacts.model_config import model_execution_from_model_config from sequifier.artifacts.model_export import model_execution_config from sequifier.config.components import ResumeConfig + from sequifier.model.execution import ModelExecution from sequifier.rl.attachment import apply_binding, resolve_binding from sequifier.rl.prepared_inputs import PreparedTrainingInputs from sequifier.rl.run_config import ResolvedPolicyRun @@ -651,7 +647,7 @@ def policy_run_config( if len(names) != 1: raise ValueError("a native policy lifecycle requires one interface") name = next(iter(names)) - execution = config + execution = ModelExecution.from_config(config) initialization = ( incoming if index or incoming is not None else config.initialize_from ) diff --git a/src/sequifier/evaluation/compatibility.py b/src/sequifier/evaluation/compatibility.py index f36687aa..6fb4c8ac 100644 --- a/src/sequifier/evaluation/compatibility.py +++ b/src/sequifier/evaluation/compatibility.py @@ -25,6 +25,7 @@ def evaluation_fingerprint(config, phase_index): sources = [] for source in phase.evaluation_sources: dataset = config.dataset_training[source.dataset] + interface = config.dataset_interface(source.dataset) learning = phase.learning.datasets.get(source.dataset) if learning is None: return None @@ -33,7 +34,7 @@ def evaluation_fingerprint(config, phase_index): { "source": source.source, "interface": dataset.model_interface, - "interface_contract": dataset.interface.model_dump(mode="json"), + "interface_contract": interface.model_dump(mode="json"), "parts": [ { "name": name, @@ -59,7 +60,7 @@ def evaluation_fingerprint(config, phase_index): "class_weights": learning.class_weights, "loss_weights": { target: (learning.loss_weights or {}).get(target, 1.0) - for target in dataset.interface.target_columns + for target in interface.target_columns }, } ) diff --git a/src/sequifier/evaluation/service.py b/src/sequifier/evaluation/service.py index a2d71d1c..7a5efadb 100644 --- a/src/sequifier/evaluation/service.py +++ b/src/sequifier/evaluation/service.py @@ -78,7 +78,7 @@ def __init__( @staticmethod def _baseline_definition(dataset: Any) -> str: """Identify inputs that affect raw baseline loss statistics.""" - interface = dataset.config.interface + interface = dataset.interface training = getattr(dataset.objective.config, "runtime_settings", None) def model_value(value: Any) -> Any: @@ -111,7 +111,7 @@ def model_value(value: Any) -> Any: def _baseline_output( self, prepared: PreparedBatch, dataset: Any, decoded_length: int, network: Any ) -> ModelOutput: - interface = dataset.config.interface + interface = dataset.interface logits = {} auxiliary_logits = {} for target in interface.target_columns: @@ -166,7 +166,7 @@ def evaluate( source = build_source_runtime(source_config, datasets) source.set_epoch(context.phase_epoch, "validation") dataset = source.dataset - targets = list(dataset.config.interface.target_columns) + targets = list(dataset.interface.target_columns) dtype = ( torch.float32 if context.device.type == "mps" else torch.float64 ) @@ -193,9 +193,9 @@ def evaluate( (), device=context.device, dtype=torch.int64 ) for target in getattr( - dataset.config.interface, "categorical_hashing", {} + dataset.interface, "categorical_hashing", {} ) - if target in dataset.config.interface.target_decoder_ids + if target in dataset.interface.target_decoder_ids } accuracy_count = torch.zeros( (), device=context.device, dtype=torch.int64 @@ -371,7 +371,7 @@ def evaluate( } distributions: dict[str, list[dict[str, Any]]] = {} index_maps = construct_index_maps( - dataset.config.interface.id_maps, + dataset.interface.id_maps, list(class_counts), True, ) diff --git a/src/sequifier/export/preflight.py b/src/sequifier/export/preflight.py index 3c767006..5e05cc61 100644 --- a/src/sequifier/export/preflight.py +++ b/src/sequifier/export/preflight.py @@ -9,23 +9,24 @@ import subprocess import sys import tempfile +from dataclasses import replace from importlib.metadata import version from pathlib import Path from sequifier.artifacts.manifests import write_manifest from sequifier.artifacts.model_export import model_execution_config +from sequifier.model.execution import ModelExecution PROTOCOL_VERSION = 1 PREFLIGHT_SEED = 20260910 -def preflight_description(config): +def preflight_description(config, execution): import torch versions = { name: version(name) for name in ("torch", "onnx", "onnxscript", "onnxruntime") } - execution = model_execution_config(config) # Initial weights and their provenance do not affect graph capability. def graph_only(value): @@ -91,17 +92,25 @@ def graph_only(value): def ensure_export_preflight(config, distributed): - if not config.export_onnx or not any( - dataset.interface.depth_layouts - or dataset.interface.image_sources - or dataset.interface.preconditioning_image is not None + if not config.export_onnx: + return + execution = ModelExecution.from_config(config) + interfaces = { + dataset.model_interface: execution.interface(dataset.model_interface) for dataset in config.dataset_training.values() + } + if not any( + interface.depth_layouts + or interface.image_sources + or interface.preconditioning_image is not None + for interface in interfaces.values() ): return failure = None if distributed.rank == 0: try: - description = preflight_description(config) + contract = model_execution_config(replace(execution, interfaces=interfaces)) + description = preflight_description(config, contract) encoded = json.dumps( description, sort_keys=True, separators=(",", ":"), allow_nan=False ).encode() @@ -128,7 +137,7 @@ def ensure_export_preflight(config, distributed): request.write_text( json.dumps( { - "execution": model_execution_config(config), + "execution": contract, "generative": config.export_generative_model, "embedding": config.export_embedding_model, "dropout": config.export_with_dropout, @@ -173,7 +182,7 @@ def _worker(request_path): import onnxruntime import torch - from sequifier.artifacts.model_config import resolved_config_from_model_config + from sequifier.artifacts.model_config import model_execution_from_model_config from sequifier.export.embedding import EmbeddingModelExporter from sequifier.export.onnx import OnnxModelExporter from sequifier.helpers import configure_determinism @@ -183,29 +192,27 @@ def _worker(request_path): request = json.loads(request_path.read_text()) configure_determinism(request["seed"]) onnxruntime.set_seed(request["seed"]) - config, _ = resolved_config_from_model_config(request["execution"], device="cpu") + config, _ = model_execution_from_model_config(request["execution"], device="cpu") network = build_transformer_network( config, device=torch.device("cpu") ).network.float() exporter = OnnxModelExporter() - for index, dataset in enumerate(config.dataset_training.values()): + for index, (name, interface) in enumerate(config.interfaces.items()): if request["generative"]: exporter.export( network, - dataset.interface, + interface, request_path.parent / f"{index}-generative.onnx", - interface_name=dataset.model_interface, + interface_name=name, batch_size=2, context_length=config.runtime_settings.context_length, training=request["dropout"], ) if request["embedding"]: - embedding = EmbeddingModelExporter().build( - network, dataset.model_interface, config - ) + embedding = EmbeddingModelExporter().build(network, name, config) exporter.export( embedding, - dataset.interface, + interface, request_path.parent / f"{index}-embedding.onnx", interface_name=None, batch_size=2, diff --git a/src/sequifier/export/service.py b/src/sequifier/export/service.py index 912a98d0..cc06b630 100644 --- a/src/sequifier/export/service.py +++ b/src/sequifier/export/service.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, Protocol @@ -15,6 +15,7 @@ from sequifier.export.pytorch import PyTorchModelExporter from sequifier.logging_paths import model_artifact_path from sequifier.model.embedding import ONNX_EMBEDDING_LAYER_NAMES_KEY +from sequifier.model.execution import ModelExecution from sequifier.model.execution_schema import MODEL_CONFIG_KEY from sequifier.model.factory import build_transformer_network from sequifier.runtime.random_state import RandomStateManager @@ -39,12 +40,6 @@ def export( ) -> Path: ... -def _onnx_execution_config(config, interface_name): - execution = model_execution_config(config) - execution["interfaces"] = {interface_name: execution["interfaces"][interface_name]} - return execution - - class ExportService: def __init__(self, config: Any, rank: int) -> None: self.config = config @@ -70,11 +65,18 @@ def _export( if self.rank != 0: return ExportResult() del network - export_config = self.config.model_copy(deep=True) - export_config.device = "cpu" - export_config.runtime_settings.distributed = False - export_config.runtime_settings.data_parallelism = None - export_config.runtime_settings.torch_compile = "none" + execution = ModelExecution.from_config(self.config) + export_config = replace( + execution, + device="cpu", + runtime_settings=execution.runtime_settings.model_copy( + update={ + "distributed": False, + "data_parallelism": None, + "torch_compile": "none", + } + ), + ) export_network = build_transformer_network( export_config, device=torch.device("cpu"), initialize=False ).network @@ -111,9 +113,21 @@ def _export( ) ) if self.config.export_onnx: - onnx_network = export_network - count = len(self.config.dataset_training) - for dataset_name, dataset in self.config.dataset_training.items(): + contract = model_execution_config(execution) + settings = execution.runtime_settings + datasets = self.config.dataset_training + dataset_count = len(datasets) + for dataset_name, dataset in datasets.items(): + name = dataset.model_interface + interface = execution.interface(name) + metadata = { + MODEL_CONFIG_KEY: json.dumps( + {**contract, "interfaces": {name: contract["interfaces"][name]}} + ), + ONNX_CATEGORICAL_TARGET_CODECS_KEY: json.dumps( + interface.target_decoder_ids + ), + } if self.config.export_generative_model: path = model_artifact_path( self.config.project_root, @@ -121,61 +135,43 @@ def _export( f"{options.suffix}-{options.epoch}", "onnx", dataset_name=dataset_name, - dataset_count=count, + dataset_count=dataset_count, ) paths.append( self.onnx.export( - onnx_network, - dataset.interface, + export_network, + interface, path, - interface_name=dataset.model_interface, - batch_size=self.config.runtime_settings.inference_batch_size, - context_length=self.config.runtime_settings.context_length, + interface_name=name, + batch_size=settings.inference_batch_size, + context_length=settings.context_length, training=self.config.export_with_dropout, - metadata={ - MODEL_CONFIG_KEY: json.dumps( - _onnx_execution_config( - self.config, dataset.model_interface - ) - ), - ONNX_CATEGORICAL_TARGET_CODECS_KEY: json.dumps( - dataset.interface.target_decoder_ids - ), - }, + metadata=metadata, ) ) if self.config.export_embedding_model: - embedding = self.embedding.build( - onnx_network, dataset.model_interface, self.config - ) + embedding = self.embedding.build(export_network, name, execution) path = model_artifact_path( self.config.project_root, self.config.model_name, f"{options.suffix}-embedding-{options.epoch}", "onnx", dataset_name=dataset_name, - dataset_count=count, + dataset_count=dataset_count, ) paths.append( self.onnx.export( embedding, - dataset.interface, + interface, path, interface_name=None, - batch_size=self.config.runtime_settings.inference_batch_size, - context_length=self.config.runtime_settings.context_length, + batch_size=settings.inference_batch_size, + context_length=settings.context_length, training=self.config.export_with_dropout, metadata={ - MODEL_CONFIG_KEY: json.dumps( - _onnx_execution_config( - self.config, dataset.model_interface - ) - ), - ONNX_CATEGORICAL_TARGET_CODECS_KEY: json.dumps( - dataset.interface.target_decoder_ids - ), + **metadata, ONNX_EMBEDDING_LAYER_NAMES_KEY: json.dumps( - self.config.embedding_layer_names + execution.embedding_layer_names ), }, ) diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py index f5e3722f..096f89a3 100644 --- a/src/sequifier/io/output_paths.py +++ b/src/sequifier/io/output_paths.py @@ -148,16 +148,16 @@ def supervised_input_paths(config): paths = set() if config.initialize_from: paths.add((root / config.initialize_from).resolve()) - for dataset in config.dataset_training.values(): + for name, dataset in config.dataset_training.items(): for part in dataset.parts.values(): - for name in ( + for field in ( "metadata_config_path", "training_data_path", "validation_data_path", ): - if value := getattr(part, name, None): + if value := getattr(part, field, None): paths.add((root / value).resolve()) - interface = dataset.interface + interface = config.dataset_interface(name) if interface.static_features_path: paths.add((root / interface.static_features_path).resolve()) for source in interface.image_sources.values(): diff --git a/src/sequifier/model/execution.py b/src/sequifier/model/execution.py index b8553421..5cdc86cf 100644 --- a/src/sequifier/model/execution.py +++ b/src/sequifier/model/execution.py @@ -1,14 +1,24 @@ """Model execution facts, independent of dataset ownership and learning phases.""" +from __future__ import annotations + from dataclasses import dataclass, field -from typing import Any +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sequifier.config.train_config import ( + ModelSpecModel, + ResolvedModelInterface, + ResolvedSequifierConfig, + TrainingRuntimeSettings, + ) @dataclass class ModelExecution: - model: Any - interfaces: dict - runtime_settings: Any + model: ModelSpecModel + interfaces: dict[str, ResolvedModelInterface] + runtime_settings: TrainingRuntimeSettings device: str = "cpu" project_root: str = "." model_name: str = "loaded-model" @@ -18,27 +28,34 @@ class ModelExecution: ) @classmethod - def from_config(cls, config): + def from_config( + cls, config: ResolvedSequifierConfig | ModelExecution + ) -> ModelExecution: if isinstance(config, cls): return config - interfaces = {} - for dataset in config.dataset_training.values(): - interfaces.setdefault(dataset.model_interface, dataset.interface) - execution = cls(config.model, interfaces, config.runtime_settings) - for name in ( - "device", - "project_root", - "model_name", - "seed", - "embedding_layer_names", - ): - setattr(execution, name, getattr(config, name, getattr(execution, name))) - return execution + return cls( + model=config.model, + interfaces=config.interfaces, + runtime_settings=config.runtime_settings, + device=config.device, + project_root=config.project_root, + model_name=config.model_name, + seed=config.seed, + embedding_layer_names=config.embedding_layer_names, + ) - def interface(self, name): + @property + def interface_names(self) -> tuple[str, ...]: + return tuple(self.interfaces) + + def interface(self, name: str | None = None) -> ResolvedModelInterface: + if name is None: + if len(self.interfaces) != 1: + raise ValueError("A model interface selection is required") + name = next(iter(self.interfaces)) return self.interfaces[name] - def validate(self): + def validate(self) -> ModelExecution: from sequifier.config.train_config import interface_build_view from sequifier.model.decoders import resolve_decoding_plan from sequifier.model.ingestion_compiler import resolve_ingestion_plan @@ -48,70 +65,3 @@ def validate(self): resolve_ingestion_plan(view) resolve_decoding_plan(view) return self - - def dataset_interface(self, name, spec, metadata, storage_layout, runtime_settings): - """Resolve a storage view while retaining model-owned task parameters.""" - from sequifier.config.train_config import _resolve_interface - - interface = _resolve_interface( - name, spec, metadata, runtime_settings, storage_layout - ) - previous = self.interface(name) - interface.policy = previous.policy - interface.prediction_distributions = dict(previous.prediction_distributions) - return interface - - def dataset(self, name): - """Construct the resolved loss and interface view needed by execution.""" - from sequifier.config.train_config import ResolvedDatasetTrainingSpec - - interface = self.interfaces[name] - return ResolvedDatasetTrainingSpec( - name=name, - model_interface=name, - interface=interface, - parts={}, - criterion={ - target: "CrossEntropyLoss" if kind == "categorical" else "MSELoss" - for target, kind in interface.target_column_types.items() - }, - class_share_log_columns=[], - ) - - def training_config(self, *, device=None, **options): - """Adapt execution to the public, serialized training schema at its boundary.""" - from sequifier.config.train_config import ResolvedSequifierConfig - - return ResolvedSequifierConfig( - runtime_settings=self.runtime_settings, - model=self.model, - dataset_training={name: self.dataset(name) for name in self.interfaces}, - **{ - "project_root": self.project_root, - "model_name": self.model_name, - "seed": self.seed, - "device": self.device if device is None else device, - "training_plan": [], - "export_generative_model": True, - "export_embedding_model": False, - "export_onnx": False, - "export_pt": False, - "embedding_layer_names": list(self.embedding_layer_names), - **options, - }, - ) - - def retain_interfaces(self, config): - """Keep inactive interfaces when selecting supervised datasets.""" - existing = { - dataset.model_interface for dataset in config.dataset_training.values() - } - for name in self.model.interfaces: - if name in existing: - continue - key = name - while key in config.dataset_training: - key += "_unused" - config.dataset_training[key] = self.dataset(name).model_copy( - update={"name": key} - ) diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index 032af723..eb7fef44 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -10,6 +10,7 @@ from sequifier.helpers import get_torch_dtype from sequifier.model.backbone import TransformerBackbone from sequifier.model.decoders import build_target_decoding +from sequifier.model.execution import ModelExecution from sequifier.model.ingestion_compiler import compile_feature_ingestion from sequifier.model.initialization import ( initialize_model_weights, @@ -67,11 +68,9 @@ class BuiltModel: def validate_hash_contracts(network: ComposableTransformerNetwork, config: Any) -> None: """Check persisted codec buffers after weights are restored.""" - from sequifier.model.execution import ModelExecution from sequifier.model.ingestions import EmbeddingFeatureIngestion, MultiHashEmbedding - interfaces = ModelExecution.from_config(config).interfaces - for name, interface in interfaces.items(): + for name, interface in ModelExecution.from_config(config).interfaces.items(): route = network.resolve_interface(name) for column, expected in interface.categorical_hash_contracts.items(): from sequifier.model.hash_codec import CategoricalHashCodec @@ -253,7 +252,7 @@ def _decoder_metadata(view: Any) -> InterfaceRuntimeMetadata: @beartype def _build_composable_network( - config: Any, + config: ModelExecution, *, device: torch.device, initialize: bool, @@ -261,11 +260,9 @@ def _build_composable_network( logger: Any, ) -> BuiltModel: from sequifier.config.train_config import interface_build_view - from sequifier.model.execution import ModelExecution - resolved_interfaces = ModelExecution.from_config(config).interfaces objectives = {} - for name, interface in resolved_interfaces.items(): + for name, interface in config.interfaces.items(): objective_view = interface_build_view(config, interface) # Next-occurrence target construction belongs to each training phase; # its forward attention policy is the ordinary causal policy. @@ -284,7 +281,7 @@ def _build_composable_network( routes = {} runtime_metadata = {} route_initialization_components: dict[str, tuple[nn.Module, Any]] = {} - for name, interface in resolved_interfaces.items(): + for name, interface in config.interfaces.items(): view = interface_build_view(config, interface) load_pretrained = initialize and name != restored_history_interface built_ingestion = compile_feature_ingestion( @@ -364,10 +361,15 @@ def _build_composable_network( if interface.preconditioning_image_encoder is not None: from sequifier.model.images import ImagePrefixEncoder + image_source = interface.preconditioning_image + if image_source is None: + raise ValueError( + "Metadata lacks the configured preconditioning image source" + ) image_prefix_encoder = ImagePrefixEncoder( interface.preconditioning_image_encoder, backbone.model_dim, - len(interface.preconditioning_image.views), + len(image_source.views), load_pretrained, ) prefix_projection = ( @@ -513,7 +515,7 @@ def build_transformer_network( resolved_logger: Any = logger if logger is not None else logging.getLogger(__name__) return _build_composable_network( - config, + ModelExecution.from_config(config), device=device, initialize=initialize, restored_history_interface=restored_history_interface, diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 78fc4fda..087469df 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -7,7 +7,7 @@ from sequifier.artifacts.model_artifact import ( ModelArtifact, build_model_artifact, - load_model_artifact, + read_artifact, restore_model_artifact, save_model_artifact, ) @@ -59,22 +59,22 @@ def load_policy( binding=None, model=None, ) -> Policy: - if isinstance(path, ModelArtifact): - network, config, artifact = restore_model_artifact( - path, device=device, interface_name=interface_name - ) - else: - network, config, artifact = load_model_artifact( - path, device=device, interface_name=interface_name - ) + artifact = ( + path + if isinstance(path, ModelArtifact) + else read_artifact(path, expected_type="sequifier_model")[0] + ) + network, execution, _ = restore_model_artifact( + artifact, device=device, interface_name=interface_name + ) if interface_name is None: candidates = ( - list(config.interface_names) + list(execution.interface_names) if binding is not None else [ name - for name, dataset in config.dataset_training.items() - if dataset.interface.policy is not None + for name, interface in execution.interfaces.items() + if interface.policy is not None ] ) if len(candidates) != 1: @@ -84,13 +84,13 @@ def load_policy( if model is not None: from sequifier.artifacts.model_config import extend_prediction_distributions - extend_prediction_distributions(config, model) + extend_prediction_distributions(execution, model) if binding is not None: from sequifier.rl.attachment import attach_policy - policy = attach_policy(network, config, interface_name, binding) + policy = attach_policy(network, execution, interface_name, binding) else: - policy = Policy.from_network(network, config, interface_name) + policy = Policy.from_network(network, execution, interface_name) policy.units = dict(policy.config.units) validate_policy_weights(policy) policy.eval() diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index e1bde758..f0a3805a 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -9,7 +9,6 @@ from torch import Tensor, nn from sequifier.config.policy_config import ResolvedPolicyBinding -from sequifier.config.train_config import ResolvedSequifierConfig from sequifier.io.batch import SequifierInputs from sequifier.io.policy_window import normalize_policy_features from sequifier.io.preprocessing import normalization_statistics @@ -24,7 +23,13 @@ def policy_execution_config( - config, device="cpu", *, execution_model=None, **run_options + config, + device="cpu", + *, + execution_model=None, + project_root=".", + model_name="loaded-model", + seed=0, ): """Select policy alignment over canonical model-owned execution facts.""" from sequifier.artifacts.model_config import model_execution_from_model_config @@ -36,9 +41,9 @@ def policy_execution_config( execution_model, device=device, interface_name=config.interface_name ) apply_binding(execution, config.interface_name, config) - for key, value in run_options.items(): - if key in ModelExecution.__dataclass_fields__: - setattr(execution, key, value) + execution.project_root = project_root + execution.model_name = model_name + execution.seed = seed return execution @@ -46,7 +51,7 @@ class Policy(nn.Module): """A convenience view; the network owns all trainable weights and execution.""" config: ResolvedPolicyBinding - execution_config: ResolvedSequifierConfig | ModelExecution + execution_config: ModelExecution interface_name: str network: ComposableTransformerNetwork units: dict[str, str] diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index e619cd26..5036e4b3 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -117,7 +117,10 @@ def _strategy( local_rank=execution.local_rank, world_size=execution.world_size, device=execution.device, - find_unused_parameters=len(config.dataset_training) > 1, + find_unused_parameters=max( + len(config.dataset_training), len(config.interfaces) + ) + > 1, ) def _warm_up(self, run: TrainingRun) -> None: @@ -138,7 +141,7 @@ def _warm_up(self, run: TrainingRun) -> None: try: for dataset in run.datasets.values(): schema = ExecutionSchema.from_interface( - dataset.config.interface, + dataset.interface, run.config.runtime_settings.context_length, ) features, metadata = schema.bind( @@ -206,6 +209,11 @@ def build( loaded = None if loaded is not None: restorer.validate_for_restore(loaded, config) + # Optimizer state uses the parameter order of the saved network. + config.interfaces = { + name: config.interfaces[name] + for name in loaded.checkpoint.model.model_config.values["interfaces"] + } state = RunState.from_state_dict(loaded.checkpoint.run_state) else: state = RunState() diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 5f2f2384..27369f9e 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -50,11 +50,7 @@ class LoadedInferenceModel: @property def interface(self) -> Any: - return next( - dataset.interface - for dataset in self.config.dataset_training.values() - if dataset.model_interface == self.interface_name - ) + return self.config.interface(self.interface_name) @property def categorical_columns(self) -> list[str]: @@ -269,18 +265,6 @@ def load_inference_model( "model_interface is required for a multi-interface artifact." ) interface_name = next(iter(network.interfaces)) - if interface_name not in network.interfaces: - mapped = next( - ( - dataset.model_interface - for name, dataset in config.dataset_training.items() - if name == interface_name - ), - None, - ) - if mapped is None: - raise ValueError(f"Unknown model interface {interface_name!r}.") - interface_name = mapped network.eval() if infer_with_dropout: for module in network.modules(): diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index bb309114..30d2c4ac 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -93,6 +93,29 @@ def run_policy_plan(config): return run_learning_plan(config) +def _legacy_learning_plan_identity(plan): + """Recover the manifest identity used before runs serialized an interface registry.""" + legacy = dict(plan) + legacy["dataset_training"] = { + name: { + "name": name, + "model_interface": name, + "interface": interface, + "parts": {}, + "criterion": { + target: "CrossEntropyLoss" if kind == "categorical" else "MSELoss" + for target, kind in interface["target_column_types"].items() + }, + "class_weights": None, + "loss_weights": None, + "class_share_log_columns": [], + } + for name, interface in legacy.pop("interfaces").items() + } + legacy.update(curriculum_training=False, curriculum_column=None) + return fingerprint(legacy) + + def run_learning_plan(config) -> LearningRunResult: phases = config.training_plan if any(phase.optimizer_state == "continue" for phase in phases): @@ -127,9 +150,13 @@ def run_learning_plan(config) -> LearningRunResult: if ( state.get("artifact_type") != "sequifier_learning_run" or state.get("format_version") != 2 - or state.get("plan") != identity + or ( + state.get("plan") != identity + and state.get("plan") != _legacy_learning_plan_identity(plan) + ) ): raise ValueError("learning run checkpoint differs from the requested plan") + state["plan"] = identity elif resume.policy == "required": raise FileNotFoundError(checkpoint) identity_scope = RunIdentity.start(state.get("run_id")) diff --git a/src/sequifier/training/loss.py b/src/sequifier/training/loss.py index bf8d8171..7e07c4ac 100644 --- a/src/sequifier/training/loss.py +++ b/src/sequifier/training/loss.py @@ -41,7 +41,7 @@ def prepare_batch( *, eval_seed: int | None = None, ) -> PreparedBatch: - interface = dataset.config.interface + interface = dataset.interface if ( interface.depth_layouts or interface.static_conditioning is not None @@ -112,7 +112,7 @@ def calculate( dataset: DatasetRuntime, network: ComposableTransformerNetwork, ) -> LossResult: - interface = dataset.config.interface + interface = dataset.interface target_names = list(interface.target_columns) missing = set(target_names).difference(batch.targets) if missing: @@ -326,7 +326,7 @@ def finalize_accounting( *, allow_empty: bool = False, ) -> tuple[Tensor, dict[str, Tensor]]: - targets = list(dataset.config.interface.target_columns) + targets = list(dataset.interface.target_columns) reduced_sums, reduced_count = self.reduce_accounting(sums, count, targets) return self.finalize_reduced_accounting( reduced_sums, @@ -363,7 +363,7 @@ def finalize_reduced_accounting( allow_empty: bool = False, ) -> tuple[Tensor, dict[str, Tensor]]: """Apply the dataset's current loss weights to reduced accounting sums.""" - logical_targets = list(dataset.config.interface.target_columns) + logical_targets = list(dataset.interface.target_columns) targets = logical_targets + sorted(set(sums) - set(logical_targets)) if count.item() == 0: if not allow_empty: @@ -377,9 +377,7 @@ def finalize_reduced_accounting( else: logical = target[len("@hash/") :].rsplit("/", 1)[0] width_count = len( - dataset.config.interface.categorical_hash_contracts[logical][ - "widths" - ] + dataset.interface.categorical_hash_contracts[logical]["widths"] ) weight = ( float((dataset.loss_weights or {}).get(logical, 1.0)) / width_count diff --git a/src/sequifier/training/phase_runners.py b/src/sequifier/training/phase_runners.py index af6ed01e..a2ef59e4 100644 --- a/src/sequifier/training/phase_runners.py +++ b/src/sequifier/training/phase_runners.py @@ -1,6 +1,6 @@ """Adapt resolved phases to native execution and exact recovery.""" -from dataclasses import dataclass +from dataclasses import dataclass, replace import torch @@ -50,10 +50,9 @@ def supervised_phase_config(config, phase, index, incoming, resume): ) training.log_interval_iterations = 10 names = {ref.dataset for ref in [*phase.sources, *phase.evaluation_sources]} - prepared = config._supervised_datasets or config.dataset_training datasets = { name: dataset.model_copy(update=phase.learning.datasets[name].model_dump()) - for name, dataset in prepared.items() + for name, dataset in config.dataset_training.items() if name in names } if datasets.keys() != names or any( @@ -62,7 +61,7 @@ def supervised_phase_config(config, phase, index, incoming, resume): raise ValueError(f"{phase.name}: supervised phase requires resolved datasets") model = config.model - source = ModelExecution.from_config(config) + interfaces = config.interfaces artifact = read_execution_artifact(incoming) if incoming is not None else None if artifact is not None: source, _ = model_execution_from_model_config( @@ -70,32 +69,21 @@ def supervised_phase_config(config, phase, index, incoming, resume): ) extend_prediction_distributions(source, config.model) model = source.model + interfaces = source.interfaces training.forward_attention = source.runtime_settings.forward_attention or ( "bidirectional" if source.runtime_settings.training_objective == "bert" else "causal" ) - datasets = { - name: dataset.model_copy( - update={ - "interface": dataset.interface.model_copy( - update={ - "policy": source.interface(dataset.model_interface).policy, - "prediction_distributions": dict( - source.interface( - dataset.model_interface - ).prediction_distributions - ), - } - ) - } - ) - for name, dataset in datasets.items() - } + planned = replace(ModelExecution.from_config(config), runtime_settings=training) + validate_execution_attachment( + artifact.model_config.values, model_execution_config(planned) + ) resolved = config.model_copy( update={ "model": model, + "interfaces": interfaces, "dataset_training": datasets, "training_plan": [phase.model_copy(deep=True)], "active_phase_index": 0, @@ -112,11 +100,6 @@ def supervised_phase_config(config, phase, index, incoming, resume): "phase_directory": str(native / "native/phases"), } ) - source.retain_interfaces(resolved) - if artifact is not None: - validate_execution_attachment( - artifact.model_config.values, model_execution_config(resolved) - ) return resolved diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 135f0008..6d88da6c 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -14,6 +14,7 @@ from sequifier.config.train_config import ( ResolvedDatasetTrainingSpec, + ResolvedModelInterface, ResolvedSequifierConfig, ResolvedTrainingPhase, ResolvedTrainingSource, @@ -265,6 +266,7 @@ class DatasetRuntime: name: str config: ResolvedDatasetTrainingSpec interface_name: str + interface: ResolvedModelInterface objective: Objective parts: dict[str, DatasetPartRuntime] criteria: dict[str, nn.Module] @@ -451,10 +453,11 @@ def iter_batches( @beartype def _criterion_modules( - dataset: ResolvedDatasetTrainingSpec, device: torch.device + dataset: ResolvedDatasetTrainingSpec, + interface: ResolvedModelInterface, + device: torch.device, ) -> dict[str, nn.Module]: modules = {} - interface = dataset.interface for target in interface.target_columns: if target in interface.categorical_hashing: if dataset.class_weights and target in dataset.class_weights: @@ -506,6 +509,7 @@ def build_dataset_runtimes( runtimes = {} for name, dataset in config.dataset_training.items(): + interface = config.interface(dataset.model_interface) frozen = frozen_parameter_ids_by_dataset[name] route = network.interfaces[dataset.model_interface] active_parameter_ids = { @@ -519,6 +523,7 @@ def build_dataset_runtimes( name=name, config=dataset, interface_name=dataset.model_interface, + interface=interface, objective=objectives[dataset.model_interface], parts={ part_name: DatasetPartRuntime( @@ -526,7 +531,7 @@ def build_dataset_runtimes( ) for part_name in dataset.parts }, - criteria=_criterion_modules(dataset, device), + criteria=_criterion_modules(dataset, interface, device), loss_weights=dataset.loss_weights, class_weights=dataset.class_weights, frozen_parameter_ids=frozen, @@ -756,13 +761,15 @@ def configure_phase(config, network, datasets, phase_index, device): if learning is not None: for field, value in learning.model_dump().items(): setattr(dataset.config, field, value) - dataset.criteria = _criterion_modules(dataset.config, device) + dataset.criteria = _criterion_modules( + dataset.config, dataset.interface, device + ) dataset.class_weights = learning.class_weights dataset.loss_weights = learning.loss_weights dataset.frozen_parameter_ids = masks[name] if name in phase.learning.datasets: dataset.objective = create_objective( - interface_build_view(config, dataset.config.interface) + interface_build_view(config, dataset.interface) ) for part in dataset.parts.values(): if rebuild_loaders: @@ -775,14 +782,13 @@ def validate_phase_learning(config, device): for phase in config.training_plan: for name, learning in phase.learning.datasets.items(): dataset = config.dataset_training[name].model_copy(deep=True) + interface = config.interface(dataset.model_interface) for key, value in learning.model_dump().items(): setattr(dataset, key, value) - learning.validate_targets( - dataset.interface.target_columns, f"{phase.name}: {name}" - ) + learning.validate_targets(interface.target_columns, f"{phase.name}: {name}") for column in dataset.class_share_log_columns: - if dataset.interface.target_column_types.get(column) != "categorical": + if interface.target_column_types.get(column) != "categorical": raise ValueError( f"{phase.name}: class share columns must be categorical targets" ) - _criterion_modules(dataset, device) + _criterion_modules(dataset, interface, device) From 84bc48b25d4019142e2f4f5f8b4506b61155824a Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 11:54:49 +0100 Subject: [PATCH 51/71] mutable runtime state --- src/sequifier/config/train_config.py | 28 +- src/sequifier/evaluation/service.py | 9 +- src/sequifier/io/memory_budget.py | 6 +- src/sequifier/runtime/builder.py | 122 ++++---- src/sequifier/training/checkpoint_service.py | 4 +- src/sequifier/training/engine.py | 84 +++--- src/sequifier/training/plan.py | 96 ++++++ src/sequifier/training/runtime.py | 295 +++++++++---------- src/sequifier/training/scheduler_budgets.py | 94 +++--- 9 files changed, 412 insertions(+), 326 deletions(-) create mode 100644 src/sequifier/training/plan.py diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index e88ef908..804a1e72 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -949,6 +949,7 @@ class ResolvedSequifierConfig(ConfigModel): preference_sources: dict[str, str] = Field(default_factory=dict) policy_bindings: dict[str, ResolvedPolicyBinding] = Field(default_factory=dict) training_plan: list[ResolvedTrainingPhase | PolicyPhase] + # Retained in the serialized schema for older training checkpoints. active_phase_index: int = 0 curriculum_training: bool = False curriculum_column: Optional[str] = None @@ -2123,17 +2124,24 @@ class SelectedDatasetPartConfig(SelectedInterfaceConfig): @beartype def interface_build_view( - config: Any, interface: ResolvedModelInterface + config: Any, + interface: ResolvedModelInterface, + *, + runtime_settings: TrainingRuntimeSettings | None = None, ) -> SelectedInterfaceConfig: """Select one interface for model and objective construction.""" + settings = runtime_settings or config.runtime_settings + objective = settings.training_objective + if objective is None: + raise ValueError("Interface construction requires a training objective") return SelectedInterfaceConfig( project_root=config.project_root, model_name=config.model_name, device=config.device, seed=config.seed, - runtime_settings=config.runtime_settings, - training_objective=config.runtime_settings.training_objective, + runtime_settings=settings, + training_objective=objective, static_columns=( interface.static_conditioning.columns if interface.static_conditioning @@ -2157,6 +2165,10 @@ def dataset_part_view( config: ResolvedSequifierConfig, dataset_name: str, part_name: str, + *, + batch_size: int, + window_stride: int | None, + curriculum_column: str | None = None, ) -> SelectedDatasetPartConfig: """Select one dataset part for data loading.""" @@ -2186,17 +2198,15 @@ def dataset_part_view( ) return SelectedDatasetPartConfig( - batch_size=config.training_plan[config.active_phase_index].batch_size - if config.training_plan - else config.runtime_settings.inference_batch_size, + batch_size=batch_size, **interface_values, dataset_training=dataset, part=part, - curriculum_training=config.curriculum_training, - curriculum_column=config.curriculum_column, + curriculum_training=curriculum_column is not None, + curriculum_column=curriculum_column, epochs=sum(getattr(phase, "epochs", 0) for phase in config.training_plan), read_format=part.read_format, - window_stride=config.runtime_settings.window_stride, + window_stride=window_stride, data_path=part.training_data_path, validation_data_path=part.validation_data_path, metadata_config_path=part.metadata_config_path, diff --git a/src/sequifier/evaluation/service.py b/src/sequifier/evaluation/service.py index 7a5efadb..97c322ea 100644 --- a/src/sequifier/evaluation/service.py +++ b/src/sequifier/evaluation/service.py @@ -6,7 +6,7 @@ import time from contextlib import nullcontext from dataclasses import asdict, dataclass, field, is_dataclass -from typing import Any, Sequence +from typing import Any import torch from torch.nn.functional import one_hot @@ -15,6 +15,7 @@ from sequifier.model.network import ModelOutput from sequifier.runtime.precision import PrecisionPolicy from sequifier.training.loss import LossService, PreparedBatch +from sequifier.training.plan import SupervisedPhaseView from sequifier.training.runtime import DatasetRuntimeRegistry, build_source_runtime @@ -151,7 +152,7 @@ def _baseline_output( def evaluate( self, network: Any, - sources: Sequence[Any], + phase: SupervisedPhaseView, datasets: DatasetRuntimeRegistry, context: EvaluationContext, ) -> EvaluationResult: @@ -161,9 +162,9 @@ def evaluate( try: autocast = self.precision.autocast() if self.precision else nullcontext() with torch.no_grad(), autocast: - for source_config in sources: + for source_config in phase.spec().evaluation_sources: started = time.perf_counter() - source = build_source_runtime(source_config, datasets) + source = build_source_runtime(source_config, datasets, phase) source.set_epoch(context.phase_epoch, "validation") dataset = source.dataset targets = list(dataset.interface.target_columns) diff --git a/src/sequifier/io/memory_budget.py b/src/sequifier/io/memory_budget.py index d58a291b..b94f0340 100644 --- a/src/sequifier/io/memory_budget.py +++ b/src/sequifier/io/memory_budget.py @@ -273,8 +273,10 @@ def plan_data_loading(config: Any) -> DataLoadingBudget: stride = min(strides) if strides else None for name, dataset in config.dataset_training.items(): for part_name, part in dataset.parts.items(): - selected = replace( - dataset_part_view(config, name, part_name), + selected = dataset_part_view( + config, + name, + part_name, batch_size=batch, window_stride=stride, ) diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 5036e4b3..ee49af2c 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -37,25 +37,29 @@ from sequifier.training.loss import LossService from sequifier.training.metrics_service import MetricsService from sequifier.training.optimization import GradientPolicy, OptimizationRuntime +from sequifier.training.plan import SupervisedPhaseView, SupervisedRunPlan from sequifier.training.runtime import ( + DatasetMetrics, DatasetRuntimeRegistry, PhaseTrainabilityPlan, + build_dataset_parts, build_dataset_runtimes, - configure_phase, validate_phase_learning, ) -from sequifier.training.scheduler_budgets import validate_supervised_scheduler_budgets +from sequifier.training.scheduler_budgets import resolve_scheduler_budgets from sequifier.training.state import RunState @dataclass class TrainingRun: config: Any + plan: SupervisedRunPlan network: Any callable_network: Any datasets: DatasetRuntimeRegistry optimization: OptimizationRuntime - optimization_factory: Callable[[int], OptimizationRuntime] + dataset_factory: Callable[[SupervisedPhaseView], DatasetRuntimeRegistry] + optimization_factory: Callable[[SupervisedPhaseView], OptimizationRuntime] optimization_phase_index: int state: RunState distributed: DistributedStrategy @@ -70,28 +74,21 @@ class TrainingRun: context: RunContext precision: PrecisionPolicy - def start_phase(self, phase_index: int) -> None: + def start_phase( + self, view: SupervisedPhaseView, datasets: DatasetRuntimeRegistry + ) -> None: """Enter the phase with its resolved reset or continuation policy.""" - - if self.config.active_phase_index != phase_index: - configure_phase( - self.config, - self.network, - self.datasets, - phase_index, - self.distributed.device, - ) + phase = view.spec() optimizer_step = self.optimization.optimizer_step - if self.config.training_plan[phase_index].optimizer_state == "reset": - self.optimization = self.optimization_factory(phase_index) + if phase.optimizer_state == "reset": + self.optimization = self.optimization_factory(view) self.optimization.gradient_policy = GradientPolicy( - self.config.training_plan[phase_index].accumulation_batches, - self.config.training_plan[phase_index] - .optimization["model"] - .gradient_clip_norm, + phase.accumulation_batches, + phase.optimization["model"].gradient_clip_norm, ) self.optimization.optimizer_step = optimizer_step - self.optimization_phase_index = phase_index + self.optimization_phase_index = view.index + self.datasets = datasets class RunBuilder: @@ -158,11 +155,12 @@ def _warm_up(self, run: TrainingRun) -> None: def _construct_loaders(self, run: TrainingRun) -> None: """Construct every configured loader before final RNG restoration.""" + phase = run.plan.phases[run.state.phase_index] for dataset in run.datasets.values(): for part_name, part in dataset.parts.items(): - part.loader("training") + part.loader(phase, "training") if dataset.config.parts[part_name].validation_data_path is not None: - part.loader("validation") + part.loader(phase, "validation") def build( self, @@ -183,6 +181,7 @@ def build( validate_training_destinations, ) + config = config.model_copy() resolve_training_output_paths(config) protected = supervised_input_paths(config) if initial_artifact is not None: @@ -210,10 +209,11 @@ def build( if loaded is not None: restorer.validate_for_restore(loaded, config) # Optimizer state uses the parameter order of the saved network. - config.interfaces = { + interfaces = { name: config.interfaces[name] for name in loaded.checkpoint.model.model_config.values["interfaces"] } + config = config.model_copy(update={"interfaces": interfaces}) state = RunState.from_state_dict(loaded.checkpoint.run_state) else: state = RunState() @@ -225,7 +225,8 @@ def build( raise RuntimeError("Rank zero did not provide a run identity") state.run_id, state.session_id = identity.run_id, identity.session_id - validate_phase_learning(config, execution.device) + plan = SupervisedRunPlan.from_config(config) + validate_phase_learning(config, plan, execution.device) from sequifier.export.preflight import ensure_export_preflight @@ -256,21 +257,31 @@ def build( load_revision(network.backbone, revision) state.backbone_parent_revision_id = revision["revision_id"] - freezing_plan = PhaseTrainabilityPlan.resolve( - config, network, state.phase_index - ) - previous_masks = None + freezing_plans = [] + planned_phases = [] permanent_masks = [] - for phase_index, phase in enumerate(config.training_plan): - masks = PhaseTrainabilityPlan.resolve( - config, network, phase_index - ).parameter_ids_by_dataset - if phase.optimizer_state == "continue" and masks != previous_masks: + for view in plan.phases: + phase = view.spec() + freezing_plan = PhaseTrainabilityPlan.resolve(config, network, phase) + masks = freezing_plan.parameter_ids_by_dataset + if phase.optimizer_state == "continue" and ( + not freezing_plans or freezing_plan != freezing_plans[-1] + ): raise ValueError( "optimizer continuation requires identical resolved parameter masks" ) - previous_masks = masks + freezing_plans.append(freezing_plan) permanent_masks.extend(set(selected) for selected in masks.values()) + summary = { + name: sorted(masks[name]) + for name in dict.fromkeys(source.dataset for source in phase.sources) + } + planned_phases.append( + view.with_spec( + phase.model_copy(update={"trainability_summary": summary}) + ) + ) + plan = SupervisedRunPlan(tuple(planned_phases)) # Compatibility: avoid building backward graphs for parameters that # every dataset in every phase leaves frozen. if permanent_masks: @@ -297,29 +308,29 @@ def build( if loaded is not None: restorer.restore_model(loaded, network, strategy) - datasets = build_dataset_runtimes( - config, - network, - execution.device, - objectives=built.objectives, - runtime_metadata=built.runtime_metadata, - freezing_plan=freezing_plan, - ) - for dataset in datasets.values(): - for part in dataset.parts.values(): - part.factory.memory_budget = memory_budget - configure_phase(config, network, datasets, state.phase_index, execution.device) - for dataset in datasets.values(): - for part in dataset.parts.values(): - part.factory.batch_size = config.training_plan[ - state.phase_index - ].batch_size + parts = build_dataset_parts(config, plan, memory_budget) + dataset_metrics = {name: DatasetMetrics() for name in parts} + plan = resolve_scheduler_budgets(plan, config, parts) + active_phase = plan.phases[state.phase_index] + + def dataset_factory(view: SupervisedPhaseView) -> DatasetRuntimeRegistry: + return build_dataset_runtimes( + config, + view, + execution.device, + parts=parts, + metrics=dataset_metrics, + runtime_metadata=built.runtime_metadata, + default_objectives=built.objectives, + masks=freezing_plans[view.index].bind(network), + ) + datasets = dataset_factory(active_phase) parameters: Any = tuple(strategy.prepare_optimizer_parameters(network)) catalog = ParameterCatalog(network) - def optimization_factory(phase_index: int) -> OptimizationRuntime: - phase = config.training_plan[phase_index] + def optimization_factory(view: SupervisedPhaseView) -> OptimizationRuntime: + phase = view.spec() role = phase.optimization["model"] selected_parameters = parameters if self.semantic_optimizer_grouping and not isinstance( @@ -338,8 +349,7 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: autocast_dtype=precision.autocast_dtype, ) - validate_supervised_scheduler_budgets(config, datasets, state.phase_index) - optimization = optimization_factory(state.phase_index) + optimization = optimization_factory(active_phase) if loaded is not None: restorer.restore_optimization(loaded, optimization, network, strategy) @@ -367,9 +377,11 @@ def optimization_factory(phase_index: int) -> OptimizationRuntime: loss_service = LossService() run = TrainingRun( config=config, + plan=plan, network=network, callable_network=callable_network, datasets=datasets, + dataset_factory=dataset_factory, optimization=optimization, optimization_factory=optimization_factory, optimization_phase_index=state.phase_index, diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index 7eb1083a..5029c10c 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -220,7 +220,9 @@ def save(self, request: CheckpointRequest, run: Any) -> Path | None: random_state=random_state, loader_state=loader_state, integration_state=IntegrationState(values=integration_state), - training_config=run.config, + training_config=run.config.model_copy( + update={"training_plan": [phase.spec() for phase in run.plan.phases]} + ), ) self.compatibility.validate(checkpoint) written = self.store.save(checkpoint, request.suffix) diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index c9525407..b6717bc8 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -45,7 +45,12 @@ from sequifier.training.checkpoint_service import CheckpointRequest from sequifier.training.metrics_service import TrainingMetrics from sequifier.training.optimization import OptimizationBoundaryState, UpdatePolicy -from sequifier.training.runtime import SourceScheduler, build_source_runtime +from sequifier.training.plan import SupervisedPhaseView +from sequifier.training.runtime import ( + DatasetRuntimeRegistry, + SourceScheduler, + build_source_runtime, +) from sequifier.training.state import RunStateSnapshot @@ -133,13 +138,14 @@ def _evaluation_context( kind=kind, ) - def _evaluate(self, run: Any, context: EvaluationContext) -> EvaluationResult: - result = run.evaluation.evaluate( - run.network, - run.config.training_plan[context.phase_index].evaluation_sources, - run.datasets, - context, - ) + def _evaluate( + self, + run: Any, + phase: SupervisedPhaseView, + datasets: DatasetRuntimeRegistry, + context: EvaluationContext, + ) -> EvaluationResult: + result = run.evaluation.evaluate(run.network, phase, datasets, context) run.metrics.record_evaluation( result, context=context, @@ -305,7 +311,7 @@ def run(self, run: Any) -> RunResult: def run_phase(self, run: Any) -> dict: """Run one prepared supervised phase for a portable handoff.""" - if len(run.config.training_plan) != 1: + if len(run.plan.phases) != 1: raise ValueError("a portable supervised phase requires one resolved phase") result = self._run(run, phase_only=True) return { @@ -316,11 +322,12 @@ def run_phase(self, run: Any) -> dict: def _run(self, run: Any, *, phase_only: bool) -> RunResult: config = run.config state = run.state + phases = run.plan.phases time_checkpoint_intervals_enabled = bool( config.runtime_settings.latest_checkpoint_interval_minutes is not None or any( - p.checkpoints.snapshot_interval_minutes is not None - for p in config.training_plan + view.spec().checkpoints.snapshot_interval_minutes is not None + for view in phases ) ) defer_ddp_gradient_sync = bool( @@ -417,15 +424,16 @@ def flush_metrics( current_batch = sum(state.iterator_positions.values()) current_batches_total = 1 - def run_phase(phase_index, phase): + def run_phase(phase_index, view): nonlocal access, active_dataset, accumulation_count, last_identity nonlocal boundary, boundary_identity, batch_in_progress, batch_committed nonlocal last_snapshot, last_snapshot_step, last_latest nonlocal current_epoch, current_batch, current_batches_total nonlocal stop_requested, completion_reason + phase = view.spec() run.metrics.set_phase(phase) if phase_index != state.phase_index: - previous = config.training_plan[phase_index - 1] + previous = phases[phase_index - 1].spec() if previous.handoff == "best": selected = run.distributed.gather_objects( state.best_model_state_dict @@ -448,16 +456,11 @@ def run_phase(phase_index, phase): state.last_snapshot_batch_step = last_snapshot_step phase_stopping = StoppingState(**state.stopping_state) phase_stop = phase_stopping.triggered - if run.optimization_phase_index != phase_index: - from sequifier.training.runtime import configure_phase - - configure_phase( - config, - run.network, - run.datasets, - phase_index, - run.distributed.device, - ) + datasets = ( + run.datasets + if run.optimization_phase_index == phase_index + else run.dataset_factory(view) + ) if ( phase.evaluation and phase.evaluation.before_phase @@ -465,6 +468,8 @@ def run_phase(phase_index, phase): ): self._evaluate( run, + view, + datasets, self._evaluation_context( run, kind="initial", @@ -476,20 +481,10 @@ def run_phase(phase_index, phase): ), ) state.initial_evaluated_phases.append(phase_index) - for dataset in run.datasets.values(): - for part in dataset.parts.values(): - if part.factory.batch_size != phase.batch_size: - part.factory.loaders.clear() - part.factory.batch_size = phase.batch_size sources = [ - build_source_runtime(source, run.datasets) for source in phase.sources + build_source_runtime(source, datasets, view) for source in phase.sources ] - scheduler = SourceScheduler( - phase, - sources, - seed=config.seed + phase_index, - phase_index=phase_index, - ) + scheduler = SourceScheduler(view, sources) if phase_index == state.phase_index and state.source_scheduler_state: scheduler.load_state_dict(state.source_scheduler_state) start_epoch = ( @@ -509,7 +504,7 @@ def run_phase(phase_index, phase): state.phase_epoch = phase_epoch state.phase_epoch_complete = False if run.optimization_phase_index != phase_index: - run.start_phase(phase_index) + run.start_phase(view, datasets) state.optimizer_step = run.optimization.optimizer_step state.epoch = current_epoch state.batch = 0 @@ -708,6 +703,8 @@ def run_phase(phase_index, phase): ): self._evaluate( run, + view, + datasets, self._evaluation_context( run, kind="interval", @@ -744,6 +741,8 @@ def run_phase(phase_index, phase): evaluation = ( self._evaluate( run, + view, + datasets, self._evaluation_context( run, kind="epoch_end", @@ -806,13 +805,14 @@ def run_phase(phase_index, phase): completion_reason if stop_requested else None, ) - def commit_phase(index, phase, outcome): - self._publish_phase(run, index, phase) + def commit_phase(index, view, outcome): + self._publish_phase(run, index, view.spec()) run.checkpoints.save(CheckpointRequest("latest", last_identity), run) try: start = state.phase_index - phase = config.training_plan[start] + view = phases[start] + phase = view.spec() if state.phase_epoch_complete and ( state.plan_stopped or state.phase_epoch >= phase.epochs @@ -823,10 +823,10 @@ def commit_phase(index, phase, outcome): self._publish_phase(run, start, phase) start += 1 if phase_only and start == 0: - execute_phase(0, phase, run_phase, commit_phase) + execute_phase(0, view, run_phase, commit_phase) elif not phase_only: plan_reason = run_phases( - config.training_plan, + phases, run_phase, commit_phase, start=start, @@ -884,7 +884,7 @@ def commit_phase(index, phase, outcome): ) if run.distributed.rank == 0: selected = select_handoff( - config.training_plan[state.phase_index].handoff, last_state, best_state + phases[state.phase_index].spec().handoff, last_state, best_state ) save_model_artifact( build_model_artifact(run.network, config, state_dict=selected), diff --git a/src/sequifier/training/plan.py b/src/sequifier/training/plan.py new file mode 100644 index 00000000..d8114794 --- /dev/null +++ b/src/sequifier/training/plan.py @@ -0,0 +1,96 @@ +"""Read-only phase views for native supervised execution.""" + +from dataclasses import dataclass +from typing import Literal + +from sequifier.config.train_config import ( + ResolvedDatasetTrainingSpec, + ResolvedSequifierConfig, + ResolvedTrainingPhase, + TrainingRuntimeSettings, +) + + +@dataclass(frozen=True) +class PhaseLoading: + training_batch_size: int + validation_batch_size: int + window_stride: int | None + curriculum_column: str | None + + def batch_size(self, split: Literal["training", "validation"]) -> int: + return ( + self.training_batch_size + if split == "training" + else self.validation_batch_size + ) + + +@dataclass(frozen=True) +class SupervisedPhaseView: + index: int + seed: int + loading: PhaseLoading + _spec: ResolvedTrainingPhase + + @classmethod + def from_spec( + cls, index: int, seed: int, spec: ResolvedTrainingPhase + ) -> "SupervisedPhaseView": + from sequifier.evaluation.compatibility import evaluation_batch_size + + return cls( + index=index, + seed=seed, + loading=PhaseLoading( + spec.batch_size, + evaluation_batch_size(spec), + spec.learning.window_stride, + spec.curriculum.column if spec.curriculum else None, + ), + _spec=spec.model_copy(deep=True), + ) + + def spec(self) -> ResolvedTrainingPhase: + """Give consumers their own copy of the configuration models.""" + return self._spec.model_copy(deep=True) + + def with_spec(self, spec: ResolvedTrainingPhase) -> "SupervisedPhaseView": + return self.from_spec(self.index, self.seed, spec) + + def objective_settings( + self, settings: TrainingRuntimeSettings + ) -> TrainingRuntimeSettings: + learning = self._spec.learning + return settings.model_copy( + update={ + "bert_spec": learning.bert_spec, + "next_occurrence_config": learning.next_occurrence_config, + "window_stride": self.loading.window_stride, + } + ).model_copy(deep=True) + + +@dataclass(frozen=True) +class SupervisedRunPlan: + phases: tuple[SupervisedPhaseView, ...] + + @classmethod + def from_config(cls, config: ResolvedSequifierConfig) -> "SupervisedRunPlan": + phases = [] + for index, spec in enumerate(config.training_plan): + if not isinstance(spec, ResolvedTrainingPhase): + raise ValueError( + "Native supervised training requires supervised phases" + ) + phases.append( + SupervisedPhaseView.from_spec(index, config.seed + index, spec) + ) + return cls(tuple(phases)) + + +def dataset_learning_view( + dataset: ResolvedDatasetTrainingSpec, phase: ResolvedTrainingPhase +) -> ResolvedDatasetTrainingSpec: + learning = phase.learning.datasets.get(dataset.name) + return dataset.model_copy(update=learning.model_dump()) if learning else dataset diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 6d88da6c..1d4d86e1 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -19,10 +19,11 @@ ResolvedTrainingPhase, ResolvedTrainingSource, dataset_part_view, + interface_build_view, ) from sequifier.helpers import normalize_path from sequifier.io.batch import SequifierBatch -from sequifier.io.memory_budget import DataLoadingBudget, plan_data_loading +from sequifier.io.memory_budget import DataLoadingBudget from sequifier.io.sequifier_dataset_from_file import SequifierDatasetFromFile from sequifier.io.sequifier_dataset_from_folder_parquet import ( SequifierDatasetFromFolderParquet, @@ -36,8 +37,14 @@ ) from sequifier.model.network import ComposableTransformerNetwork from sequifier.model.parameter_catalog import ParameterCatalog -from sequifier.objectives import Objective +from sequifier.objectives import Objective, create_objective from sequifier.training.accurate_cross_entropy import AccurateCrossEntropyLoss +from sequifier.training.plan import ( + PhaseLoading, + SupervisedPhaseView, + SupervisedRunPlan, + dataset_learning_view, +) from sequifier.typechecking import beartype @@ -48,26 +55,14 @@ class RuntimeBatch: batch: SequifierBatch -def _training_plan_uses_part( - config: ResolvedSequifierConfig, dataset_name: str, part_name: str -) -> bool: - """Return whether any training phase selects this dataset part.""" - return any( - source.dataset == dataset_name - and (source.part is None or source.part == part_name) - for phase in config.training_plan - if isinstance(phase, ResolvedTrainingPhase) - for source in phase.sources - ) - - @dataclass class PartLoaderFactory: config: ResolvedSequifierConfig dataset_name: str part_name: str - batch_size: int = 1 - memory_budget: DataLoadingBudget | None = None + memory_budget: DataLoadingBudget + curriculum_eligible: bool + _loading: PhaseLoading | None = field(default=None, init=False) loaders: dict[Literal["training", "validation"], DataLoader] = field( default_factory=dict, init=False, @@ -77,32 +72,24 @@ class PartLoaderFactory: init=False, ) - @beartype - def build(self, split: Literal["training", "validation"]) -> DataLoader: + def _build_dataset( + self, loading: PhaseLoading, split: Literal["training", "validation"] + ): global_spec = self.config.runtime_settings - if split in self.loaders: - return self.loaders[split] - if self.memory_budget is None: - self.memory_budget = plan_data_loading(self.config) - - phase = self.config.training_plan[self.config.active_phase_index] - from sequifier.evaluation.compatibility import evaluation_batch_size - - dataset_config: Any = replace( - dataset_part_view(self.config, self.dataset_name, self.part_name), - batch_size=self.batch_size - if split == "training" - else evaluation_batch_size(phase), + curriculum_column = ( + loading.curriculum_column + if split == "training" and self.curriculum_eligible + else None ) - if split != "training" or not _training_plan_uses_part( - self.config, self.dataset_name, self.part_name - ): - dataset_config = replace( - dataset_config, - curriculum_training=False, - curriculum_column=None, - ) - part = self.config.dataset_training[self.dataset_name].parts[self.part_name] + dataset_config = dataset_part_view( + self.config, + self.dataset_name, + self.part_name, + batch_size=loading.batch_size(split), + window_stride=loading.window_stride, + curriculum_column=curriculum_column, + ) + part = dataset_config.part path = ( part.training_data_path if split == "training" @@ -153,6 +140,23 @@ def build(self, split: Literal["training", "validation"]) -> DataLoader: dataset, global_spec.num_workers, ) + return dataset + + def count_training_batches(self, phase: SupervisedPhaseView) -> int: + # The dataset emits pre-batched items and the loader uses batch_size=None. + return len(self._build_dataset(phase.loading, "training")) + + @beartype + def build( + self, phase: SupervisedPhaseView, split: Literal["training", "validation"] + ) -> DataLoader: + if self._loading != phase.loading: + self.loaders.clear() + self._loading = phase.loading + if split in self.loaders: + return self.loaders[split] + dataset = self._build_dataset(phase.loading, split) + global_spec = self.config.runtime_settings generator = self.generators.get(split) if generator is None: generator = torch.Generator().manual_seed( @@ -207,11 +211,15 @@ def source_metadata(self) -> dict[str, Any]: "validation_data_path": part.validation_data_path, } - def loader(self, split: Literal["training", "validation"]) -> DataLoader: - return self.factory.build(split) + def loader( + self, phase: SupervisedPhaseView, split: Literal["training", "validation"] + ) -> DataLoader: + return self.factory.build(phase, split) - def iter_loader(self, split: Literal["training", "validation"]) -> Iterator[Any]: - loader = self.loader(split) + def iter_loader( + self, phase: SupervisedPhaseView, split: Literal["training", "validation"] + ) -> Iterator[Any]: + loader = self.loader(phase, split) generator = self.factory.generators[split] if split in self._restore_from_iterator_start: generator.set_state(self.iterator_start_states[split]) @@ -314,7 +322,7 @@ def resolve( cls, config: ResolvedSequifierConfig, network: ComposableTransformerNetwork, - phase_index: int = 0, + phase: ResolvedTrainingPhase, ) -> PhaseTrainabilityPlan: catalog = ParameterCatalog(network) parameter_id_by_identity = { @@ -322,11 +330,6 @@ def resolve( for descriptor in catalog.descriptors() } parameter_ids_by_dataset = {} - phase = config.training_plan[phase_index] - if not isinstance(phase, ResolvedTrainingPhase): - raise ValueError( - "native supervised trainability requires a resolved supervised phase" - ) active_datasets = {ref.dataset for ref in phase.sources} phase_parameters = {id(p) for p in network.backbone.parameters()} for name in active_datasets: @@ -371,11 +374,6 @@ def resolve( parameter_ids_by_dataset[name] = frozenset( parameter_id_by_identity[identity] for identity in frozen ) - phase.trainability_summary = { - name: sorted(ids) - for name, ids in parameter_ids_by_dataset.items() - if name in active_datasets - } return cls(parameter_ids_by_dataset) @beartype @@ -394,19 +392,18 @@ class TrainingSourceRuntime: config: ResolvedTrainingSource dataset: DatasetRuntime part_names: tuple[str, ...] - loaders: dict[str, list[tuple[str, DataLoader]]] = field(default_factory=dict) + phase: SupervisedPhaseView + + @property + def dataset_config(self) -> ResolvedDatasetTrainingSpec: + return self.dataset.config @beartype def _loaders(self, split: Literal["training", "validation"]): - if split not in self.loaders: - self.loaders[split] = [ - ( - part_name, - self.dataset.parts[part_name].loader(split), - ) - for part_name in self.part_names - ] - return self.loaders[split] + return [ + (part_name, self.dataset.parts[part_name].loader(self.phase, split)) + for part_name in self.part_names + ] @beartype def set_epoch(self, epoch: int, split: Literal["training", "validation"]): @@ -420,7 +417,7 @@ def iter_batches( self, split: Literal["training", "validation"] = "training" ) -> Iterator[RuntimeBatch]: for part_name, _ in self._loaders(split): - for batch in self.dataset.parts[part_name].iter_loader(split): + for batch in self.dataset.parts[part_name].iter_loader(self.phase, split): if not isinstance(batch, SequifierBatch): raise TypeError( "Sequifier loaders must yield SequifierBatch, got " @@ -438,7 +435,9 @@ class ScheduledSource(Protocol): """Structural source contract consumed by the deterministic scheduler.""" config: ResolvedTrainingSource - dataset: DatasetRuntime + + @property + def dataset_config(self) -> ResolvedDatasetTrainingSpec: ... @beartype def set_epoch( @@ -493,63 +492,87 @@ def _criterion_modules( return modules -@beartype +def build_dataset_parts( + config: ResolvedSequifierConfig, + plan: SupervisedRunPlan, + memory_budget: DataLoadingBudget, +) -> dict[str, dict[str, DatasetPartRuntime]]: + training_parts = { + (source.dataset, part) + for phase in plan.phases + for source in phase.spec().sources + for part in source_part_names(source, config.dataset_training[source.dataset]) + } + return { + name: { + part: DatasetPartRuntime( + part, + PartLoaderFactory( + config, name, part, memory_budget, (name, part) in training_parts + ), + ) + for part in dataset.parts + } + for name, dataset in config.dataset_training.items() + } + + def build_dataset_runtimes( config: ResolvedSequifierConfig, - network: ComposableTransformerNetwork, + phase: SupervisedPhaseView, device: torch.device, *, - objectives: dict[str, Objective], + parts: dict[str, dict[str, DatasetPartRuntime]], + metrics: dict[str, DatasetMetrics], runtime_metadata: Any, - freezing_plan: PhaseTrainabilityPlan | None = None, + default_objectives: dict[str, Objective], + masks: dict[str, frozenset[int]], ) -> DatasetRuntimeRegistry: - if freezing_plan is None: - freezing_plan = PhaseTrainabilityPlan.resolve(config, network) - frozen_parameter_ids_by_dataset = freezing_plan.bind(network) - + spec = phase.spec() + settings = phase.objective_settings(config.runtime_settings) runtimes = {} - for name, dataset in config.dataset_training.items(): + for name, base in config.dataset_training.items(): + dataset = dataset_learning_view(base, spec) interface = config.interface(dataset.model_interface) - frozen = frozen_parameter_ids_by_dataset[name] - route = network.interfaces[dataset.model_interface] - active_parameter_ids = { - id(parameter) - for module in (network.backbone, route) - for parameter in module.parameters() - } - if frozen >= active_parameter_ids: - raise ValueError(f"Dataset {name!r} leaves no trainable parameters") + objective = ( + create_objective( + interface_build_view(config, interface, runtime_settings=settings) + ) + if name in spec.learning.datasets + else default_objectives[dataset.model_interface] + ) runtimes[name] = DatasetRuntime( name=name, config=dataset, interface_name=dataset.model_interface, interface=interface, - objective=objectives[dataset.model_interface], - parts={ - part_name: DatasetPartRuntime( - part_name, PartLoaderFactory(config, name, part_name) - ) - for part_name in dataset.parts - }, + objective=objective, + parts=parts[name], criteria=_criterion_modules(dataset, interface, device), loss_weights=dataset.loss_weights, class_weights=dataset.class_weights, - frozen_parameter_ids=frozen, + frozen_parameter_ids=masks[name], runtime_metadata=runtime_metadata.interfaces[dataset.model_interface], + metrics=metrics[name], ) return DatasetRuntimeRegistry(runtimes) -@beartype +def source_part_names( + source: ResolvedTrainingSource, dataset: ResolvedDatasetTrainingSpec +) -> tuple[str, ...]: + return (source.part,) if source.part is not None else tuple(dataset.parts) + + def build_source_runtime( source: ResolvedTrainingSource, datasets: DatasetRuntimeRegistry, + phase: SupervisedPhaseView, ) -> TrainingSourceRuntime: dataset = datasets.resolve(source.dataset) - part_names = ( - (source.part,) if source.part is not None else tuple(dataset.config.parts) + return TrainingSourceRuntime( + source, dataset, source_part_names(source, dataset.config), phase ) - return TrainingSourceRuntime(source, dataset, part_names) class SourceScheduler: @@ -558,16 +581,13 @@ class SourceScheduler: @beartype def __init__( self, - phase: ResolvedTrainingPhase, + phase: SupervisedPhaseView, sources: Sequence[ScheduledSource], - *, - seed: int, - phase_index: int = 0, ) -> None: - self.phase = phase + self.phase = phase.spec() self.sources = sources - self.phase_index = phase_index - self.rng = random.Random(seed) + self.phase_index = phase.index + self.rng = random.Random(phase.seed) self.round_robin_cursor = 0 self.epoch = 0 self.consumed_by_source = [0 for _ in sources] @@ -611,7 +631,7 @@ def _choose(self, active: list[int]) -> int: return self.rng.choices( active, weights=[ - self.phase.sources[index].selection_weight for index in active + self.sources[index].config.selection_weight for index in active ], k=1, )[0] @@ -660,7 +680,7 @@ def iter_epoch(self, epoch: int) -> Iterator[RuntimeBatch]: dataset_indices = { name: index for index, name in enumerate( - dict.fromkeys(source.dataset.config.name for source in self.sources) + dict.fromkeys(source.dataset_config.name for source in self.sources) ) } while active: @@ -674,12 +694,12 @@ def iter_epoch(self, epoch: int) -> Iterator[RuntimeBatch]: part_index = ( -1 if source.config.part is None - else list(source.dataset.config.parts).index(source.config.part) + else list(source.dataset_config.parts).index(source.config.part) ) record = ( self.phase_index, selected, - dataset_indices[source.dataset.config.name], + dataset_indices[source.dataset_config.name], part_index, source.config.batches_per_selection, ) @@ -701,13 +721,13 @@ def iter_epoch(self, epoch: int) -> Iterator[RuntimeBatch]: expected_part = ( -1 if selected_source.config.part is None - else list(selected_source.dataset.config.parts).index( + else list(selected_source.dataset_config.parts).index( selected_source.config.part ) ) if ( record[2] - != dataset_indices[selected_source.dataset.config.name] + != dataset_indices[selected_source.dataset_config.name] or record[3] != expected_part ): raise RuntimeError( @@ -733,58 +753,13 @@ def iter_epoch(self, epoch: int) -> Iterator[RuntimeBatch]: self.pending_source = None -def configure_phase(config, network, datasets, phase_index, device): - """Apply phase learning intent without replacing loader recovery identities.""" - from sequifier.config.train_config import interface_build_view - from sequifier.evaluation.compatibility import evaluation_batch_size - from sequifier.objectives import create_objective - - phase = config.training_plan[phase_index] - previous = config.training_plan[config.active_phase_index] - rebuild_loaders = ( - previous.batch_size != phase.batch_size - or evaluation_batch_size(previous) != evaluation_batch_size(phase) - or previous.learning.window_stride != phase.learning.window_stride - or previous.curriculum != phase.curriculum - ) - config.active_phase_index = phase_index - config.runtime_settings.bert_spec = phase.learning.bert_spec - config.runtime_settings.next_occurrence_config = ( - phase.learning.next_occurrence_config - ) - config.runtime_settings.window_stride = phase.learning.window_stride - config.curriculum_training = phase.curriculum is not None - config.curriculum_column = phase.curriculum.column if phase.curriculum else None - masks = PhaseTrainabilityPlan.resolve(config, network, phase_index).bind(network) - for name, dataset in datasets.items(): - learning = phase.learning.datasets.get(name) - if learning is not None: - for field, value in learning.model_dump().items(): - setattr(dataset.config, field, value) - dataset.criteria = _criterion_modules( - dataset.config, dataset.interface, device - ) - dataset.class_weights = learning.class_weights - dataset.loss_weights = learning.loss_weights - dataset.frozen_parameter_ids = masks[name] - if name in phase.learning.datasets: - dataset.objective = create_objective( - interface_build_view(config, dataset.interface) - ) - for part in dataset.parts.values(): - if rebuild_loaders: - part.factory.loaders.clear() - part.factory.batch_size = phase.batch_size - - -def validate_phase_learning(config, device): +def validate_phase_learning(config, plan: SupervisedRunPlan, device): """Validate every phase's losses before native outputs are created.""" - for phase in config.training_plan: + for view in plan.phases: + phase = view.spec() for name, learning in phase.learning.datasets.items(): - dataset = config.dataset_training[name].model_copy(deep=True) + dataset = dataset_learning_view(config.dataset_training[name], phase) interface = config.interface(dataset.model_interface) - for key, value in learning.model_dump().items(): - setattr(dataset, key, value) learning.validate_targets(interface.target_columns, f"{phase.name}: {name}") for column in dataset.class_share_log_columns: if interface.target_column_types.get(column) != "categorical": diff --git a/src/sequifier/training/scheduler_budgets.py b/src/sequifier/training/scheduler_budgets.py index fedaa82a..50299196 100644 --- a/src/sequifier/training/scheduler_budgets.py +++ b/src/sequifier/training/scheduler_budgets.py @@ -4,14 +4,13 @@ import torch -from sequifier.config.train_config import ResolvedTrainingSource -from sequifier.io.batch import SequifierBatch -from sequifier.training.runtime import ( - DatasetRuntime, - RuntimeBatch, - SourceScheduler, - build_source_runtime, +from sequifier.config.train_config import ( + ResolvedDatasetTrainingSpec, + ResolvedTrainingSource, ) +from sequifier.io.batch import SequifierBatch +from sequifier.training.plan import SupervisedRunPlan +from sequifier.training.runtime import RuntimeBatch, SourceScheduler, source_part_names @dataclass @@ -19,7 +18,7 @@ class _CountedSource: """Supply batch identities to the scheduler without consuming training data.""" config: ResolvedTrainingSource - dataset: DatasetRuntime + dataset_config: ResolvedDatasetTrainingSpec batches: int def set_epoch(self, epoch, split): @@ -27,7 +26,7 @@ def set_epoch(self, epoch, split): def iter_batches(self, split="training"): batch = RuntimeBatch( - self.dataset.name, + self.dataset_config.name, self.config.part or "", SequifierBatch( {}, @@ -42,24 +41,22 @@ def iter_batches(self, split="training"): yield batch -def _phase_update_budget(config, datasets, phase_index): - phase = config.training_plan[phase_index] - config.active_phase_index = phase_index - config.runtime_settings.window_stride = phase.learning.window_stride - config.curriculum_training = phase.curriculum is not None - config.curriculum_column = phase.curriculum.column if phase.curriculum else None - for dataset in datasets.values(): - for part in dataset.parts.values(): - part.factory.loaders.clear() - part.factory.batch_size = phase.batch_size - sources = [build_source_runtime(source, datasets) for source in phase.sources] - counted = [ - _CountedSource(source.config, source.dataset, source.num_batches()) - for source in sources - ] - scheduler = SourceScheduler( - phase, counted, seed=config.seed + phase_index, phase_index=phase_index - ) +def _phase_update_budget(config, parts, view): + phase = view.spec() + counts = {} + sources = [] + for source in phase.sources: + dataset = config.dataset_training[source.dataset] + batches = 0 + for part in source_part_names(source, dataset): + key = (source.dataset, part) + if key not in counts: + counts[key] = parts[source.dataset][ + part + ].factory.count_training_batches(view) + batches += counts[key] + sources.append(_CountedSource(source, dataset, batches)) + scheduler = SourceScheduler(view, sources) updates = 0 for epoch in range(1, phase.epochs + 1): active_dataset, accumulated = None, 0 @@ -76,10 +73,12 @@ def _phase_update_budget(config, datasets, phase_index): return updates -def validate_supervised_scheduler_budgets(config, datasets, active_phase_index): - """Validate maximum applied-update budgets, including dataset-boundary flushes.""" +def resolve_scheduler_budgets(plan, config, parts): + """Resolve applied-update budgets, including dataset-boundary flushes.""" + views = list(plan.phases) + phases = [view.spec() for view in views] phase_budgets = {} - for index, phase in enumerate(config.training_plan): + for index, phase in enumerate(phases): role = phase.optimization["model"] if ( role.scheduler.name != "OneCycleLR" @@ -88,40 +87,29 @@ def validate_supervised_scheduler_budgets(config, datasets, active_phase_index): ): continue start = index - while start > 0 and config.training_plan[start].optimizer_state == "continue": + while start > 0 and phases[start].optimizer_state == "continue": start -= 1 end = index + 1 - while ( - end < len(config.training_plan) - and config.training_plan[end].optimizer_state == "continue" - ): + while end < len(phases) and phases[end].optimizer_state == "continue": end += 1 for selected in range(start, end): if selected not in phase_budgets: phase_budgets[selected] = _phase_update_budget( - config, datasets, selected + config, parts, views[selected] ) budget = sum(phase_budgets[selected] for selected in range(start, end)) if budget <= 0: raise ValueError( f"{phase.name}: OneCycleLR requires a positive optimizer-update budget" ) - role.scheduler = role.scheduler.model_copy( + scheduler = role.scheduler.model_copy( update={"arguments": {**role.scheduler.arguments, "total_steps": budget}} ) - phase = config.training_plan[active_phase_index] - config.active_phase_index = active_phase_index - config.runtime_settings.window_stride = phase.learning.window_stride - config.curriculum_training = phase.curriculum is not None - config.curriculum_column = phase.curriculum.column if phase.curriculum else None - for dataset in datasets.values(): - for part in dataset.parts.values(): - part.factory.loaders.clear() - if ( - part.factory.batch_size - != config.training_plan[active_phase_index].batch_size - ): - part.factory.batch_size = config.training_plan[ - active_phase_index - ].batch_size - part.factory.loaders.clear() + optimization = { + **phase.optimization, + "model": role.model_copy(update={"scheduler": scheduler}), + } + views[index] = views[index].with_spec( + phase.model_copy(update={"optimization": optimization}) + ) + return SupervisedRunPlan(tuple(views)) From 49204a09123ee18046907d215c9077fe64ce9dc7 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 12:09:00 +0100 Subject: [PATCH 52/71] remove unnecessary heads in support networks --- src/sequifier/model/factory.py | 92 +++++++++++++------- src/sequifier/model/ingestions.py | 14 ++- src/sequifier/model/network.py | 7 +- src/sequifier/rl/iql.py | 3 +- src/sequifier/rl/models.py | 103 +++++++++++----------- src/sequifier/rl/ppo.py | 3 +- src/sequifier/rl/preparation.py | 15 ++-- src/sequifier/rl/run_state.py | 136 ++++++++++++++++++++++-------- 8 files changed, 232 insertions(+), 141 deletions(-) diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index eb7fef44..d9a78ae4 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -258,11 +258,17 @@ def _build_composable_network( initialize: bool, restored_history_interface: str | None, logger: Any, + encoder_interface: str | None = None, ) -> BuiltModel: from sequifier.config.train_config import interface_build_view + interfaces = ( + config.interfaces + if encoder_interface is None + else {encoder_interface: config.interface(encoder_interface)} + ) objectives = {} - for name, interface in config.interfaces.items(): + for name, interface in interfaces.items(): objective_view = interface_build_view(config, interface) # Next-occurrence target construction belongs to each training phase; # its forward attention policy is the ordinary causal policy. @@ -281,9 +287,11 @@ def _build_composable_network( routes = {} runtime_metadata = {} route_initialization_components: dict[str, tuple[nn.Module, Any]] = {} - for name, interface in config.interfaces.items(): + for name, interface in interfaces.items(): view = interface_build_view(config, interface) - load_pretrained = initialize and name != restored_history_interface + load_pretrained = ( + initialize and device.type != "meta" and name != restored_history_interface + ) built_ingestion = compile_feature_ingestion( hparams=view, direct_real_dtype_provider=lambda: backbone.layers[ @@ -322,23 +330,26 @@ def _build_composable_network( interface.ingestion.initialization_seed, ("adapter",), "initializer" ) adapter._sequifier_layer_group = "ingestion.output_projection" # type: ignore[attr-defined] - metadata = _decoder_metadata(view) - decoder = build_target_decoding( - view, - target_n_classes=metadata.target_n_classes, - target_global_to_decoder=metadata.target_global_to_decoder, - ) - for column, expected in interface.categorical_hash_contracts.items(): - actual = ( - decoder.hash_codecs[column].contract() - if column in decoder.hash_codecs - else None + decoder = None + if encoder_interface is None: + metadata = _decoder_metadata(view) + runtime_metadata[name] = metadata + decoder = build_target_decoding( + view, + target_n_classes=metadata.target_n_classes, + target_global_to_decoder=metadata.target_global_to_decoder, ) - if column in interface.target_decoder_ids and actual != expected: - raise ValueError( - f"Persisted categorical hash contract for {column!r} differs " - "from the resolved decoder codec." + for column, expected in interface.categorical_hash_contracts.items(): + actual = ( + decoder.hash_codecs[column].contract() + if column in decoder.hash_codecs + else None ) + if column in interface.target_decoder_ids and actual != expected: + raise ValueError( + f"Persisted categorical hash contract for {column!r} differs " + "from the resolved decoder codec." + ) static_encoder = None if interface.static_conditioning is not None: static_seed = interface.ingestion.initialization_seed @@ -393,12 +404,11 @@ def _build_composable_network( from sequifier.model.prediction_distribution import PredictionScales distributions = interface.prediction_distributions - if distributions: + if encoder_interface is None and distributions: routes[name].prediction_scales = PredictionScales(distributions).to(device) routes[name].depth_layouts = interface.depth_layouts routes[name].input_columns = tuple(interface.input_columns) routes[name].input_n_classes = interface.n_classes - runtime_metadata[name] = metadata route_initialization_components.update( { f"interfaces.{name}.ingestion": ( @@ -409,12 +419,13 @@ def _build_composable_network( adapter, interface.ingestion.initialization, ), - f"interfaces.{name}.decoder": ( - decoder, - interface.decoder.initialization, - ), } ) + if decoder is not None: + route_initialization_components[f"interfaces.{name}.decoder"] = ( + decoder, + interface.decoder.initialization, + ) if routes[name].static_condition_encoder is not None: route_initialization_components[ f"interfaces.{name}.static_condition_encoder" @@ -451,7 +462,7 @@ def _build_composable_network( context_length=config.runtime_settings.context_length, ) if initialize: - if len(routes) == 1: + if len(config.interfaces) == 1: # Preserve the legacy single-interface seeded initialization order. # The order is observable because default initializers consume RNG. name = next(iter(routes)) @@ -468,14 +479,13 @@ def _build_composable_network( backbone, config.model.backbone.initialization, ), - f"interfaces.{name}.decoder": route_initialization_components[ - f"interfaces.{name}.decoder" - ], } - if routes[name].static_condition_encoder is not None: - key = f"interfaces.{name}.static_condition_encoder" - initialization_components[key] = route_initialization_components[key] - for component in ("preconditioning_image_encoder", "prefix_projection"): + for component in ( + "decoder", + "static_condition_encoder", + "preconditioning_image_encoder", + "prefix_projection", + ): key = f"interfaces.{name}.{component}" if key in route_initialization_components: initialization_components[key] = route_initialization_components[ @@ -521,3 +531,21 @@ def build_transformer_network( restored_history_interface=restored_history_interface, logger=resolved_logger, ) + + +def build_history_encoder( + config: Any, + interface_name: str, + *, + device: torch.device, + restored_history: bool, +) -> ComposableTransformerNetwork: + """Build the selected ingestion path and backbone, using the actor's namespaces.""" + return _build_composable_network( + ModelExecution.from_config(config), + device=device, + initialize=True, + restored_history_interface=interface_name if restored_history else None, + logger=logging.getLogger(__name__), + encoder_interface=interface_name, + ).network diff --git a/src/sequifier/model/ingestions.py b/src/sequifier/model/ingestions.py index a908414d..a8fd6600 100644 --- a/src/sequifier/model/ingestions.py +++ b/src/sequifier/model/ingestions.py @@ -184,13 +184,23 @@ def __init__( self.register_buffer( "multipliers", torch.randint( - 1, HASH_PRIME, (num_hashes,), generator=generator, dtype=torch.int64 + 1, + HASH_PRIME, + (num_hashes,), + generator=generator, + dtype=torch.int64, + device="cpu", ), ) self.register_buffer( "offsets", torch.randint( - 0, HASH_PRIME, (num_hashes,), generator=generator, dtype=torch.int64 + 0, + HASH_PRIME, + (num_hashes,), + generator=generator, + dtype=torch.int64, + device="cpu", ), ) diff --git a/src/sequifier/model/network.py b/src/sequifier/model/network.py index 36959264..1f053618 100644 --- a/src/sequifier/model/network.py +++ b/src/sequifier/model/network.py @@ -51,7 +51,7 @@ class TracedModelOutput: class ModelInterfaceModule(nn.Module): - """One named ingestion/adapter/decoder route around the shared backbone.""" + """One named ingestion route, optionally with a prediction decoder.""" @beartype def __init__( @@ -59,7 +59,7 @@ def __init__( *, ingestion: nn.Module, ingestion_adapter: nn.Module, - decoder: nn.Module, + decoder: nn.Module | None, decoder_input_width: int, decoding_support: int, prediction_length: int, @@ -74,7 +74,8 @@ def __init__( raise ValueError("decoding_support must be positive.") self.ingestion = ingestion self.ingestion_adapter = ingestion_adapter - self.decoder = decoder + if decoder is not None: + self.decoder = decoder self.decoder_input_width = decoder_input_width self.decoding_support = decoding_support self.prediction_length = prediction_length diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index 257f7391..e75ba453 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -73,8 +73,7 @@ def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): actor.config.actions, **critic_history_options( actor, - context.run.phases, - context.index, + phase, restore=context.resume_state is not None, ), ) diff --git a/src/sequifier/rl/models.py b/src/sequifier/rl/models.py index 880d3ab2..4435cbcc 100644 --- a/src/sequifier/rl/models.py +++ b/src/sequifier/rl/models.py @@ -1,5 +1,6 @@ """Independent history value and state-action critics for offline IQL.""" +from contextlib import nullcontext from copy import deepcopy import torch @@ -8,23 +9,17 @@ from sequifier.io.batch import SequifierInputs from sequifier.io.policy_window import normalize_policy_features from sequifier.model.dtypes import cast_floating_to_module_dtype -from sequifier.model.factory import build_transformer_network -from sequifier.model.history_transfer import copy_history_weights, history_parameter_ids +from sequifier.model.factory import build_history_encoder +from sequifier.model.history_transfer import copy_history_weights -def history_transfer_enabled(phases, index): - """Continue with the same parameter eligibility as the last fresh phase.""" - while index > 0 and phases[index].optimizer_state == "continue": - index -= 1 - return getattr(phases[index], "history_initialization", None) == "phase_start_actor" - - -def critic_history_options(actor, phases, index, *, restore=False): - enabled = history_transfer_enabled(phases, index) - fresh = phases[index].optimizer_state == "reset" and not restore +def critic_history_options(actor, phase, *, restore=False): + restore = restore or phase.optimizer_state == "continue" return { - "history_source": actor.network if enabled and fresh else None, - "preserve_history_trainability": enabled, + "history_source": actor.network + if phase.history_initialization == "phase_start_actor" and not restore + else None, + "restore_history": restore, } @@ -93,55 +88,46 @@ def __init__( action_count: int | dict | None, *, history_source=None, - preserve_history_trainability=False, + restore_history=False, + metadata_only=False, ): super().__init__() - restored_history = preserve_history_trainability or history_source is not None - self.network = build_transformer_network( - execution_config, - device=torch.device(execution_config.device), - restored_history_interface=interface_name if restored_history else None, - ).network + device = torch.device("meta" if metadata_only else execution_config.device) self.interface_name = interface_name self.interface = execution_config.interface(interface_name) - history_ids = ( - history_parameter_ids(self.network, interface_name) - if restored_history - else set() - ) - for parameter in self.network.resolve_interface( - interface_name - ).decoder.parameters(): - if id(parameter) not in history_ids: - parameter.requires_grad_(False) - width = self.network.model_dim - self.action_embedding = ( - ActionEncoder( - action_count, - width, - self.interface.selected_columns_statistics - if self.interface.normalize_real_columns - else {}, + with device if metadata_only else nullcontext(): + self.network = build_history_encoder( + execution_config, + interface_name, + device=device, + restored_history=restore_history or history_source is not None, ) - if isinstance(action_count, dict) - else nn.Embedding(action_count, width) - if action_count - else None - ) - self.head = nn.Sequential( - nn.Linear(width * (2 if action_count else 1), width), - nn.ReLU(), - nn.Linear(width, 1), - ) - self.to(execution_config.device) + width = self.network.model_dim + self.action_embedding = ( + ActionEncoder( + action_count, + width, + self.interface.selected_columns_statistics + if self.interface.normalize_real_columns + else {}, + ) + if isinstance(action_count, dict) + else nn.Embedding(action_count, width) + if action_count + else None + ) + self.head = nn.Sequential( + nn.Linear(width * (2 if action_count else 1), width), + nn.ReLU(), + nn.Linear(width, 1), + ) + self.to(device) if history_source is not None: copy_history_weights(history_source, self.network, interface_name) @classmethod def ppo_value(cls, actor, **history): - value = cls( - deepcopy(actor.execution_config), actor.interface_name, None, **history - ) + value = cls(actor.execution_config, actor.interface_name, None, **history) if history.get("history_source") is not None: validate_independent_networks(actor.network, value) return value @@ -181,12 +167,21 @@ def prepared_value(self, features, metadata, action=None): ) -def make_iql_models( +def make_iql_critics( execution_config, interface_name: str, action_count: int | dict, **history ): value = HistoryCritic(execution_config, interface_name, None, **history) q1 = HistoryCritic(execution_config, interface_name, action_count, **history) q2 = HistoryCritic(execution_config, interface_name, action_count, **history) + return value, q1, q2 + + +def make_iql_models( + execution_config, interface_name: str, action_count: int | dict, **history +): + value, q1, q2 = make_iql_critics( + execution_config, interface_name, action_count, **history + ) targets = ( deepcopy(q1).requires_grad_(False), deepcopy(q2).requires_grad_(False), diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index 763387eb..8095e91f 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -22,8 +22,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): actor, **critic_history_options( actor, - context.run.phases, - context.index, + phase, restore=context.resume_state is not None, ), ).eval() diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 2be85ac2..cf4819d2 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -173,11 +173,7 @@ def preflight_environments(config): def preflight_optimization(config, actor, data): """Resolve constructors and authored scheduler budgets before creating outputs.""" - from sequifier.rl.models import ( - HistoryCritic, - critic_history_options, - make_iql_models, - ) + from sequifier.rl.models import HistoryCritic, make_iql_critics baseline = { name: parameter.requires_grad @@ -185,21 +181,20 @@ def preflight_optimization(config, actor, data): } budgets, previous_masks = [], None try: - for index, (phase, part) in enumerate(zip(config.phases, data)): + for phase, part in zip(config.phases, data): for name, parameter in actor.network.named_parameters(): parameter.requires_grad_(baseline[name]) modules = {"actor": actor.network} - history = critic_history_options(actor, config.phases, index, restore=True) if phase.algorithm == "iql": - value, q1, q2, _ = make_iql_models( + value, q1, q2 = make_iql_critics( actor.execution_config, actor.interface_name, actor.config.actions, - **history, + metadata_only=True, ) modules.update(value=value, critic=torch.nn.ModuleList([q1, q2])) elif phase.algorithm == "ppo": - modules["value"] = HistoryCritic.ppo_value(actor, **history) + modules["value"] = HistoryCritic.ppo_value(actor, metadata_only=True) masks = {} for role, module in modules.items(): frozen = phase.trainability.policy(role).resolve(module) diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index b6fc974b..95ddbb66 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -11,6 +11,7 @@ from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_plan import PhaseConfig +from sequifier.rl.models import HistoryCritic from sequifier.rl.policy import Policy from sequifier.runtime.plan import metric_improved, select_handoff, stop_scope from sequifier.runtime.random_state import capture_rng_state, restore_rng_state @@ -54,6 +55,53 @@ def require_equal(expected, actual, field): raise ValueError(f"{field}: expected {expected!r}, received {actual!r}") +def _legacy_critic_prefixes(module, interface_names): + prefixes = [] + for path, critic in module.named_modules(): + if isinstance(critic, HistoryCritic): + root = f"{path + '.' if path else ''}network.interfaces." + active = root + critic.interface_name + prefixes.extend((f"{active}.decoder.", f"{active}.prediction_scales.")) + prefixes.extend( + f"{root}{name}." + for name in interface_names + if name != critic.interface_name + ) + return tuple(prefixes) + + +def _prune_legacy_optimizer(state, optimizer, module, weights, saved_mask, mask): + """Restore positional optimizer slots after dropping unused critic parameters.""" + groups = state["param_groups"] + if len(groups) != 1 or len(optimizer.param_groups) != 1: + raise ValueError( + "legacy critic optimizer recovery cannot remove parameters from a multi-group plan" + ) + names = set(saved_mask) + saved_order = [name for name in weights if name in names] + group = groups[0] + slots = group["params"] + require_equal(len(names), len(saved_order), "legacy critic optimizer names") + require_equal(len(saved_order), len(slots), "legacy critic optimizer slots") + current_names = {id(p): name for name, p in module.named_parameters()} + keep = set(mask) + require_equal( + [current_names[id(p)] for p in optimizer.param_groups[0]["params"]], + [name for name in saved_order if name in keep], + "legacy critic optimizer order", + ) + dropped = {slot for name, slot in zip(saved_order, slots) if name not in keep} + return { + **state, + "param_groups": [ + {**group, "params": [slot for slot in slots if slot not in dropped]} + ], + "state": { + slot: value for slot, value in state["state"].items() if slot not in dropped + }, + } + + @dataclass(frozen=True) class ActorRepresentation: # Canonical JSON provides an immutable owned value, even for nested schemas. @@ -268,25 +316,7 @@ def __init__(self, context, *, modules=None, frozen=None): continuation["algorithm"], "optimization.algorithm", ) - require_equal( - sorted(self.networks), - sorted(continuation["networks"]), - "optimization.roles", - ) - require_equal( - self.mask_ids, - continuation["trainable_parameters"], - "optimization.trainability", - ) - for name, module in self.networks.items(): - module.load_state_dict(continuation["networks"][name], strict=True) - for role, (optimizer, scheduler) in self.optimization.items(): - optimizer.load_state_dict( - continuation["optimization"][role]["optimizer"] - ) - scheduler.load_state_dict( - continuation["optimization"][role]["scheduler"] - ) + self._restore_networks_and_optimizers(continuation, "optimization") self.steps = dict.fromkeys(self.modules, 0) self.sampler = torch.Generator().manual_seed(context.run.seed + context.index) self.state = context.resume_state or {} @@ -315,23 +345,8 @@ def __init__(self, context, *, modules=None, frozen=None): context.phase, monitor, selection.direction, self.state.get("selection") ) if self.state: - require_equal( - self.mask_ids, self.state["trainable_parameters"], "phase.trainability" - ) - require_equal( - sorted(self.networks), sorted(self.state["networks"]), "phase.networks" - ) - require_equal( - sorted(self.optimization), - sorted(self.state["optimization"]), - "phase.optimization", - ) - for name, module in self.networks.items(): - module.load_state_dict(self.state["networks"][name], strict=True) - for role, (optimizer, scheduler) in self.optimization.items(): - saved = self.state["optimization"][role] - optimizer.load_state_dict(saved["optimizer"]) - scheduler.load_state_dict(saved["scheduler"]) + self._restore_networks_and_optimizers(self.state, "phase") + for role, saved in self.state["optimization"].items(): steps = saved["steps"] budget = context.step_budget if ( @@ -346,6 +361,55 @@ def __init__(self, context, *, modules=None, frozen=None): self.sampler.set_state(self.state["sampler"]) self.restore_random() + def _restore_networks_and_optimizers(self, state, field): + require_equal( + sorted(self.networks), sorted(state["networks"]), f"{field}.networks" + ) + require_equal( + sorted(self.mask_ids), + sorted(state["trainable_parameters"]), + f"{field}.trainability roles", + ) + require_equal( + sorted(self.optimization), + sorted(state["optimization"]), + f"{field}.optimization", + ) + interface_names = self.context.actor.execution_config.interfaces + for role, module in self.networks.items(): + weights = state["networks"][role] + prefixes = ( + _legacy_critic_prefixes(module, interface_names) + if role != "actor" + else () + ) + obsolete = ( + {name for name in weights if name.startswith(prefixes)} + if prefixes + else set() + ) + module.load_state_dict( + {name: value for name, value in weights.items() if name not in obsolete} + if obsolete + else weights, + strict=True, + ) + if role not in self.optimization: + continue + mask = self.mask_ids[role] + saved_mask = state["trainable_parameters"][role] + retained = [name for name in saved_mask if name not in obsolete] + require_equal(mask, retained, f"{field}.trainability.{role}") + optimizer, scheduler = self.optimization[role] + saved = state["optimization"][role] + optimizer_state = saved["optimizer"] + if retained != saved_mask: + optimizer_state = _prune_legacy_optimizer( + optimizer_state, optimizer, module, weights, saved_mask, mask + ) + optimizer.load_state_dict(optimizer_state) + scheduler.load_state_dict(saved["scheduler"]) + def restore_random(self): if self.state: restore_rng_state(self.context.actor.device, self.state["random"]) From 4c94978bffc8c7625b21ed245def25f2c65fa224 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 12:26:50 +0100 Subject: [PATCH 53/71] refactor telemetry --- documentation/policy-learning.md | 18 ++-- src/sequifier/artifacts/phase.py | 4 + src/sequifier/io/output_paths.py | 2 + src/sequifier/rl/grpo.py | 28 ++---- src/sequifier/rl/learner.py | 79 +++++++++------- src/sequifier/rl/ppo.py | 22 ++--- src/sequifier/rl/run_checkpoint.py | 125 ++++++++++++++++++-------- src/sequifier/rl/run_state.py | 28 +++--- src/sequifier/rl/telemetry.py | 87 ++++++++++++++++++ src/sequifier/training/coordinator.py | 80 ++++++++++++++--- 10 files changed, 339 insertions(+), 134 deletions(-) create mode 100644 src/sequifier/rl/telemetry.py diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 714ae99c..3fd4e753 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -243,13 +243,17 @@ Online intervals count completed collection/update iterations. ## Checkpoints and inference -Policy runs use the shared version-4 run-checkpoint envelope; their native -recovery payload uses policy state version 7. See -[Training artifacts](training/artifacts.md). They retain representation, -plan/source/partition/adapter identities, selected phase results and checksums, -RNG and live phase networks/optimization/sampler/selection state. Keep committed -phase model artifacts alongside the checkpoint; reports are regenerated from committed -results. Source files and output-path conflicts are rechecked at phase boundaries. +Policy runs use the shared version-4 run-checkpoint envelope with a version-8 +native recovery payload. They retain representation, plan/source/partition/adapter +identities, selected phase summaries and checksums, RNG and live phase +networks/optimization/sampler/selection state. Training and evaluation histories +are journaled under each native phase directory. Keep the journal and committed +phase models with the checkpoint so reports can be regenerated from the recorded +positions. Mixed plans also retain their phase completion receipts. Version-7 +policy checkpoints import their embedded histories on resume. See +[Training artifacts](training/artifacts.md). + +Source files and output-path conflicts are rechecked at phase boundaries. Atomic publication precedes checkpoint advancement, so completed phases never rerun. `resume.mode: exact` validates simulator snapshots or reproducible group starts. diff --git a/src/sequifier/artifacts/phase.py b/src/sequifier/artifacts/phase.py index a9bb6ab4..55f331d1 100644 --- a/src/sequifier/artifacts/phase.py +++ b/src/sequifier/artifacts/phase.py @@ -25,6 +25,10 @@ def report(self): def receipt(self): return self.model.with_name("completed.json") + @property + def telemetry(self): + return self.model.parent / "telemetry" / "records.sqlite3" + def describe(self): return {"actor_path": str(self.model), "report_path": str(self.report)} diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py index 096f89a3..f689c926 100644 --- a/src/sequifier/io/output_paths.py +++ b/src/sequifier/io/output_paths.py @@ -69,6 +69,8 @@ def validate_training_destinations(config, checkpoint, protected): artifacts.model.with_name("run.pt"), artifacts.model.parent / "native", artifacts.model.parent / "checkpoints", + artifacts.telemetry.parent, + artifacts.telemetry, ): resolved = path.resolve() if not contains_path(directories[0], resolved): diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 034bc33e..0607201c 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -2,7 +2,6 @@ import statistics from contextlib import ExitStack -from copy import deepcopy from pathlib import Path from time import perf_counter @@ -32,10 +31,6 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): session = PhaseSession(context) state = session.state groups, env_steps = state.get("groups", 0), state.get("env_steps", 0) - evaluations, continuations = ( - list(state.get("evaluations", [])), - list(state.get("continuations", [])), - ) restart_stream = state.get("restart_stream", 0) if ( not 0 <= groups <= phase.total_groups @@ -64,13 +59,10 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): require_equal(state["adapter_identity"], identity, "grpo.adapter_identity") if context.run.resume.mode != "exact" and groups < phase.total_groups: restart_stream += 1 - continuations.append( - { - "transaction": session.transaction, - "mode": "new_groups", - "restart_stream": restart_stream, - "trajectory_equivalent": False, - } + session.record_continuation( + mode="new_groups", + restart_stream=restart_stream, + trajectory_equivalent=False, ) session.restore_random() @@ -101,7 +93,7 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): resources.callback(workers.close) if session.should_evaluate_initial(): - session.evaluate_online(adapter, evaluations, initial=True) + session.evaluate_online(adapter, initial=True) while groups < phase.total_groups and not session.stopping.triggered: count = min(phase.groups_per_iteration, phase.total_groups - groups) collection_start = perf_counter() @@ -142,7 +134,7 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): ) evaluation_start = perf_counter() if session.should_evaluate(final): - session.evaluate_online(adapter, evaluations) + session.evaluate_online(adapter) evaluation_seconds = perf_counter() - evaluation_start scores = [t.score for t in buffer.trajectories] record.update( @@ -159,9 +151,7 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): ), "lengths": [len(t.actions) for t in buffer.trajectories], "cutoffs": [t.cutoff_reason for t in buffer.trajectories], - "start_tokens": [ - deepcopy(group[0].start_token) for group in buffer.groups - ], + "start_tokens": [group[0].start_token for group in buffer.groups], "buffer_bytes": buffer.estimated_bytes, "simulator_workers": len(workers) if workers is not None else 1, "rollout_inference_batch_size": phase.rollout_inference_batch_size, @@ -189,8 +179,6 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): groups=groups, env_steps=env_steps, adapter_identity=identity, - evaluations=evaluations, - continuations=continuations, restart_stream=restart_stream, ) finally: @@ -207,7 +195,5 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): { "reference_weight_hash": context.reference_identity, "adapter": identity, - "evaluations": evaluations, - "continuations": continuations, }, ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 0abdb7b1..7601107a 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -9,14 +9,10 @@ import torch from sequifier.artifacts.phase import PhaseArtifacts +from sequifier.artifacts.reports import phase_report, publish_run_report, run_report from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism, configure_logger -from sequifier.rl.checkpoint import ( - load_policy, - phase_actor_path, - save_policy, - validate_policy_weights, -) +from sequifier.rl.checkpoint import load_policy, save_policy, validate_policy_weights from sequifier.rl.preparation import ( build_actor, preflight_environments, @@ -40,6 +36,7 @@ require_equal, weight_fingerprint, ) +from sequifier.rl.telemetry import TelemetryJournal from sequifier.runtime.plan import PhaseOutcome, RunIdentity, execute_phase, run_phases from sequifier.runtime.random_state import capture_rng_state, restore_rng_state @@ -62,6 +59,20 @@ def _plan_contract(config): return values +def _phase_report(result, checkpoint): + return phase_report( + model_path=result.provenance["actor_path"], + checkpoint_path=checkpoint, + **result.as_dict(), + ) + + +def _publish_phase_report(record, checkpoint): + result = record.report_result() + artifacts = PhaseArtifacts(Path(result.provenance["actor_path"])) + artifacts.publish_report(_phase_report(result, checkpoint)) + + def _new_actor(config): return build_actor( config, @@ -252,7 +263,8 @@ def _run_policy(config, *, phase_only=False, identity=None, legacy_recovery=Fals if not resuming: # Persist the complete fitted representation before the first update. recovery.save(checkpoint, actor.device) - recovery.write_reports(checkpoint) + for record in recovery.completed: + _publish_phase_report(record, checkpoint) def run_phase(phase_index, phase): current_state = recovery.phase_state @@ -292,23 +304,29 @@ def commit(state): if recovery.continuation is not None: actor._phase_continuation = recovery.continuation - context = PhaseContext( - config, - phase_index, - phase, - actor, - data[phase_index], - reference, - reference_identity, - current_state, - commit, - step_budgets[phase_index], + artifacts = PhaseArtifacts.for_phase( + config.phase_directory, phase_index, phase.name ) - selected, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) + position = current_state["telemetry_position"] if current_state else None + with TelemetryJournal.open(artifacts.telemetry, position) as telemetry: + context = PhaseContext( + config, + phase_index, + phase, + actor, + data[phase_index], + reference, + reference_identity, + current_state, + commit, + step_budgets[phase_index], + telemetry, + ) + selected, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) + position = telemetry.commit() representation.validate_actor( selected, config.units, f"phases.{phase.name}.selected_actor" ) - actor_path = phase_actor_path(config.phase_directory, phase_index, phase.name) result = replace( result, provenance={ @@ -318,21 +336,22 @@ def commit(state): "representation_fingerprint": representation.identity, "incoming_weight_hash": incoming_weight_hash, **initialization, - **PhaseArtifacts(actor_path).describe(), + **artifacts.describe(), }, ) - return PhaseOutcome((selected, result), result.stop_scope) + return PhaseOutcome((selected, result, position), result.stop_scope) def commit_phase(index, phase, outcome): nonlocal actor - actor, result = outcome.result + actor, result, position = outcome.result recovery.continuation = getattr(actor, "_phase_continuation", None) - recovery.finish_phase(actor, result) + recovery.finish_phase(actor, result, position) recovery.save(checkpoint, actor.device) - recovery.write_reports(checkpoint) + _publish_phase_report(recovery.completed[-1], checkpoint) + completion_reason = None if phase_only: - if not recovery.results: + if not recovery.completed: execute_phase(0, config.phases[0], run_phase, commit_phase) else: completion_reason = run_phases( @@ -342,12 +361,12 @@ def commit_phase(index, phase, outcome): start=recovery.phase_index, stopped=recovery.stopped, ) - from sequifier.artifacts.reports import publish_run_report, run_report - + results = tuple(record.report_result() for record in recovery.completed) + if completion_reason is not None: report = run_report( model_path=config.model_path, checkpoint_path=checkpoint, - phases=recovery.phase_reports(checkpoint), + phases=[_phase_report(result, checkpoint) for result in results], run_id=recovery.run_id, session_id=config._metrics_service.session_id, status="complete", @@ -366,7 +385,7 @@ def commit_phase(index, phase, outcome): recovery.save(checkpoint, actor.device) return PolicyRunResult( actor, - tuple(recovery.results), + results, representation, config.model_path, config.report_path, diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index 8095e91f..aa3da5f3 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -29,8 +29,6 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): session = PhaseSession(context, modules={"value": value}) state = session.state env_steps = state.get("env_steps", 0) - evaluations = list(state.get("evaluations", [])) - continuations = list(state.get("continuations", [])) if ( env_steps != min(session.transaction * phase.rollout_env_steps, phase.total_env_steps) @@ -60,16 +58,12 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): ): raise ValueError("PPO adapter did not restore its snapshot exactly") collector.load_state_dict(state["collector"], exact=exact) - continuations.append( - { - "transaction": session.transaction, - "mode": context.run.resume.mode, - "trajectory_equivalent": exact, - } + session.record_continuation( + mode=context.run.resume.mode, trajectory_equivalent=exact ) session.restore_random() if session.should_evaluate_initial(): - session.evaluate_online(adapter, evaluations, initial=True) + session.evaluate_online(adapter, initial=True) while env_steps < phase.total_env_steps and not session.stopping.triggered: horizon = min(phase.rollout_env_steps, phase.total_env_steps - env_steps) buffer, episodes = collector.collect(horizon, session.transaction) @@ -84,7 +78,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): session.transaction += 1 final = env_steps == phase.total_env_steps if session.should_evaluate(final): - session.evaluate_online(adapter, evaluations, env_steps=env_steps) + session.evaluate_online(adapter, env_steps=env_steps) session.record_training( final=final, **{ @@ -109,8 +103,6 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): session.checkpoint( final, env_steps=env_steps, - evaluations=evaluations, - continuations=continuations, collector=collector.state_dict(), environment_state=environment_state, adapter_identity=identity, @@ -122,9 +114,5 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): "env_steps": env_steps, **session.optimizer_clocks(), }, - { - "adapter": identity, - "evaluations": evaluations, - "continuations": continuations, - }, + {"adapter": identity}, ) diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 6cd4807d..7abea3ef 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -1,7 +1,7 @@ """Policy recovery with artifact-backed, authoritative committed phase boundaries.""" import math -from dataclasses import asdict, dataclass, field, fields +from dataclasses import asdict, dataclass, field, fields, replace from pathlib import Path import torch @@ -28,29 +28,51 @@ require_equal, weight_fingerprint, ) +from sequifier.rl.telemetry import TelemetryJournal from sequifier.runtime.plan import new_run_id from sequifier.runtime.random_state import capture_rng_state from sequifier.trajectory.sources import file_checksum +POLICY_RECOVERY_VERSION = 8 +ONLINE_TELEMETRY = ("evaluations", "continuations") + @dataclass(frozen=True) class CompletedPhase: result: PolicyPhaseResult sha256: str result_fingerprint: str + telemetry_position: str | None = None @classmethod - def from_dict(cls, values): + def from_dict(cls, values, *, legacy): result = values["result"] return cls( PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}), values["sha256"], values["result_fingerprint"], + None if legacy else values["telemetry_position"], ) def as_dict(self): return asdict(self) + def report_result(self): + artifacts = PhaseArtifacts(Path(self.result.provenance["actor_path"])) + history = TelemetryJournal.read(artifacts.telemetry, self.telemetry_position) + return replace( + self.result, + metrics=tuple(history.get("metrics", ())), + provenance={ + **self.result.provenance, + **{ + key: history.get(key, []) + for key in ONLINE_TELEMETRY + if self.result.algorithm in {"ppo", "grpo"} + }, + }, + ) + @dataclass class RunCheckpoint: @@ -69,10 +91,6 @@ class RunCheckpoint: stopped: bool = False run_id: str = field(default_factory=new_run_id) - @property - def results(self): - return tuple(record.result for record in self.completed) - @property def phase_index(self): return len(self.completed) @@ -92,19 +110,21 @@ def load(cls, path, config, plan, data_identity, *, run_id=None): if ( not isinstance(payload, dict) or payload.get("artifact_type") != "sequifier_policy_run" - or payload.get("format_version") != 7 + or payload.get("format_version") not in {7, POLICY_RECOVERY_VERSION} ): raise ValueError( - "policy resume requires a version-7 policy run checkpoint; initialize from its model artifact to start a new run" + "policy resume requires a version-7 or version-8 policy run checkpoint; initialize from its model artifact to start a new run" ) if run_id is not None and payload.get("run_id") != run_id: return None try: + legacy = payload["format_version"] == 7 saved_plan = payload["plan"] require_equal(plan, saved_plan, "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") completed = [ - CompletedPhase.from_dict(record) for record in payload["completed"] + CompletedPhase.from_dict(record, legacy=legacy) + for record in payload["completed"] ] checkpoint = cls( **{ @@ -121,12 +141,14 @@ def load(cls, path, config, plan, data_identity, *, run_id=None): payload["representation_fingerprint"], "resume.representation_fingerprint", ) - checkpoint.validate(config) + checkpoint.validate(config, legacy=legacy) + if legacy: + checkpoint.migrate_telemetry(config) return checkpoint except (KeyError, TypeError, AttributeError, IndexError) as exc: raise ValueError(f"malformed policy run checkpoint: {exc}") from exc - def validate(self, config): + def validate(self, config, *, legacy=False): if self.phase_index > len(config.phases): raise ValueError("checkpoint completed handoffs disagree with the plan") validate_finite_state(self.incoming_actor, "incoming_actor") @@ -152,16 +174,19 @@ def validate(self, config): require_equal( record.sha256, file_checksum(path), "resume.phase_actor.sha256" ) + result_data = result.as_dict() + if not legacy: + result_data["telemetry_position"] = record.telemetry_position require_equal( record.result_fingerprint, - fingerprint(result.as_dict()), + fingerprint(result_data), "resume.phase_result", ) - validate_finite_state(result.as_dict(), "completed_phase") + validate_finite_state(result_data, "completed_phase") incoming_identity = weight_fingerprint(self.incoming_actor) require_equal( - self.results[-1].selected_weight_hash - if self.results + self.completed[-1].result.selected_weight_hash + if self.completed else self.initialization["weight_hash"], incoming_identity, "resume.incoming_actor", @@ -171,8 +196,10 @@ def validate(self, config): return if self.phase_index == len(config.phases): raise ValueError("completed run cannot have a live phase state") - if state["version"] != PHASE_STATE_VERSION: + if state["version"] != (3 if legacy else PHASE_STATE_VERSION): raise ValueError("unsupported phase checkpoint state version") + if not legacy and not isinstance(state["telemetry_position"], str): + raise ValueError("phase checkpoint requires a committed telemetry position") transaction = state["transaction"] if ( not isinstance(transaction, int) or isinstance(transaction, bool) @@ -198,6 +225,36 @@ def validate(self, config): validate_reference_state(config.phases[self.phase_index], self) validate_finite_state(state, "phase") + def migrate_telemetry(self, config): + for index, record in enumerate(self.completed): + result = record.result + artifacts = PhaseArtifacts(Path(result.provenance["actor_path"])) + provenance = dict(result.provenance) + histories = {key: provenance.pop(key, ()) for key in ONLINE_TELEMETRY} + position = import_telemetry( + artifacts.telemetry, {"metrics": result.metrics, **histories} + ) + compact = replace(result, metrics=(), provenance=provenance) + self.completed[index] = replace( + record, + result=compact, + result_fingerprint=fingerprint( + {**compact.as_dict(), "telemetry_position": position} + ), + telemetry_position=position, + ) + if self.phase_state is not None: + state = self.phase_state + phase = config.phases[self.phase_index] + artifacts = PhaseArtifacts.for_phase( + config.phase_directory, self.phase_index, phase.name + ) + state["telemetry_position"] = import_telemetry( + artifacts.telemetry, + {key: state.pop(key, ()) for key in ("metrics", *ONLINE_TELEMETRY)}, + ) + state["version"] = PHASE_STATE_VERSION + def validate_actor(self, actor, units): self.representation.validate_actor(actor, units, "resume.representation") expected = actor.network.state_dict() @@ -237,7 +294,7 @@ def save(self, path, device, state=None): "policy", { "artifact_type": "sequifier_policy_run", - "format_version": 7, + "format_version": POLICY_RECOVERY_VERSION, **{ f.name: getattr(self, f.name) for f in fields(self) @@ -251,7 +308,7 @@ def save(self, path, device, state=None): ), ) - def finish_phase(self, actor, result): + def finish_phase(self, actor, result, telemetry_position): require_equal(self.phase_index, result.index, "handoff.phase_index") require_equal( result.selected_weight_hash, @@ -259,31 +316,29 @@ def finish_phase(self, actor, result): "handoff.selected_actor", ) artifacts = PhaseArtifacts(Path(result.provenance["actor_path"])) - artifacts.publish_model(policy_model_artifact(actor, metadata=result.as_dict())) + result_data = result.as_dict() + artifacts.publish_model(policy_model_artifact(actor, metadata=result_data)) weights = clone_weights(actor.network) self.completed.append( - CompletedPhase(result, **artifacts.record(result.as_dict())) + CompletedPhase( + result, + **artifacts.record( + {**result_data, "telemetry_position": telemetry_position} + ), + telemetry_position=telemetry_position, + ) ) self.stopped = result.stop_scope == "end_plan" self.incoming_actor = weights self.phase_state = None - def phase_reports(self, checkpoint_path): - from sequifier.artifacts.reports import phase_report - - return [ - phase_report( - model_path=result.provenance["actor_path"], - checkpoint_path=checkpoint_path, - **result.as_dict(), - ) - for result in self.results - ] - def write_reports(self, checkpoint_path): - """Reports are derived views; deleting one does not invalidate recovery.""" - for result, report in zip(self.results, self.phase_reports(checkpoint_path)): - PhaseArtifacts(Path(result.provenance["actor_path"])).publish_report(report) +def import_telemetry(path, histories): + with TelemetryJournal.open(path) as journal: + for kind, records in histories.items(): + for record in records: + journal.append(kind, record) + return journal.commit() def validate_finite_state(value, field): diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 95ddbb66..de89aac3 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -13,6 +13,7 @@ from sequifier.config.policy_plan import PhaseConfig from sequifier.rl.models import HistoryCritic from sequifier.rl.policy import Policy +from sequifier.rl.telemetry import TelemetryJournal from sequifier.runtime.plan import metric_improved, select_handoff, stop_scope from sequifier.runtime.random_state import capture_rng_state, restore_rng_state from sequifier.runtime.stopping import StoppingState @@ -22,7 +23,7 @@ PhaseType = TypeVar("PhaseType", bound=PhaseConfig) DataType = TypeVar("DataType") -PHASE_STATE_VERSION = 3 +PHASE_STATE_VERSION = 4 def clone_weights(module): @@ -192,6 +193,7 @@ class PhaseContext(Generic[PhaseType, DataType]): resume_state: dict | None commit: Callable[[dict], None] step_budget: int | None + telemetry: TelemetryJournal class Selection: @@ -229,7 +231,7 @@ def state_dict(self): ) } - def finish(self, context, transaction: int, clocks, metrics, provenance): + def finish(self, context, transaction: int, clocks, provenance): actor = context.actor.eval() if self.phase.handoff == "best" and self.best_weights is None: raise ValueError( @@ -255,7 +257,7 @@ def finish(self, context, transaction: int, clocks, metrics, provenance): selected, weight_fingerprint(actor.network.state_dict()), clocks, - tuple(metrics), + (), provenance, ) return actor, result @@ -338,7 +340,7 @@ def __init__(self, context, *, modules=None, frozen=None): self._count_stopping = True self.last_checkpoint_time = time.monotonic() self.last_snapshot_time = self.last_checkpoint_time - self.metrics = list(self.state.get("metrics", [])) + self.telemetry = context.telemetry selection = context.phase.evaluation.monitor monitor = selection.metric self.selection = Selection( @@ -488,10 +490,10 @@ def observe(self, metric, **record): def record_evaluation(self, metric, **record): self.observe(metric, **record) - self.metrics.append({"transaction": self.transaction, **record}) + self.telemetry.append("metrics", {"transaction": self.transaction, **record}) def record_training(self, *, final=False, **record): - self.metrics.append({"transaction": self.transaction, **record}) + self.telemetry.append("metrics", {"transaction": self.transaction, **record}) if self.context.run._metrics_service is not None: self.context.run._metrics_service.record(self, record, final=final) @@ -528,7 +530,12 @@ def complete_update( def optimizer_clocks(self): return {f"{role}_optimizer_steps": steps for role, steps in self.steps.items()} - def evaluate_online(self, adapter, evaluations, *, initial=False, **progress): + def record_continuation(self, **record): + self.telemetry.append( + "continuations", {"transaction": self.transaction, **record} + ) + + def evaluate_online(self, adapter, *, initial=False, **progress): from sequifier.rl.ppo_evaluation import evaluate_policy phase = self.context.phase @@ -551,7 +558,9 @@ def evaluate_online(self, adapter, evaluations, *, initial=False, **progress): self.observe(result["selection_value"], **result) if initial: progress["initial"] = True - evaluations.append({**result, "transaction": self.transaction, **progress}) + self.telemetry.append( + "evaluations", {**result, "transaction": self.transaction, **progress} + ) def optimization_state(self, *, include_steps=False): return { @@ -594,7 +603,7 @@ def checkpoint(self, final, **extra): "initial_evaluated": self.initial_evaluated, "trainable_parameters": self.mask_ids, "transaction": self.transaction, - "metrics": self.metrics, + "telemetry_position": self.telemetry.commit(), "reference": clone_weights(self.context.reference.network) if self.context.reference is not None else None, @@ -651,7 +660,6 @@ def finish(self, clocks, provenance): self.context, self.transaction, clocks, - self.metrics, { **provenance, "trainable_parameters": self.mask_ids, diff --git a/src/sequifier/rl/telemetry.py b/src/sequifier/rl/telemetry.py new file mode 100644 index 00000000..51ccbd15 --- /dev/null +++ b/src/sequifier/rl/telemetry.py @@ -0,0 +1,87 @@ +"""Append-only policy telemetry with checkpoint-addressable histories.""" + +import json +import sqlite3 +from contextlib import closing, contextmanager +from uuid import uuid4 + + +def _connect(path, mode): + if mode != "rwc" and not path.is_file(): + raise FileNotFoundError(f"Missing policy telemetry journal: {path}") + return closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode={mode}", uri=True)) + + +class TelemetryJournal: + def __init__(self, connection, position): + self.connection = connection + self.position = position + + @classmethod + @contextmanager + def open(cls, path, position=None): + if position is None: + path.parent.mkdir(parents=True, exist_ok=True) + with _connect(path, "rw" if position is not None else "rwc") as connection: + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA synchronous = EXTRA") + connection.execute( + """CREATE TABLE IF NOT EXISTS records ( + id TEXT PRIMARY KEY, + previous TEXT REFERENCES records(id), + kind TEXT NOT NULL, + payload TEXT NOT NULL + )""" + ) + if ( + position is not None + and not connection.execute( + "SELECT 1 FROM records WHERE id = ?", (position,) + ).fetchone() + ): + raise ValueError(f"Missing committed telemetry position in {path}") + yield cls(connection, position) + + def append(self, kind, record): + position = uuid4().hex + self.connection.execute( + "INSERT INTO records VALUES (?, ?, ?, ?)", + ( + position, + self.position, + kind, + json.dumps(record, allow_nan=False, separators=(",", ":")), + ), + ) + self.position = position + + def commit(self): + # Publish the journal prefix before the checkpoint that references it. + self.connection.commit() + return self.position + + @staticmethod + def read(path, position): + if position is None: + return {} + # Following parent links leaves older snapshots readable after a resumed + # run adds a different suffix to their history. + # Allow SQLite to roll back an interrupted writer before reading. + with _connect(path, "rw") as connection: + rows = connection.execute( + """WITH RECURSIVE history AS ( + SELECT rowid, previous FROM records WHERE id = ? + UNION ALL + SELECT r.rowid, r.previous + FROM records r JOIN history h ON r.id = h.previous + ) + SELECT kind, payload FROM records + WHERE rowid IN (SELECT rowid FROM history) ORDER BY rowid""", + (position,), + ) + history = {} + for kind, payload in rows: + history.setdefault(kind, []).append(json.loads(payload)) + if not history: + raise ValueError(f"Missing committed telemetry position in {path}") + return history diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 30d2c4ac..f3b1d3e4 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -5,7 +5,7 @@ Completing a phase publishes its selected artifact before advancing the envelope. """ -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Any @@ -116,6 +116,18 @@ def _legacy_learning_plan_identity(plan): return fingerprint(legacy) +def _completed_report(item): + if "report" in item: + return item["report"] + artifacts = PhaseArtifacts(Path(item["model_path"])) + receipt = artifacts.read_receipt(**item["report_receipt"]) + if receipt is None: + raise ValueError( + f"Missing committed policy report receipt: {artifacts.receipt}" + ) + return receipt["report"] + + def run_learning_plan(config) -> LearningRunResult: phases = config.training_plan if any(phase.optimizer_state == "continue" for phase in phases): @@ -133,7 +145,7 @@ def run_learning_plan(config) -> LearningRunResult: identity = fingerprint(plan) state: dict[str, Any] = { "artifact_type": "sequifier_learning_run", - "format_version": 2, + "format_version": 3, "plan": identity, "completed": [], "incoming": config.initialize_from, @@ -149,7 +161,7 @@ def run_learning_plan(config) -> LearningRunResult: ) if ( state.get("artifact_type") != "sequifier_learning_run" - or state.get("format_version") != 2 + or state.get("format_version") not in {2, 3} or ( state.get("plan") != identity and state.get("plan") != _legacy_learning_plan_identity(plan) @@ -170,7 +182,7 @@ def run_learning_plan(config) -> LearningRunResult: ) initialization = {"path": config.initialize_from} if config.actor_initialization == "history_only" and state["completed"]: - initialization = state["completed"][0]["report"]["provenance"][ + initialization = _completed_report(state["completed"][0])["provenance"][ "actor_initialization" ] verify_artifact( @@ -242,9 +254,27 @@ def run_learning_plan(config) -> LearningRunResult: validate_training_destinations(config, checkpoint, protected) def save_boundary(): + state["format_version"] = 3 model = ( read_execution_artifact(state["incoming"]) if state["incoming"] else None ) + if ( + model is not None + and state["completed"] + and (state["completed"][-1]["algorithm"] != "supervised") + ): + # Older phase models may still embed their policy histories. + provenance = dict(model.metadata.provenance) + provenance.pop("metrics", None) + if "provenance" in provenance: + provenance["provenance"] = { + key: value + for key, value in provenance["provenance"].items() + if key not in {"evaluations", "continuations"} + } + model = replace( + model, metadata=replace(model.metadata, provenance=provenance) + ) atomic_save(checkpoint, checkpoint_envelope("learning", state, model)) def run_phase(index, phase): @@ -332,14 +362,19 @@ def commit_phase(index, phase, outcome): initialization = report["provenance"]["actor_initialization"] state["stopped"] = outcome.stop_scope == "end_plan" checksum = file_checksum(artifacts.model) - state["completed"].append( - { - "name": phase.name, - "algorithm": phase.algorithm, - "model_path": str(artifacts.model), - "report": report, + item = { + "name": phase.name, + "algorithm": phase.algorithm, + "model_path": str(artifacts.model), + } + if phase.algorithm == "supervised": + item["report"] = report + else: + item["report_receipt"] = { + "identity": commit.fingerprint, + "run_id": state.get("active_run_id"), } - ) + state["completed"].append(item) state.update( incoming=str(artifacts.model), incoming_sha256=checksum, @@ -348,11 +383,22 @@ def commit_phase(index, phase, outcome): state.pop("active_run_id", None) save_boundary() + migrated = False for index, item in enumerate(state["completed"]): artifacts = PhaseArtifacts.for_phase( config.phase_directory, index, phases[index].name ) - artifacts.publish_report(item["report"]) + report = _completed_report(item) + if item["algorithm"] != "supervised" and "report" in item: + receipt = {"identity": fingerprint(report), "run_id": identity_scope.run_id} + artifacts.write_receipt(report=report, **receipt) + item["report_receipt"] = receipt + del item["report"] + migrated = True + artifacts.publish_report(report) + del report + if migrated: + save_boundary() completion_reason = run_phases( phases, run_phase, @@ -362,11 +408,17 @@ def commit_phase(index, phase, outcome): ) final = read_execution_artifact(state["incoming"]) save_model_artifact(final, config.model_path) + completed = [] + for item in state["completed"]: + result = dict(item) + result.pop("report_receipt", None) + result["report"] = _completed_report(item) + completed.append(result) publish_run_report( run_report( model_path=config.model_path, checkpoint_path=checkpoint, - phases=[item["report"] for item in state["completed"]], + phases=[item["report"] for item in completed], plan_fingerprint=identity, run_id=state["run_id"], session_id=identity_scope.session_id, @@ -383,5 +435,5 @@ def commit_phase(index, phase, outcome): config.model_path, str(checkpoint), config.report_path, - tuple(state["completed"]), + tuple(completed), ) From 58b7e12cc341b3bf1a11c6d4ed0107a5bbb28f64 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 12:42:15 +0100 Subject: [PATCH 54/71] action plan --- src/sequifier/model/decoders.py | 80 +++-- src/sequifier/rl/action_plan.py | 147 +++++++++ src/sequifier/rl/collection.py | 6 +- src/sequifier/rl/inference.py | 24 +- src/sequifier/rl/joint_distribution.py | 335 +++++++++++--------- src/sequifier/rl/parallel_grpo_collector.py | 8 +- src/sequifier/rl/policy.py | 93 +----- 7 files changed, 418 insertions(+), 275 deletions(-) create mode 100644 src/sequifier/rl/action_plan.py diff --git a/src/sequifier/model/decoders.py b/src/sequifier/model/decoders.py index ca1593d3..a4f0aa5f 100644 --- a/src/sequifier/model/decoders.py +++ b/src/sequifier/model/decoders.py @@ -151,7 +151,13 @@ def project_hidden_with_activations( @conditional_beartype def decode(self, target_column: str, x: Tensor) -> Tensor: + return self._project_target(target_column, self._project_hidden(x)) + + def decode_targets(self, x: Tensor, targets: tuple[str, ...]) -> dict[str, Tensor]: hidden = self._project_hidden(x) + return {target: self._project_target(target, hidden) for target in targets} + + def _project_target(self, target_column: str, hidden: Tensor) -> Tensor: output_layer = self.output_layers[target_column] if isinstance(output_layer, HashTargetHead): return output_layer(hidden) @@ -198,14 +204,7 @@ def forward( ) outputs = {} for target_column in self.target_columns: - output_layer = self.output_layers[target_column] - output = ( - output_layer(hidden) - if isinstance(output_layer, HashTargetHead) - else output_layer( - cast_floating_to_module_dtype(hidden, output_layer) - ).float() - ) + output = self._project_target(target_column, hidden) if trace is not None: output = trace.emit( f"decoder.branch.{branch_name}.logits.{target_column}", @@ -409,6 +408,45 @@ def _project_target(self, target: str, hidden: Tensor) -> Tensor: return layer(hidden) return layer(cast_floating_to_module_dtype(hidden, layer)).float() + def _initial_slots(self, context: Tensor) -> tuple[Tensor, Tensor]: + projected = self.context_projection( + cast_floating_to_module_dtype(context, self.context_projection) + ).reshape(-1, 1, self.width) + return projected, self.bos_embedding.expand(projected.shape[0], 1, self.width) + + def start_prefix( + self, context: Tensor, teachers: dict[str, Tensor] | None = None + ) -> Tensor: + """Build a decision prefix from context, BOS, and any recorded values.""" + slots = list(self._initial_slots(context)) + if teachers is not None: + slots.extend( + self._prefix_embedding(target, values) + for target, values in teachers.items() + ) + return torch.cat(slots, dim=1) + + def _prefix_embedding(self, target: str, values: Tensor) -> Tensor: + if self.target_column_types[target] == "categorical": + return self.value_embeddings[target](values).unsqueeze(1) + return self._embed_value(target, values).unsqueeze(1) + + def append_prefix(self, prefix: Tensor, target: str, values: Tensor) -> Tensor: + """Append real teacher values or already resolved categorical decoder IDs.""" + return torch.cat([prefix, self._prefix_embedding(target, values)], dim=1) + + def decode_prefix( + self, prefix: Tensor, targets: tuple[str, ...] + ) -> dict[str, Tensor]: + """Decode the requested trailing slots of a causal decision prefix.""" + hidden = self._run_transformer( + prefix, batch=prefix.shape[0], time=1, trace=None, branch_name="" + )[:, -len(targets) :] + return { + target: self._project_target(target, hidden[:, index]) + for index, target in enumerate(targets) + } + def trace_sites(self, branch_name: str) -> tuple[Any, ...]: from sequifier.model.tracing import TraceSite @@ -497,10 +535,7 @@ def _teacher_forced( raise ValueError( f"Missing teacher targets for decoder branch: {sorted(missing)!r}." ) - projected = self.context_projection( - cast_floating_to_module_dtype(context, self.context_projection) - ).reshape(batch * time, 1, self.width) - shifted = [self.bos_embedding.expand(batch * time, 1, self.width)] + shifted = list(self._initial_slots(context)) valid_mask = ( None if teacher_valid_mask is None else teacher_valid_mask[:, -time:] ) @@ -520,7 +555,7 @@ def _teacher_forced( batch * time, 1, self.width ) shifted.append(embedded) - sequence = torch.cat([projected, *shifted], dim=1) + sequence = torch.cat(shifted, dim=1) if trace is not None: shaped = trace.emit( f"decoder.branch.{branch_name}.slot_input", @@ -559,12 +594,7 @@ def _greedy( branch_name: str, ) -> dict[str, Tensor]: batch, time, _ = context.shape - projected = self.context_projection( - cast_floating_to_module_dtype(context, self.context_projection) - ).reshape(batch * time, 1, self.width) - sequence = torch.cat( - [projected, self.bos_embedding.expand(batch * time, 1, self.width)], dim=1 - ) + sequence = torch.cat(self._initial_slots(context), dim=1) outputs: dict[str, Tensor] = {} for index, target in enumerate(self.target_columns): transformer_input = sequence @@ -609,13 +639,7 @@ def _greedy( if self.target_column_types[target] == "categorical" else logits.squeeze(-1) ) - if self.target_column_types[target] == "categorical": - embedded = cast(nn.Embedding, self.value_embeddings[target])( - generated.to(torch.long) - ) - else: - embedded = self._embed_value(target, generated) - sequence = torch.cat([sequence, embedded.unsqueeze(1)], dim=1) + sequence = self.append_prefix(sequence, target, generated) return outputs def forward( @@ -668,7 +692,9 @@ def decode(self, target_column: str, x: Tensor) -> Tensor: TargetDecoderBranch, self.branches[self.target_to_branch[target_column]], ) - raw = branch.decode(target_column, x) + return self.resolve_target(target_column, branch.decode(target_column, x)) + + def resolve_target(self, target_column: str, raw: Tensor) -> Tensor: if target_column in self.hash_codecs: codec = self.hash_codecs[target_column] return codec.resolve(torch.split(raw, codec.widths, dim=-1)) diff --git a/src/sequifier/rl/action_plan.py b/src/sequifier/rl/action_plan.py new file mode 100644 index 00000000..702253f4 --- /dev/null +++ b/src/sequifier/rl/action_plan.py @@ -0,0 +1,147 @@ +"""Compile the policy binding against the model's decoder branches.""" + +from dataclasses import dataclass + +import torch + +from sequifier.config.policy_config import ActionFieldConfig +from sequifier.io.actions import ( + action_component_key, + action_latent_key, + inactive_action, +) +from sequifier.io.values import DomainValue +from sequifier.model.decoders import AutoregressiveTransformerDecoderBranch + + +@dataclass(frozen=True) +class ActionField: + spec: ActionFieldConfig + inactive: DomainValue + activity: dict[str, tuple[DomainValue, ...]] + support: tuple[int, ...] + latent_key: str | None + component_key: str | None + + +class ActionExecutionPlan: + def __init__(self, actor, decoder): + interface = actor.interface + actions = actor.config.actions + dependencies = {name: set(field.active_when) for name, field in actions.items()} + self.independent = {} + self.autoregressive = {} + self.prefix_steps = {} + for branch_name, branch in decoder.branches.items(): + targets = tuple(name for name in branch.target_columns if name in actions) + if not targets: + continue + if isinstance(branch, AutoregressiveTransformerDecoderBranch): + if tuple(branch.target_columns[: len(targets)]) != targets: + raise ValueError( + "autoregressive actions must include all preceding decoder targets" + ) + self.autoregressive[branch_name] = targets + previous = None + for name in targets: + self.prefix_steps[name] = (branch_name, previous) + if previous is not None: + dependencies[name].add(previous) + previous = name + else: + self.independent[branch_name] = targets + + pending = [name for name in interface.target_columns if name in actions] + if set(pending) != set(actions): + raise ValueError("policy actions must name existing model targets") + order = [] + resolved = set() + while pending: + ready = next( + (name for name in pending if dependencies[name] <= resolved), None + ) + if ready is None: + raise ValueError( + "action activity/decoder dependencies are cyclic or unknown" + ) + order.append(ready) + resolved.add(ready) + pending.remove(ready) + + self.order = tuple(order) + self.position = actor.config.decision_position + self.has_autoregression = any( + len(targets) > 1 for targets in self.autoregressive.values() + ) + self.fields = {} + record_fields = set(actions) + for name in self.order: + spec = actions[name] + if spec.type != interface.target_column_types[name]: + raise ValueError(f"{name}: action type differs from the model target") + if spec.scale == "model" and name not in interface.prediction_distributions: + raise ValueError( + f"{name}: scale: model requires a model prediction_distributions entry" + ) + support = () + latent_key = component_key = None + if spec.type == "categorical": + decoder_indices = { + global_id: index + for index, global_id in enumerate( + interface.target_decoder_ids[name] + ) + } + support = tuple( + decoder_indices[i] for i in actor.action_decoder_ids(name) + ) + if spec.active_when and spec.inactive_value not in spec.values: + raise ValueError( + f"{name}: inactive categorical value must be executable" + ) + else: + latent_key = action_latent_key(name, actor.config) + component_key = action_component_key(name, actor.config) + record_fields.update((latent_key, component_key)) + + activity = {} + for predecessor, labels in spec.active_when.items(): + parent = actions[predecessor] + if parent.type == "categorical": + if any(label not in parent.values for label in labels): + raise ValueError( + f"{name}: activity condition uses an unknown category" + ) + labels = [parent.values.index(label) for label in labels] + activity[predecessor] = tuple(labels) + self.fields[name] = ActionField( + spec, + inactive_action(spec), + activity, + support, + latent_key, + component_key, + ) + self.record_fields = frozenset(record_fields) + self._indices = {} + + def indices(self, device): + # The shared network can move independently of its policy view. + if device not in self._indices: + # Replay autograd can reuse indices first materialized in inference. + with torch.inference_mode(False): + support = { + name: torch.tensor(field.support, dtype=torch.long, device=device) + for name, field in self.fields.items() + if field.support + } + activity = { + name: { + parent: torch.tensor(values, device=device) + for parent, values in field.activity.items() + } + for name, field in self.fields.items() + if field.activity + } + self._indices[device] = support, activity + return self._indices[device] diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index 5ceae8d4..9950e6fb 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -85,7 +85,7 @@ def decide(self, value=None, *, deterministic=False, compute_entropy=True): value.eval() distribution = actor.distribution(self.inputs) features, metadata = distribution.features, distribution.metadata - action = distribution.select(deterministic, self.generator) + selected = distribution.select_and_score(deterministic, self.generator) estimate = ( value.prepared_value(features, metadata) if value is not None @@ -93,11 +93,11 @@ def decide(self, value=None, *, deterministic=False, compute_entropy=True): ) if not torch.isfinite(estimate).all(): raise ValueError("non-finite collection value") - index = action_record(action) + index = action_record(selected.actions) return PolicyDecision( decode_action(index, actor.config), index, - float(distribution.log_prob(action)[0]), + float(selected.log_prob[0]), float(distribution.entropy(self.generator)[0]) if compute_entropy else 0.0, float(estimate[0]), ) diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index a760c428..071235db 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -144,25 +144,25 @@ def predict_policy(policy, data, config): policy.device ) distribution = policy.distribution(inputs) - sampled = distribution.select( - config.action_selection == "deterministic", generator - ) + deterministic = config.action_selection == "deterministic" + if config.output_probabilities: + selected = distribution.select_and_score( + deterministic, generator, include_probabilities=True + ) + sampled = selected.actions + scores, probabilities = selected.log_prob, selected.probabilities + else: + sampled = distribution.select(deterministic, generator) + scores, probabilities = None, {} actions = [action_record(sampled, i) for i in range(len(rows))] - probabilities = ( - distribution.categorical_probabilities(sampled) - if config.output_probabilities - else {} - ) for index, (row_index, action) in enumerate(zip(rows, actions)): output = { row_key: row_index, **identities[index], **decode_action(action, spec), } - if config.output_probabilities: - output["action_log_probability"] = float( - distribution.log_prob(sampled)[index] - ) + if scores is not None: + output["action_log_probability"] = float(scores[index]) for name, values in probabilities.items(): probability_rows[name].append( { diff --git a/src/sequifier/rl/joint_distribution.py b/src/sequifier/rl/joint_distribution.py index aa72430e..ca685757 100644 --- a/src/sequifier/rl/joint_distribution.py +++ b/src/sequifier/rl/joint_distribution.py @@ -1,20 +1,24 @@ """Joint sampling and recorded-prefix scoring over existing model decoders.""" +from collections.abc import Mapping +from dataclasses import dataclass + import torch +from torch import Tensor -from sequifier.io.actions import ( - action_component_key, - action_latent_key, - action_record, - action_record_fields, - decode_action, - inactive_action, -) +from sequifier.io.actions import action_record, decode_action from sequifier.io.preprocessing import denormalize_values, normalize_values from sequifier.model.prediction_distribution import RealPredictionDistribution from sequifier.rl.distributions import CategoricalActionDistribution +@dataclass(frozen=True) +class ScoredSelection: + actions: dict[str, Tensor] + log_prob: Tensor + probabilities: dict[str, Tensor] + + class JointActionDistribution: """The model owns all weights; this object owns one encoded input window.""" @@ -27,14 +31,26 @@ def __init__(self, actor, inputs, *, _encoded=None): ) else: self.features, self.metadata, self.representation = _encoded - self.batch, self.time = self.representation.shape[:2] + self.plan = actor.action_plan + self.route = actor.network.resolve_interface(actor.interface_name) + self.decoder = self.route.decoder + self.batch = self.representation.shape[0] + end = self.representation.shape[1] + self.plan.position + 1 + start = end - self.route.decoding_support + if start < 0: + raise ValueError("decision_position lies outside the source decoder output") + self.context = self.route.decoder_input(self.representation[:, start:end])[:, 0] + self.valid = self.metadata["attention_valid_mask"][:, self.plan.position] + self.support, self.activity = self.plan.indices(self.context.device) + interface = actor.interface + self.statistics = ( + interface.selected_columns_statistics + if interface.normalize_real_columns + else {} + ) def row(self, index): - """View one encoded batch row without re-encoding or sharing action RNG. - - Only the representation and immutable input views are shared. Selection, - autoregressive decoding, likelihood and entropy retain their normal path. - """ + """View one encoded batch row without re-encoding or sharing action RNG.""" if not 0 <= index < self.batch: raise IndexError("joint-distribution row is outside the batch") return type(self)( @@ -47,179 +63,209 @@ def row(self, index): ), ) - def _teachers(self, actions): - interface = self.actor.interface - teachers = {} - for name in interface.target_columns: - categorical = interface.target_column_types[name] == "categorical" - fill = interface.target_decoder_ids[name][0] if categorical else 0.0 - values = torch.full( - (self.batch, self.time), - fill, - dtype=torch.long if categorical else torch.float32, - device=self.representation.device, + def _independent_parameters(self): + outputs = {} + for branch_name, targets in self.plan.independent.items(): + branch = self.decoder.branches[branch_name] + outputs.update(branch.decode_targets(self.context, targets)) + return outputs + + def _teacher(self, name, action): + field = self.plan.fields[name] + if action is None: + teacher = self.context.new_zeros( + self.batch, dtype=torch.long if field.support else torch.float32 ) - if name in actions: - action = actions[name] - if categorical: - ids = self.actor.action_decoder_ids(name) - action = torch.tensor(ids, device=values.device)[action.long()] - elif ( - interface.normalize_real_columns - and name in interface.selected_columns_statistics - ): - action = normalize_values( - action, interface.selected_columns_statistics[name] - ) - values[:, self.actor.config.decision_position] = action.to(values.dtype) - teachers[name] = values - return teachers + elif field.support: + teacher = self.support[name][action.long()].masked_fill(~self.valid, 0) + else: + stats = self.statistics.get(name) + teacher = normalize_values(action, stats) if stats is not None else action + teacher = teacher.float().expand(self.batch) + return teacher def _parameters(self, actions): - return self.actor.network.decode_representation( - self.representation, - teacher_targets=self._teachers(actions), - teacher_valid_mask=self.metadata["attention_valid_mask"], - interface_name=self.actor.interface_name, - ) + outputs = self._independent_parameters() + for branch_name, targets in self.plan.autoregressive.items(): + branch = self.decoder.branches[branch_name] + teachers = { + name: self._teacher(name, actions.get(name)) for name in targets[:-1] + } + prefix = branch.start_prefix(self.context, teachers) + outputs.update(branch.decode_prefix(prefix, targets)) + return outputs - def _field(self, name, parameters): - actor, field = self.actor, self.actor.config.actions[name] - output = parameters[name][:, actor.config.decision_position] + def _field(self, name, output): + field = self.plan.fields[name].spec + output = self.decoder.resolve_target(name, output) if field.type == "categorical": - indices = [ - actor.interface.target_decoder_ids[name].index(i) - for i in actor.action_decoder_ids(name) - ] - return CategoricalActionDistribution(output[:, indices]) + return CategoricalActionDistribution( + output.index_select(-1, self.support[name]) + ) location = output.squeeze(-1) - stats = actor.interface.selected_columns_statistics.get(name) - if stats is not None and actor.interface.normalize_real_columns: + stats = self.statistics.get(name) + if stats is not None: location = denormalize_values(location, stats) - return RealPredictionDistribution( - location, - actor.network.resolve_interface( - actor.interface_name - ).prediction_scales.scale(name) + scale = ( + self.route.prediction_scales.scale(name) if field.scale == "model" - else field.scale.fixed, - field, + else field.scale.fixed ) + return RealPredictionDistribution(location, scale, field) def _active(self, name, actions): - mask = torch.ones( - self.batch, dtype=torch.bool, device=self.representation.device - ) - for predecessor, labels in self.actor.config.actions[name].active_when.items(): + active = None + for predecessor, allowed in self.activity.get(name, {}).items(): if predecessor not in actions: raise ValueError( "conditional activity requires a preceding action field" ) - field = self.actor.config.actions[predecessor] - allowed = ( - [field.values.index(label) for label in labels] - if field.type == "categorical" - else labels - ) - mask &= torch.isin( - actions[predecessor], torch.tensor(allowed, device=mask.device) - ) - return mask + matches = torch.isin(actions[predecessor], allowed) + active = matches if active is None else active & matches + return active - def select(self, deterministic, generator): - actions = {} - for name in self.actor.action_order: - distribution = self._field(name, self._parameters(actions)) + def _select(self, deterministic, generator): + actions, fields, prefixes = {}, {}, {} + outputs = self._independent_parameters() + for name in self.plan.order: + if name in self.plan.prefix_steps: + branch_name, previous = self.plan.prefix_steps[name] + branch = self.decoder.branches[branch_name] + prefix = ( + branch.start_prefix(self.context) + if previous is None + else branch.append_prefix( + prefixes[branch_name], + previous, + self._teacher(previous, actions[previous]), + ) + ) + prefixes[branch_name] = prefix + outputs.update(branch.decode_prefix(prefix, (name,))) + distribution = self._field(name, outputs[name]) value = ( distribution.mode() if deterministic else distribution.sample(generator) ) - field = self.actor.config.actions[name] - inactive = inactive_action(field) + field = self.plan.fields[name] active = self._active(name, actions) - actions[name] = torch.where(active, value, value.new_tensor(inactive)) + actions[name] = ( + value + if active is None + else torch.where(active, value, value.new_tensor(field.inactive)) + ) if isinstance(distribution, RealPredictionDistribution): - latent = ( + actions[field.latent_key] = ( distribution.location if deterministic else distribution.sampled_latent ) - actions[action_latent_key(name, self.actor.config)] = latent - actions[action_component_key(name, self.actor.config)] = ( + actions[field.component_key] = ( torch.full_like(value, -1, dtype=torch.long) if deterministic else distribution.sampled_component ) + fields[name] = distribution, active + return actions, fields + + def select(self, deterministic, generator): + actions, _ = self._select(deterministic, generator) return actions + def select_and_score( + self, deterministic, generator, *, include_probabilities=False + ): + """Score the selected action with the distributions used to select it.""" + actions, fields = self._select(deterministic, generator) + return ScoredSelection( + actions, + self._score(actions, fields), + self._probabilities(fields) if include_probabilities else {}, + ) + def sample(self, generator=None): return self.select(False, generator) def mode(self): return self.select(True, None) - def log_prob(self, actions): - fields = {key for key in actions if key in self.actor.config.actions} - if fields != set(self.actor.config.actions): - raise ValueError("joint likelihood requires every action field") - allowed = action_record_fields(self.actor.config) - if set(actions) - allowed: - raise ValueError("joint action contains unknown fields") - parameters = self._parameters(actions) - score = self.representation.new_zeros( + def _score(self, actions, fields): + score = self.context.new_zeros( self.batch, - dtype=torch.float32 - if self.representation.device.type == "mps" - else torch.float64, + dtype=torch.float32 if self.context.device.type == "mps" else torch.float64, ) - for name in self.actor.action_order: + for name in self.plan.order: + distribution, active = fields[name] + field = self.plan.fields[name] value = actions[name] - if value.shape != (self.batch,): - raise ValueError(f"{name}: expected one action per decision") - distribution = self._field(name, parameters) - active = self._active(name, actions) - field = self.actor.config.actions[name] - inactive = inactive_action(field) - if (value[~active] != inactive).any(): - raise ValueError(f"{name}: inactive action differs from its point mass") - safe = torch.where(active, value, distribution.mode().detach()) - latent = actions.get(action_latent_key(name, self.actor.config)) + if active is not None: + if ((value != field.inactive) & ~active).any(): + raise ValueError( + f"{name}: inactive action differs from its point mass" + ) + value = torch.where(active, value, distribution.mode().detach()) + latent = actions.get(field.latent_key) if ( isinstance(distribution, RealPredictionDistribution) and latent is not None ): - latent = torch.where(active, latent, distribution.location.detach()) - mixture = actions.get(action_component_key(name, self.actor.config)) - if mixture is not None: - mixture = torch.where(active, mixture, -1) - component = distribution.log_prob( - safe, latent=latent, component=mixture + component = actions.get(field.component_key) + if active is not None: + latent = torch.where(active, latent, distribution.location.detach()) + if component is not None: + component = torch.where(active, component, -1) + log_prob = distribution.log_prob( + value, latent=latent, component=component ) else: - component = distribution.log_prob(safe) - score = score + torch.where(active, component, 0.0) + log_prob = distribution.log_prob(value) + score = score + ( + log_prob if active is None else torch.where(active, log_prob, 0.0) + ) return score.float() - def categorical_probabilities(self, actions): - """Per-field conditional probabilities at the supplied action prefix.""" - parameters = self._parameters(actions) + def log_prob(self, actions): + if not isinstance(actions, Mapping) or self.plan.fields.keys() - actions.keys(): + raise ValueError("joint likelihood requires every action field") + if actions.keys() - self.plan.record_fields: + raise ValueError("joint action contains unknown fields") + for name in self.plan.order: + if actions[name].shape != (self.batch,): + raise ValueError(f"{name}: expected one action per decision") + outputs = self._parameters(actions) + fields = { + name: (self._field(name, outputs[name]), self._active(name, actions)) + for name in self.plan.order + } + return self._score(actions, fields) + + def _probabilities(self, fields): result = {} - for name, field in self.actor.config.actions.items(): - if field.type == "categorical": - distribution = self._field(name, parameters) - assert isinstance(distribution, CategoricalActionDistribution) + for name in self.actor.config.actions: + if name not in fields: + continue + distribution, active = fields[name] + if isinstance(distribution, CategoricalActionDistribution): probabilities = distribution.distribution.probs - if field.active_when: + if active is not None: point = torch.zeros_like(probabilities) - point[:, inactive_action(field)] = 1 - probabilities = torch.where( - self._active(name, actions)[:, None], probabilities, point - ) + point[:, self.plan.fields[name].inactive] = 1 + probabilities = torch.where(active[:, None], probabilities, point) result[name] = probabilities return result + def categorical_probabilities(self, actions): + """Per-field conditional probabilities at the supplied action prefix.""" + outputs = self._parameters(actions) + return self._probabilities( + { + name: (self._field(name, outputs[name]), self._active(name, actions)) + for name in self.support + } + ) + @property def entropy_is_exact(self): - return not self.actor.autoregressive_actions and all( + return not self.plan.has_autoregression and all( not field.active_when and field.lower is None and field.upper is None @@ -233,18 +279,16 @@ def entropy(self, generator=None): return ( torch.stack( [ - self._field(name, parameters).entropy() - for name in self.actor.action_order + self._field(name, parameters[name]).entropy() + for name in self.plan.order ] ) .sum(dim=0) .float() ) - # Score-function estimator includes the distribution of sampled prefixes. - # Its value estimates -E log p; its surrogate has the entropy gradient. - with torch.no_grad(): - actions = self.sample(generator) - logp = self.log_prob(actions) + # Sampled actions are detached. The score-function surrogate includes + # the likelihood of the sampled prefixes. + logp = self.select_and_score(False, generator).log_prob surrogate = -(logp.detach() + 1) * logp return -logp.detach() + surrogate - surrogate.detach() @@ -253,13 +297,12 @@ def kl(self, reference, generator=None): raise ValueError( "KL requires matching action measures and physical transforms" ) - if not self.actor.autoregressive_actions and not any( - field.active_when for field in self.actor.config.actions.values() - ): + if not self.plan.has_autoregression and not self.activity: current, frozen = self._parameters({}), reference._parameters({}) values = [] for name, field in self.actor.config.actions.items(): - p, q = self._field(name, current), reference._field(name, frozen) + p = self._field(name, current[name]) + q = reference._field(name, frozen[name]) if isinstance(p, CategoricalActionDistribution) and isinstance( q, CategoricalActionDistribution ): @@ -278,11 +321,9 @@ def kl(self, reference, generator=None): * torch.distributions.kl_divergence(p.normal, q.normal) ) return torch.stack(values).sum(dim=0).float() - with torch.no_grad(): - actions = self.sample(generator) - logp = self.log_prob(actions) - delta = logp - reference.log_prob(actions) - surrogate = (delta.detach() + 1) * logp + selected = self.select_and_score(False, generator) + delta = selected.log_prob - reference.log_prob(selected.actions) + surrogate = (delta.detach() + 1) * selected.log_prob return delta.detach() + surrogate - surrogate.detach() def domain_actions(self, actions): diff --git a/src/sequifier/rl/parallel_grpo_collector.py b/src/sequifier/rl/parallel_grpo_collector.py index 3ff73262..a2d9272f 100644 --- a/src/sequifier/rl/parallel_grpo_collector.py +++ b/src/sequifier/rl/parallel_grpo_collector.py @@ -23,10 +23,12 @@ def _decide(actor, reference, cursors): generators = [cursor.generator for cursor in cursors] # Decode and validate distributions across the full environment batch. # Only random draws use individual generators; no per-row decoder calls. - actions = distribution.select(False, generators) - old_scores = distribution.log_prob(actions) + selected = distribution.select_and_score(False, generators) + actions = selected.actions scores = ( - torch.stack([old_scores, reference.evaluate_actions(inputs, actions)], dim=1) + torch.stack( + [selected.log_prob, reference.evaluate_actions(inputs, actions)], dim=1 + ) .cpu() .tolist() ) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index f0a3805a..462eb1a5 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -15,7 +15,7 @@ from sequifier.model.execution import ModelExecution from sequifier.model.execution_schema import ExecutionSchema from sequifier.model.prediction_distribution import PredictionScales -from sequifier.rl.distributions import CategoricalActionDistribution +from sequifier.rl.action_plan import ActionExecutionPlan from sequifier.rl.joint_distribution import JointActionDistribution if TYPE_CHECKING: @@ -87,78 +87,10 @@ def backbone(self): return self.network.backbone def _resolve_actions(self): - from sequifier.model.decoders import AutoregressiveTransformerDecoderBranch - route = self.network.resolve_interface(self.interface_name) - prerequisites = { - name: set(field.active_when) for name, field in self.config.actions.items() - } - for branch in route.decoder.branches.values(): - targets = [ - name for name in branch.target_columns if name in self.config.actions - ] - if isinstance(branch, AutoregressiveTransformerDecoderBranch) and targets: - last = max(branch.target_columns.index(name) for name in targets) - if set(branch.target_columns[: last + 1]) - set(targets): - raise ValueError( - "autoregressive actions must include all preceding decoder targets" - ) - for index, name in enumerate(targets): - prerequisites[name].update(targets[:index]) - pending = [ - name for name in self.interface.target_columns if name in prerequisites - ] - if set(pending) != set(self.config.actions): - raise ValueError("policy actions must name existing model targets") - order = [] - while pending: - ready = next( - (name for name in pending if prerequisites[name] <= set(order)), None - ) - if ready is None: - raise ValueError( - "action activity/decoder dependencies are cyclic or unknown" - ) - order.append(ready) - pending.remove(ready) - self.action_order = tuple(order) - self.autoregressive_actions = any( - isinstance(branch, AutoregressiveTransformerDecoderBranch) - and len( - [name for name in branch.target_columns if name in self.config.actions] - ) - > 1 - for branch in route.decoder.branches.values() - ) - for index, name in enumerate(order): - field = self.config.actions[name] - if ( - field.scale == "model" - and name not in self.interface.prediction_distributions - ): - raise ValueError( - f"{name}: scale: model requires a model prediction_distributions entry" - ) - if field.type != self.interface.target_column_types[name]: - raise ValueError(f"{name}: action type differs from the model target") - if set(field.active_when) - set(order[:index]): - raise ValueError( - "activity conditions must reference earlier action fields" - ) - if field.type == "categorical": - self.action_decoder_ids(name) - if field.active_when and field.inactive_value not in field.values: - raise ValueError( - f"{name}: inactive categorical value must be executable" - ) - for predecessor, labels in field.active_when.items(): - parent = self.config.actions[predecessor] - if parent.type == "categorical" and any( - label not in parent.values for label in labels - ): - raise ValueError( - f"{name}: activity condition uses an unknown category" - ) + self.action_plan = ActionExecutionPlan(self, route.decoder) + self.action_order = self.action_plan.order + self.autoregressive_actions = self.action_plan.has_autoregression if not hasattr(route, "prediction_scales"): route.prediction_scales = PredictionScales( self.interface.prediction_distributions @@ -172,6 +104,7 @@ def _resolve_actions(self): def action_decoder_ids(self, name): field = self.config.actions[name] mapping = self.interface.id_maps.get(name, {}) + support = set(self.interface.target_decoder_ids.get(name, ())) ids = [] for label in field.values: key = ( @@ -183,16 +116,13 @@ def action_decoder_ids(self, name): ) global_id = mapping.get(key, mapping.get(str(label), mapping.get(str(key)))) if global_id is None: - if ( - isinstance(label, int) - and label in self.interface.target_decoder_ids[name] - ): + if isinstance(label, int) and label in support: global_id = label else: raise ValueError( f"{name}: action label {label!r} is absent from the model codec" ) - if global_id not in self.interface.target_decoder_ids[name]: + if global_id not in support: raise ValueError(f"{name}: action is absent from decoder support") ids.append(global_id) if not ids or len(ids) != len(set(ids)): @@ -241,15 +171,12 @@ def prepared_logits(self, features, metadata) -> Tensor | dict[str, Tensor]: for name, value in outputs.items() } - def distribution( - self, inputs: SequifierInputs - ) -> CategoricalActionDistribution | JointActionDistribution: + def distribution(self, inputs: SequifierInputs) -> JointActionDistribution: return JointActionDistribution(self, inputs) def act(self, inputs: SequifierInputs, *, deterministic: bool, generator): - distribution = self.distribution(inputs) - action = distribution.select(deterministic, generator) - return action, distribution.log_prob(action) + selected = self.distribution(inputs).select_and_score(deterministic, generator) + return selected.actions, selected.log_prob def evaluate_actions( self, inputs: SequifierInputs, actions: Tensor | dict[str, Tensor] From 84178516c1f5c960a2a64dc8df6a443eb5de1ed8 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 12:59:22 +0100 Subject: [PATCH 55/71] centralised resource accounting --- documentation/training/resource-controls.md | 23 ++++ src/sequifier/config/training_resolution.py | 1 + src/sequifier/io/episode_cache.py | 35 +++++ src/sequifier/io/episode_corpus.py | 20 +-- src/sequifier/io/policy_dataset.py | 62 ++++++--- src/sequifier/model/frozen_image_cache.py | 55 ++++++-- src/sequifier/model/images.py | 2 +- src/sequifier/rl/collection.py | 30 ++++- src/sequifier/rl/group_collection.py | 31 ++++- src/sequifier/rl/grpo.py | 133 ++++++++++--------- src/sequifier/rl/grpo_buffer.py | 43 +++--- src/sequifier/rl/grpo_collector.py | 9 +- src/sequifier/rl/grpo_update.py | 3 + src/sequifier/rl/iql.py | 124 +++++++++-------- src/sequifier/rl/learner.py | 32 ++++- src/sequifier/rl/parallel_grpo_collector.py | 76 ++++++----- src/sequifier/rl/ppo.py | 24 +++- src/sequifier/rl/ppo_buffer.py | 86 +++++++----- src/sequifier/rl/ppo_collector.py | 3 +- src/sequifier/rl/ppo_evaluation.py | 1 + src/sequifier/rl/ppo_update.py | 102 +++++++------- src/sequifier/rl/prepared_inputs.py | 64 ++++++++- src/sequifier/rl/scoring.py | 34 +++-- src/sequifier/rl/simulator_workers.py | 58 ++++++-- src/sequifier/rl/training.py | 37 +++--- src/sequifier/rl/window_storage.py | 59 ++++++++ src/sequifier/runtime/resource_accounting.py | 72 ++++++++++ src/sequifier/training/coordinator.py | 22 ++- 28 files changed, 866 insertions(+), 375 deletions(-) create mode 100644 src/sequifier/io/episode_cache.py create mode 100644 src/sequifier/rl/window_storage.py create mode 100644 src/sequifier/runtime/resource_accounting.py diff --git a/documentation/training/resource-controls.md b/documentation/training/resource-controls.md index 483a7381..f9409c71 100644 --- a/documentation/training/resource-controls.md +++ b/documentation/training/resource-controls.md @@ -38,6 +38,29 @@ Lazy folders gather at most a microbatch at a time, including when view expansio produces many logical examples from one stored chunk. Chunk loading remains bounded by the shard size, so an individual shard must fit its working allowance. +## Policy run resource accounting + +Policy run reports include a `resource_accounting` section. Policy phase reports +carry the cumulative snapshot in `provenance.resource_accounting`. The snapshots +show current and peak estimated bytes by category and the peak of their combined +demand at a single point in execution. A resumed invocation starts a fresh +accounting session. + +Accounting covers prepared policy catalogs and partitions, decoded episode and +label caches, reconstructed scoring and update windows, rollout estimates and +archive working space, parent-side simulator request and response payloads, and +retained frozen-image features. Prepared partitions are shared through the last +phase that needs them. Each rollout is released after its update and any archive +or summary work, before evaluation begins. + +`episode_cache_size` still counts episodes, `max_buffer_bytes` still uses the +algorithm's conservative payload estimates, and `image_feature_cache_bytes` +still applies to each distinct cache. A cache shared by the actor and reference +contributes once. The combined estimate is observational. Model and optimizer +storage, activations, allocator overhead, and simulator process internals are +outside its scope. The supervised data-loading budget continues to apply to +supervised loaders. + ## Autocast and matrix precision `layer_autocast: true` applies Torch autocast to local/DDP training forward and diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 41db2f5c..b87c42c5 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -303,6 +303,7 @@ def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): fitting.use_config(fitted) for column, statistics in fitting.normalization_statistics().items(): values["selected_columns_statistics"].setdefault(column, statistics) + prepared_inputs.measure() return DatasetMetadata.model_validate(values) diff --git a/src/sequifier/io/episode_cache.py b/src/sequifier/io/episode_cache.py new file mode 100644 index 00000000..ade26bef --- /dev/null +++ b/src/sequifier/io/episode_cache.py @@ -0,0 +1,35 @@ +"""Count-bounded LRU storage for decoded episodes and decision labels.""" + +from collections import OrderedDict + +from sequifier.io.memory_budget import resident_bytes + + +class EpisodeCache: + def __init__(self, capacity, storage=None): + self.capacity = capacity + self.storage = storage + self.entries = OrderedDict() + self.bytes = 0 + + def get(self, key, load): + if key not in self.entries: + value = load(key) + size = resident_bytes(value) if self.storage is not None else 0 + self.entries[key] = value, size + self.bytes += size + if self.storage is not None: + self.storage.resize(self.bytes) + if len(self.entries) > self.capacity: + _, (_, evicted) = self.entries.popitem(last=False) + self.bytes -= evicted + if self.storage is not None: + self.storage.resize(self.bytes) + self.entries.move_to_end(key) + return self.entries[key][0] + + def clear(self): + self.entries.clear() + self.bytes = 0 + if self.storage is not None: + self.storage.resize(0) diff --git a/src/sequifier/io/episode_corpus.py b/src/sequifier/io/episode_corpus.py index 05930bae..2c9f30c5 100644 --- a/src/sequifier/io/episode_corpus.py +++ b/src/sequifier/io/episode_corpus.py @@ -2,12 +2,12 @@ import hashlib import json -from collections import OrderedDict from dataclasses import dataclass from decimal import Decimal from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.fingerprints import fingerprint +from sequifier.io.episode_cache import EpisodeCache from sequifier.io.policy_window import action_index from sequifier.io.sequence import SequenceEpisode from sequifier.trajectory.importers import source_episodes @@ -32,7 +32,7 @@ class EpisodeMetadata: class EpisodeCorpus: """Own validated episode metadata and one bounded storage cache.""" - def __init__(self, sources, config, cache_size=8): + def __init__(self, sources, config, cache_size=8, *, cache_owner=None): if cache_size < 1: raise ValueError("episode_cache_size must be positive") self.sources = [ @@ -45,7 +45,11 @@ def __init__(self, sources, config, cache_size=8): ) ] self.cache_size = cache_size - self.cache = OrderedDict() + self.cache = ( + cache_owner.episode_cache(cache_size) + if cache_owner is not None + else EpisodeCache(cache_size) + ) self.metadata = [] seen = set() for index in range(len(self.sources)): @@ -81,12 +85,10 @@ def __len__(self): return len(self.sources) def __getitem__(self, index): - if index not in self.cache: - self.cache[index] = _load(self.sources[index]) - if len(self.cache) > self.cache_size: - self.cache.popitem(last=False) - self.cache.move_to_end(index) - return self.cache[index] + return self.cache.get(index, self._load_episode) + + def _load_episode(self, index): + return _load(self.sources[index]) def view(self): return EpisodeView([(self, i) for i in range(len(self))]) diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index 0a9ddd9e..e124ad13 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -1,7 +1,6 @@ """Episode-safe policy views over ordinary sequences and imported rollout logs.""" from bisect import bisect_right -from collections import OrderedDict from collections.abc import Sequence import torch @@ -9,6 +8,7 @@ from sequifier.config.policy_inputs import PolicyEpisodeSpec from sequifier.io.batch import SequifierBatch +from sequifier.io.episode_cache import EpisodeCache from sequifier.io.episode_corpus import EpisodeCorpus, EpisodeView, episode_content_key from sequifier.io.episode_features import EpisodeFeatures, return_labels from sequifier.io.partitioning import split_components @@ -24,6 +24,7 @@ def __init__( config: PolicyEpisodeSpec, *, episode_cache_size: int = 8, + cache_owner=None, ): if episode_cache_size < 1: raise ValueError("episode_cache_size must be positive") @@ -36,11 +37,18 @@ def __init__( self.group_ids = [] self.lengths = [] self.cumulative_lengths = [] - self._labels = OrderedDict() + self.cache_owner = cache_owner + self._labels = ( + cache_owner.episode_cache(episode_cache_size) + if cache_owner is not None + else EpisodeCache(episode_cache_size) + ) view = ( sources if isinstance(sources, EpisodeView) - else EpisodeCorpus(sources, config, episode_cache_size).view() + else EpisodeCorpus( + sources, config, episode_cache_size, cache_owner=cache_owner + ).view() ) eligible = [] seen = set() @@ -91,22 +99,20 @@ def rebind(self, config): super().rebind(config) def _episode_labels(self, index: int): - if index not in self._labels: - episode = self.episodes[index] - length = self.lengths[index] - actions = [ - action_index(value, self.config) for value in episode.actions[:length] - ] - returns = ( - return_labels(episode, length, self.config.returns.gamma) - if self.config.return_conditioned - else None - ) - self._labels[index] = actions, returns - if len(self._labels) > self.episode_cache_size: - self._labels.popitem(last=False) - self._labels.move_to_end(index) - return self._labels[index] + return self._labels.get(index, self._load_labels) + + def _load_labels(self, index): + episode = self.episodes[index] + length = self.lengths[index] + actions = [ + action_index(value, self.config) for value in episode.actions[:length] + ] + returns = ( + return_labels(episode, length, self.config.returns.gamma) + if self.config.return_conditioned + else None + ) + return actions, returns def locate(self, index): if index < 0: @@ -161,6 +167,7 @@ def keys(index): dataset.episodes.subset(indices), dataset.config, episode_cache_size=dataset.episode_cache_size, + cache_owner=dataset.cache_owner, ) for indices in partitions ) @@ -169,8 +176,17 @@ def keys(index): class PolicyDataCatalog: """Run-owned corpora and reusable decision views for named input sources.""" - def __init__(self, sources, config, cache_size, *, preference_manifests=None): + def __init__( + self, + sources, + config, + cache_size, + *, + preference_manifests=None, + cache_owner=None, + ): self.sources, self.config, self.cache_size = sources, config, cache_size + self.cache_owner = cache_owner self.corpora = {} self.datasets = {} self.preference_manifests = ( @@ -195,12 +211,16 @@ def dataset(self, names): for name in key: if name not in self.corpora: self.corpora[name] = EpisodeCorpus( - [self.sources[name]], self.config, self.cache_size + [self.sources[name]], + self.config, + self.cache_size, + cache_owner=self.cache_owner, ) self.datasets[key] = EpisodeDecisions( EpisodeView.combine([self.corpora[name].view() for name in key]), self.config, episode_cache_size=self.cache_size, + cache_owner=self.cache_owner, ) return self.datasets[key] diff --git a/src/sequifier/model/frozen_image_cache.py b/src/sequifier/model/frozen_image_cache.py index 584c82a0..60595a7b 100644 --- a/src/sequifier/model/frozen_image_cache.py +++ b/src/sequifier/model/frozen_image_cache.py @@ -10,6 +10,7 @@ image_feature_cache_enabled, image_identity_scope, ) +from sequifier.runtime.resource_accounting import Allocation, allocation @dataclass @@ -19,13 +20,42 @@ class FrozenImageCache: retained_bytes: int = 0 hits: int = 0 misses: int = 0 + storage: Allocation | None = field(default=None, repr=False, compare=False) + owners: int = field(default=0, init=False, repr=False, compare=False) + + @classmethod + def scoped(cls, budget): + return cls(budget, storage=allocation("frozen_image_features")) + + def acquire(self): + self.owners += 1 + return self + + def release(self): + if self.owners: + self.owners -= 1 + if not self.owners: + self.clear() + if self.storage is not None: + self.storage.close() + + def replacement(self): + if self.storage is None: + return type(self)(self.budget) + self.release() + return type(self).scoped(self.budget).acquire() def __deepcopy__(self, memo): return type(self)(self.budget) def clear(self): self.entries.clear() - self.retained_bytes = 0 + self._resize(0) + + def _resize(self, size): + self.retained_bytes = size + if self.storage is not None: + self.storage.resize(size) def get(self, key): feature = self.entries.get(key) @@ -42,14 +72,16 @@ def put(self, key, feature): return previous = self.entries.pop(key, None) if previous is not None: - self.retained_bytes -= previous.numel() * previous.element_size() + self._resize( + self.retained_bytes - previous.numel() * previous.element_size() + ) while self.retained_bytes + size > self.budget: _, evicted = self.entries.popitem(last=False) - self.retained_bytes -= evicted.numel() * evicted.element_size() + self._resize(self.retained_bytes - evicted.numel() * evicted.element_size()) # Own a row, not a view retaining an entire microbatch's GPU storage. with torch.inference_mode(False), torch.no_grad(): self.entries[key] = feature.detach().clone() - self.retained_bytes += size + self._resize(self.retained_bytes + size) def encoder_signature(encoder): @@ -156,20 +188,25 @@ def frozen_image_cache_scope(actor, reference, budget): signature = encoder_signature(encoder) if signature is None: continue - saved.append( - (encoder, encoder._feature_cache, encoder._cache_signature) - ) other = active.get(name) - encoder._feature_cache = ( + shared = ( other._feature_cache if other is not None and _same_backbone(encoder, other) - else FrozenImageCache(budget) + else None ) + saved.append( + (encoder, encoder._feature_cache, encoder._cache_signature) + ) + encoder._feature_cache = ( + shared if shared is not None else FrozenImageCache.scoped(budget) + ).acquire() encoder._cache_signature = signature active[name] = encoder with image_identity_scope(bool(saved)): yield finally: for encoder, cache, signature in reversed(saved): + if encoder._feature_cache is not None: + encoder._feature_cache.release() encoder._feature_cache = cache encoder._cache_signature = signature diff --git a/src/sequifier/model/images.py b/src/sequifier/model/images.py index fe25c8eb..cbe25b9d 100644 --- a/src/sequifier/model/images.py +++ b/src/sequifier/model/images.py @@ -90,7 +90,7 @@ def train(self, mode: bool = True): def clear_feature_cache(self): """Invalidate transient features after an external frozen-weight change.""" if self._feature_cache is not None: - self._feature_cache = FrozenImageCache(self._feature_cache.budget) + self._feature_cache = self._feature_cache.replacement() self._cache_signature = None def _backbone_microbatch(self, images: Tensor) -> Tensor: diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index 9950e6fb..c7f74bea 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -10,6 +10,8 @@ from sequifier.rl.distributions import action_generator from sequifier.rl.environment import validate_reset, validate_step from sequifier.rl.history import PolicyHistory +from sequifier.rl.window_storage import tensor_storage_bytes +from sequifier.runtime.resource_accounting import allocation from sequifier.trajectory.returns import remaining_return @@ -53,12 +55,34 @@ def __init__( self.desired_return = None self.episode_return, self.episode_length, self.ended = 0.0, 0, True self.last_result = None + self._inputs = None + self._window_storage = allocation("reconstructed_windows") + + @property + def inputs(self): + if self._inputs is None: + raise ValueError("policy cursor has no reconstructed inputs") + return self._inputs def refresh(self): inputs = self.history.inputs() - self.inputs = ( - inputs if self.defer_device_transfer else inputs.to(self.actor.device) - ) + moved = inputs if self.defer_device_transfer else inputs.to(self.actor.device) + if self._window_storage.accounting is not None: + seen = set() + size = tensor_storage_bytes(moved, seen) + self._window_storage.resize( + size + + tensor_storage_bytes(inputs, seen) + + tensor_storage_bytes(self._inputs, seen) + ) + self._window_storage.resize(size) + self._inputs = moved + + def close(self): + self._inputs = None + self.last_result = None + self.history.frames.clear() + self._window_storage.close() def reset(self, result, episode_id, desired_return=None): result = validate_reset(result, self.units, compact_images=True) diff --git a/src/sequifier/rl/group_collection.py b/src/sequifier/rl/group_collection.py index ce13a901..0216bbbf 100644 --- a/src/sequifier/rl/group_collection.py +++ b/src/sequifier/rl/group_collection.py @@ -75,7 +75,17 @@ def seal(self, generation, result: OnlineStepResult): class GroupCollection: - def __init__(self, context, adapter, count, generation, env_steps, restart_stream): + def __init__( + self, + context, + adapter, + count, + generation, + env_steps, + restart_stream, + *, + storage=None, + ): self.context, self.adapter = context, adapter self.phase, self.generation = context.phase, generation bound = count * self.phase.group_size * self.phase.max_steps @@ -96,16 +106,26 @@ def __init__(self, context, adapter, count, generation, env_steps, restart_strea ) self.used, self.executed = 0, 0 self.initial = {} + self.cursors = set() + self.storage = storage + self.window_bytes = 0 def __enter__(self): return self def __exit__(self, kind, error, traceback): + for cursor in self.cursors: + cursor.close() + self.cursors.clear() if error is not None and isinstance(error, Exception): raise GroupCollectionError(str(error), self.executed) from error - def account(self, size): + def account(self, size, *, window=False): self.used += size + if window: + self.window_bytes += size + if self.storage is not None: + self.storage.resize(self.used - self.window_bytes) if self.used > self.phase.max_buffer_bytes: raise ValueError("GRPO collection exceeded max_buffer_bytes") @@ -126,8 +146,15 @@ def trajectory(self, group, sample, token, *, deferred=False): stream_seed(self.seed, f"group-{group}-actions", sample), defer_device_transfer=deferred, ) + self.cursors.add(cursor) return GroupTrajectory(group, sample, token, cursor) + def seal_trajectory(self, trajectory, result): + sealed = trajectory.seal(self.generation, result) + trajectory.cursor.close() + self.cursors.remove(trajectory.cursor) + return sealed + def environment_seed(self, group, sample): return stream_seed(self.seed, f"group-{group}-environment", sample) diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 0607201c..0631e8fe 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -23,6 +23,7 @@ from sequifier.rl.grpo_collector import collect_groups from sequifier.rl.grpo_update import update_grpo from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal +from sequifier.runtime.resource_accounting import allocation from sequifier.trajectory.archives import collection_archive @@ -96,71 +97,83 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): session.evaluate_online(adapter, initial=True) while groups < phase.total_groups and not session.stopping.triggered: count = min(phase.groups_per_iteration, phase.total_groups - groups) - collection_start = perf_counter() - buffer, executed = collect_groups( - context, - adapter, - groups, - count, - session.transaction, - env_steps, - restart_stream, - workers=workers, - ) - collection_seconds = perf_counter() - collection_start - update_start = perf_counter() - record = update_grpo(session, buffer) - if actor.device.type == "cuda": - torch.cuda.synchronize(actor.device) - update_seconds = perf_counter() - update_start - env_steps += executed - groups += count - session.transaction += 1 - final = groups == phase.total_groups - if phase.archive_directory: - write_json_atomic( - Path(phase.archive_directory) - / f"groups-{session.transaction:08d}.json", - collection_archive( - "grpo", - { - "format": "sequifier.grpo-archive", - "version": 1, - "reference_identity": context.reference_identity, - "generation": buffer.generation, - "trajectories": [t.as_dict() for t in buffer.trajectories], - }, - ), + with allocation("rollout_storage") as storage: + collection_start = perf_counter() + buffer, executed = collect_groups( + context, + adapter, + groups, + count, + session.transaction, + env_steps, + restart_stream, + workers=workers, + storage=storage, ) + collection_seconds = perf_counter() - collection_start + update_start = perf_counter() + record = update_grpo(session, buffer) + if actor.device.type == "cuda": + torch.cuda.synchronize(actor.device) + update_seconds = perf_counter() - update_start + env_steps += executed + groups += count + session.transaction += 1 + final = groups == phase.total_groups + if phase.archive_directory: + with allocation("rollout_archive") as archive: + archive.resize(buffer.estimated_bytes) + write_json_atomic( + Path(phase.archive_directory) + / f"groups-{session.transaction:08d}.json", + collection_archive( + "grpo", + { + "format": "sequifier.grpo-archive", + "version": 1, + "reference_identity": context.reference_identity, + "generation": buffer.generation, + "trajectories": [ + t.as_dict() for t in buffer.trajectories + ], + }, + ), + ) + scores = [t.score for t in buffer.trajectories] + record.update( + { + "transaction": session.transaction, + "groups": groups, + "env_steps": env_steps, + "policy_version": actor.grpo_policy_version, + "collected_policy_version": buffer.generation, + "score_mean": statistics.fmean(scores), + "score_std": statistics.pstdev(scores), + "zero_variance_groups": sum( + len({t.score for t in group}) == 1 + for group in buffer.groups + ), + "lengths": [len(t.actions) for t in buffer.trajectories], + "cutoffs": [t.cutoff_reason for t in buffer.trajectories], + "start_tokens": [ + group[0].start_token for group in buffer.groups + ], + "buffer_bytes": buffer.estimated_bytes, + "simulator_workers": len(workers) if workers is not None else 1, + "rollout_inference_batch_size": phase.rollout_inference_batch_size, + "collection_seconds": collection_seconds, + "collection_env_steps_per_second": ( + executed / collection_seconds + ), + "update_seconds": update_seconds, + } + ) + del buffer, scores evaluation_start = perf_counter() if session.should_evaluate(final): session.evaluate_online(adapter) evaluation_seconds = perf_counter() - evaluation_start - scores = [t.score for t in buffer.trajectories] - record.update( - { - "transaction": session.transaction, - "groups": groups, - "env_steps": env_steps, - "policy_version": actor.grpo_policy_version, - "collected_policy_version": buffer.generation, - "score_mean": statistics.fmean(scores), - "score_std": statistics.pstdev(scores), - "zero_variance_groups": sum( - len({t.score for t in group}) == 1 for group in buffer.groups - ), - "lengths": [len(t.actions) for t in buffer.trajectories], - "cutoffs": [t.cutoff_reason for t in buffer.trajectories], - "start_tokens": [group[0].start_token for group in buffer.groups], - "buffer_bytes": buffer.estimated_bytes, - "simulator_workers": len(workers) if workers is not None else 1, - "rollout_inference_batch_size": phase.rollout_inference_batch_size, - "collection_seconds": collection_seconds, - "collection_env_steps_per_second": executed / collection_seconds, - "update_seconds": update_seconds, - "evaluation_seconds": evaluation_seconds, - } - ) + record["evaluation_seconds"] = evaluation_seconds record.update(image_cache_statistics(actor, context.reference)) release_image_cache_entries(actor, context.reference) session.record_training(final=final, **record) diff --git a/src/sequifier/rl/grpo_buffer.py b/src/sequifier/rl/grpo_buffer.py index 226ca87c..19faa933 100644 --- a/src/sequifier/rl/grpo_buffer.py +++ b/src/sequifier/rl/grpo_buffer.py @@ -1,13 +1,12 @@ """Bounded complete groups with reconstructable causal frames and fixed scores.""" from dataclasses import dataclass -from typing import cast import torch from sequifier.io.actions import action_batch -from sequifier.io.batch import SequifierInputs, collate_model_inputs from sequifier.rl.history import history_inputs +from sequifier.rl.window_storage import collated_windows @dataclass(frozen=True) @@ -35,27 +34,27 @@ def inputs(self, spec, step): def batches(self, actor, chunk_size): for start in range(0, len(self.actions), chunk_size): end = min(start + chunk_size, len(self.actions)) - yield ( - cast( - SequifierInputs, - collate_model_inputs( - [self.inputs(actor.config, t) for t in range(start, end)] + with collated_windows( + (self.inputs(actor.config, t) for t in range(start, end)), + actor.device, + ) as inputs: + yield ( + inputs, + action_batch( + self.actions[start:end], actor.config, device=actor.device ), - ).to(actor.device), - action_batch( - self.actions[start:end], actor.config, device=actor.device - ), - torch.tensor( - self.old_log_probabilities[start:end], - dtype=torch.float32, - device=actor.device, - ), - torch.tensor( - self.reference_log_probabilities[start:end], - dtype=torch.float32, - device=actor.device, - ), - ) + torch.tensor( + self.old_log_probabilities[start:end], + dtype=torch.float32, + device=actor.device, + ), + torch.tensor( + self.reference_log_probabilities[start:end], + dtype=torch.float32, + device=actor.device, + ), + ) + del inputs def as_dict(self): return { diff --git a/src/sequifier/rl/grpo_collector.py b/src/sequifier/rl/grpo_collector.py index 805525f1..18c5246c 100644 --- a/src/sequifier/rl/grpo_collector.py +++ b/src/sequifier/rl/grpo_collector.py @@ -22,6 +22,7 @@ def collect_groups( restart_stream=0, *, workers=None, + storage=None, ): if workers is not None: from sequifier.rl.parallel_grpo_collector import collect_parallel_groups @@ -35,11 +36,12 @@ def collect_groups( generation, env_steps, restart_stream, + storage=storage, ) actor, reference, phase = context.actor.eval(), context.reference, context.phase require_group_start(adapter) with GroupCollection( - context, adapter, count, generation, env_steps, restart_stream + context, adapter, count, generation, env_steps, restart_stream, storage=storage ) as collection: groups = [] for group_index in range(start_group, start_group + count): @@ -60,12 +62,11 @@ def collect_groups( for _ in range(phase.max_steps): frame = cursor.history.current collection.account(frame_storage_bytes(frame) + 1024) - inputs = cursor.inputs decision = cursor.decide(compute_entropy=False) collection.account(4 * _json_payload_bytes(decision.action)) reference_score = float( reference.evaluate_actions( - inputs, + cursor.inputs, action_batch( [decision.action], actor.config, device=actor.device ), @@ -80,6 +81,6 @@ def collect_groups( break if result is None: raise ValueError("GRPO requires a positive trajectory horizon") - trajectories.append(state.seal(generation, result)) + trajectories.append(collection.seal_trajectory(state, result)) groups.append(collection.seal_group(trajectories)) return collection.finish(groups) diff --git a/src/sequifier/rl/grpo_update.py b/src/sequifier/rl/grpo_update.py index 81e7be09..b1e455b5 100644 --- a/src/sequifier/rl/grpo_update.py +++ b/src/sequifier/rl/grpo_update.py @@ -36,6 +36,7 @@ def validate_old_scores(actor, buffer, chunk_size): raise ValueError( "GRPO reconstructed old-policy scores differ before first update" ) + del inputs, actions, old, _ def update_grpo(session, buffer): @@ -76,6 +77,7 @@ def update_grpo(session, buffer): old_kl += ( float(kl.sum()) / len(trajectory.actions) / len(indices) ) + del inputs, actions, old, ref if old_kl > phase.old_policy_target_kl: stopped = True break @@ -96,6 +98,7 @@ def update_grpo(session, buffer): values[0].sum() / len(trajectory.actions) / len(indices) ).backward() totals.add(dict(zip(totals, values)), len(trajectory.actions)) + del inputs, actions, old, ref scored += 1 session.step() if stopped: diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index e75ba453..cd42f362 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -1,16 +1,15 @@ """Fixed-dataset categorical IQL as a phase-local actor/value/twin-Q runtime.""" import math -from typing import cast import torch from sequifier.config.policy_plan import IQLPhaseConfig -from sequifier.io.iql_dataset import IQLBatch -from sequifier.io.tensors import collate_tensor_records +from sequifier.io.iql_dataset import IQLTransitionDataset from sequifier.rl.models import critic_history_options, make_iql_models from sequifier.rl.offline_data import OfflineData from sequifier.rl.run_state import PhaseContext, PhaseSession +from sequifier.rl.window_storage import collated_windows def _select(inputs, mask): @@ -30,34 +29,30 @@ def _polyak(target, source, rate): target_buffer.copy_(buffer) -def _evaluate(actor, value, q1, q2, dataset, batch_size, gamma, device): +def _evaluate( + actor, value, q1, q2, dataset: IQLTransitionDataset, batch_size, gamma, device +): totals = {"action_nll": 0.0, "bellman_residual": 0.0, "value": 0.0, "q_min": 0.0} with torch.no_grad(): for start in range(0, len(dataset), batch_size): - batch = cast( - IQLBatch, - collate_tensor_records( - [ - dataset[index] - for index in range(start, min(start + batch_size, len(dataset))) - ] - ), - ).to(device) - action = batch.action - v = value(batch.current) - q = torch.minimum(q1(batch.current, action), q2(batch.current, action)) - nll = -actor.evaluate_actions(batch.current, action) - successor_value = torch.zeros_like(v) - if batch.bootstrap.any(): - successor_value[batch.bootstrap] = value( - _select(batch.successor, batch.bootstrap) + indices = range(start, min(start + batch_size, len(dataset))) + with collated_windows((dataset[i] for i in indices), device) as batch: + action = batch.action + v = value(batch.current) + q = torch.minimum(q1(batch.current, action), q2(batch.current, action)) + nll = -actor.evaluate_actions(batch.current, action) + successor_value = torch.zeros_like(v) + if batch.bootstrap.any(): + successor_value[batch.bootstrap] = value( + _select(batch.successor, batch.bootstrap) + ) + totals["action_nll"] += float(nll.sum()) + totals["bellman_residual"] += float( + (q - batch.reward - gamma * successor_value).abs().sum() ) - totals["action_nll"] += float(nll.sum()) - totals["bellman_residual"] += float( - (q - batch.reward - gamma * successor_value).abs().sum() - ) - totals["value"] += float(v.sum()) - totals["q_min"] += float(q.sum()) + totals["value"] += float(v.sum()) + totals["q_min"] += float(q.sum()) + del batch, action result = {key: value / len(dataset) for key, value in totals.items()} if not all(math.isfinite(value) for value in result.values()): raise ValueError("non-finite IQL held-out likelihood or critic diagnostics") @@ -112,45 +107,46 @@ def evaluate(**record): indices = torch.randint( len(data[0]), (phase.batch_size,), generator=session.sampler ) - batch = cast( - IQLBatch, collate_tensor_records([data[0][int(i)] for i in indices]) - ).to(actor.device) - with torch.no_grad(): - qbar = torch.minimum( - targets[0](batch.current, batch.action), - targets[1](batch.current, batch.action), - ) - old_v = value(batch.current) - advantage = qbar - old_v - weight = torch.exp( - (phase.inverse_temperature * advantage).clamp( - max=math.log(phase.max_advantage_weight) + with collated_windows( + (data[0][int(i)] for i in indices), actor.device + ) as batch: + with torch.no_grad(): + qbar = torch.minimum( + targets[0](batch.current, batch.action), + targets[1](batch.current, batch.action), ) - ) - successor_value = torch.zeros_like(old_v) - if batch.bootstrap.any(): - successor_value[batch.bootstrap] = value( - _select(batch.successor, batch.bootstrap) + old_v = value(batch.current) + advantage = qbar - old_v + weight = torch.exp( + (phase.inverse_temperature * advantage).clamp( + max=math.log(phase.max_advantage_weight) + ) ) - target = batch.reward + phase.gamma * successor_value - delta = qbar - value(batch.current) - v_loss = ( - torch.where(delta < 0, 1 - phase.expectile, phase.expectile) - * delta.square() - ).mean() - q_loss = ( - (q1(batch.current, batch.action) - target).square() - + (q2(batch.current, batch.action) - target).square() - ).mean() - actor_loss = -( - weight * actor.evaluate_actions(batch.current, batch.action) - ).mean() - losses = {"value": v_loss, "critic": q_loss, "actor": actor_loss} - session.zero_grad() - for role, loss in losses.items(): - if not torch.isfinite(loss): - raise ValueError(f"non-finite IQL {role} objective") - loss.backward() + successor_value = torch.zeros_like(old_v) + if batch.bootstrap.any(): + successor_value[batch.bootstrap] = value( + _select(batch.successor, batch.bootstrap) + ) + target = batch.reward + phase.gamma * successor_value + delta = qbar - value(batch.current) + v_loss = ( + torch.where(delta < 0, 1 - phase.expectile, phase.expectile) + * delta.square() + ).mean() + q_loss = ( + (q1(batch.current, batch.action) - target).square() + + (q2(batch.current, batch.action) - target).square() + ).mean() + actor_loss = -( + weight * actor.evaluate_actions(batch.current, batch.action) + ).mean() + losses = {"value": v_loss, "critic": q_loss, "actor": actor_loss} + session.zero_grad() + for role, loss in losses.items(): + if not torch.isfinite(loss): + raise ValueError(f"non-finite IQL {role} objective") + loss.backward() + del batch session.step() _polyak(targets[0], q1, phase.target_update_rate) _polyak(targets[1], q2, phase.target_update_rate) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 7601107a..b1d2d34f 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -183,10 +183,21 @@ def run_policy_phase( def _run_policy(config, *, phase_only=False, identity=None, legacy_recovery=False): + prepared = PreparedTrainingInputs.for_run(config) + with prepared.run_scope(): + try: + return _execute_policy( + config, prepared, phase_only, identity, legacy_recovery + ) + finally: + config._metrics_service = None + + +def _execute_policy(config, prepared, phase_only, identity, legacy_recovery): torch.set_float32_matmul_precision("highest") configure_determinism(config.seed, config.enforce_determinism) plan = _plan_contract(config) - inputs = PreparedTrainingInputs.for_run(config).resolve(config) + inputs = prepared.resolve(config) inputs.revalidate(inputs.identity, "run.source_identity") data_identity = inputs.identity validate_output_paths(config, inputs) @@ -232,6 +243,7 @@ def _run_policy(config, *, phase_only=False, identity=None, legacy_recovery=Fals "run.normalization", ) validate_data(data, actor) + prepared.measure() data_contract = [item.contract if item is not None else None for item in data] if recovery is not None: require_equal(data_contract, recovery.data_contract, "resume.partitions") @@ -314,7 +326,7 @@ def commit(state): phase_index, phase, actor, - data[phase_index], + data[phase_index] if data else None, reference, reference_identity, current_state, @@ -337,6 +349,7 @@ def commit(state): "incoming_weight_hash": incoming_weight_hash, **initialization, **artifacts.describe(), + "resource_accounting": prepared.accounting.snapshot(), }, ) return PhaseOutcome((selected, result, position), result.stop_scope) @@ -348,12 +361,24 @@ def commit_phase(index, phase, outcome): recovery.finish_phase(actor, result, position) recovery.save(checkpoint, actor.device) _publish_phase_report(recovery.completed[-1], checkpoint) + if not phase_only and index == last_data_phase: + release_phase_data() + + last_data_phase = max( + (index for index, part in enumerate(data) if part is not None), default=-1 + ) + + def release_phase_data(): + data.clear() + prepared.release_data() completion_reason = None if phase_only: if not recovery.completed: execute_phase(0, config.phases[0], run_phase, commit_phase) else: + if last_data_phase < recovery.phase_index: + release_phase_data() completion_reason = run_phases( config.phases, run_phase, @@ -363,6 +388,8 @@ def commit_phase(index, phase, outcome): ) results = tuple(record.report_result() for record in recovery.completed) if completion_reason is not None: + data.clear() + prepared.close() report = run_report( model_path=config.model_path, checkpoint_path=checkpoint, @@ -379,6 +406,7 @@ def commit_phase(index, phase, outcome): reference_provenance=recovery.reference_contract, initialization=recovery.initialization, resume=config.resume.model_dump(mode="json"), + resource_accounting=prepared.accounting.snapshot(), ) save_policy(actor.eval(), config.model_path, metadata=report) publish_run_report(report, config.report_path) diff --git a/src/sequifier/rl/parallel_grpo_collector.py b/src/sequifier/rl/parallel_grpo_collector.py index a2d9272f..f5b74b8c 100644 --- a/src/sequifier/rl/parallel_grpo_collector.py +++ b/src/sequifier/rl/parallel_grpo_collector.py @@ -6,32 +6,33 @@ import torch from sequifier.io.actions import action_record, decode_action -from sequifier.io.batch import collate_model_inputs from sequifier.rl.collection import PolicyDecision from sequifier.rl.environment import validate_start_token from sequifier.rl.group_collection import GroupCollection from sequifier.rl.history import frame_storage_bytes from sequifier.rl.simulator_workers import require_parallel_group_start +from sequifier.rl.window_storage import collated_windows @torch.no_grad() def _decide(actor, reference, cursors): - inputs = collate_model_inputs([cursor.inputs for cursor in cursors]).to( - actor.device - ) - distribution = actor.distribution(inputs) - generators = [cursor.generator for cursor in cursors] - # Decode and validate distributions across the full environment batch. - # Only random draws use individual generators; no per-row decoder calls. - selected = distribution.select_and_score(False, generators) - actions = selected.actions - scores = ( - torch.stack( - [selected.log_prob, reference.evaluate_actions(inputs, actions)], dim=1 + with collated_windows( + (cursor.inputs for cursor in cursors), actor.device, borrowed=True + ) as inputs: + distribution = actor.distribution(inputs) + generators = [cursor.generator for cursor in cursors] + # Decode and validate distributions across the full environment batch. + # Only random draws use individual generators; no per-row decoder calls. + selected = distribution.select_and_score(False, generators) + actions = selected.actions + scores = ( + torch.stack( + [selected.log_prob, reference.evaluate_actions(inputs, actions)], dim=1 + ) + .cpu() + .tolist() ) - .cpu() - .tolist() - ) + del inputs, distribution host_actions = {name: value.cpu() for name, value in actions.items()} decisions = [] for index, (old, ref) in enumerate(scores): @@ -59,6 +60,8 @@ def collect_parallel_groups( generation, env_steps, restart_stream, + *, + storage=None, ): actor, reference, phase = ( context.actor.eval(), @@ -68,7 +71,7 @@ def collect_parallel_groups( require_parallel_group_start(adapter) pending, active, completed = deque(), {}, {} with GroupCollection( - context, adapter, count, generation, env_steps, restart_stream + context, adapter, count, generation, env_steps, restart_stream, storage=storage ) as collection: from sequifier.rl.history import _json_payload_bytes @@ -95,14 +98,15 @@ def collect_parallel_groups( resetting.append(slot) for slot in resetting: state = active[slot] - state.cursor.reset( - workers.receive(slot), - f"group-{state.group}-sample-{state.sample}", - ) - frame = state.cursor.history.current - collection.validate_initial(state) - # Include all live history before any environment step is issued. - account(frame_storage_bytes(frame) + 1024) + with workers.response(slot) as response: + state.cursor.reset( + response, f"group-{state.group}-sample-{state.sample}" + ) + del response + frame = state.cursor.history.current + collection.validate_initial(state) + # Include live history before any environment step is issued. + account(frame_storage_bytes(frame) + 1024) account( sum( value.numel() * value.element_size() @@ -110,7 +114,8 @@ def collect_parallel_groups( *state.cursor.inputs.inputs.values(), *state.cursor.inputs.metadata.values(), ) - ) + ), + window=True, ) decisions = {} slots = sorted(active) @@ -128,18 +133,23 @@ def collect_parallel_groups( for slot, (decision, ref) in decisions.items(): state = active[slot] frame = state.cursor.history.current - result = state.cursor.accept_step(decision, workers.receive(slot)) - state.append(frame, decision, ref, result, phase.reward_scale) - # Account the successor retained by a live cursor, even if the - # episode ends or the collector's finite horizon is reached. - if result.observation is not None: - account(frame_storage_bytes(state.cursor.history.current) + 1024) + with workers.response(slot) as response: + result = state.cursor.accept_step(decision, response) + del response + state.append(frame, decision, ref, result, phase.reward_scale) + # Include the successor even on the last step of a trajectory. + if result.observation is not None: + account( + frame_storage_bytes(state.cursor.history.current) + 1024 + ) if ( result.terminated or result.truncated or len(state.actions) == phase.max_steps ): - completed[state.group].append(state.seal(generation, result)) + completed[state.group].append( + collection.seal_trajectory(state, result) + ) del active[slot] return collection.finish( [collection.seal_group(group) for group in completed.values()] diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index aa3da5f3..f9c423c5 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -14,6 +14,7 @@ from sequifier.rl.ppo_update import update_ppo from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal from sequifier.rl.snapshots import snapshots_equal +from sequifier.runtime.resource_accounting import allocation def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): @@ -37,6 +38,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): raise ValueError("PPO checkpoint is not a complete collection/update boundary") actor.ppo_policy_version = session.transaction adapter = load_online_adapter(phase.environment) + collector = None try: if context.run.resume.mode == "exact": require_exact_snapshot(adapter) @@ -66,14 +68,20 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): session.evaluate_online(adapter, initial=True) while env_steps < phase.total_env_steps and not session.stopping.triggered: horizon = min(phase.rollout_env_steps, phase.total_env_steps - env_steps) - buffer, episodes = collector.collect(horizon, session.transaction) - if phase.archive_directory: - buffer.archive( - Path(phase.archive_directory) - / f"collection-{session.transaction + 1:08d}.json", - include_scores=phase.archive_scores, + with allocation("rollout_storage") as storage: + buffer, episodes = collector.collect( + horizon, session.transaction, storage=storage ) - record = update_ppo(session, value, buffer) + if phase.archive_directory: + with allocation("rollout_archive") as archive: + archive.resize(buffer.estimated_bytes) + buffer.archive( + Path(phase.archive_directory) + / f"collection-{session.transaction + 1:08d}.json", + include_scores=phase.archive_scores, + ) + record = update_ppo(session, value, buffer) + del buffer env_steps += horizon session.transaction += 1 final = env_steps == phase.total_env_steps @@ -108,6 +116,8 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): adapter_identity=identity, ) finally: + if collector is not None: + collector.cursor.close() adapter.close() return session.finish( { diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py index f06f640b..1b80ff12 100644 --- a/src/sequifier/rl/ppo_buffer.py +++ b/src/sequifier/rl/ppo_buffer.py @@ -1,17 +1,16 @@ """Compact, single-generation on-policy storage and fixed FP32 GAE targets.""" import json +from contextlib import contextmanager from dataclasses import asdict, dataclass -from typing import cast import torch from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.io.actions import action_batch, decode_action from sequifier.io.atomic import write_json_atomic -from sequifier.io.batch import SequifierInputs -from sequifier.io.tensors import collate_tensor_records from sequifier.rl.history import frame_storage_bytes, history_inputs +from sequifier.rl.window_storage import collated_windows @dataclass(frozen=True) @@ -73,40 +72,43 @@ def window(self, reference): return history_inputs(self.spec, self.frames[start : reference + 1]) def batch(self, indices, device): + with self.batch_scope(indices, device) as batch: + return batch + + @contextmanager + def batch_scope(self, indices, device): indices = [int(index) for index in indices] - return ( - cast( - SequifierInputs, - collate_tensor_records( - [self.window(self.decisions[index].frame) for index in indices] + with collated_windows( + (self.window(self.decisions[index].frame) for index in indices), device + ) as inputs: + yield ( + inputs, + action_batch( + [self.decisions[index].action for index in indices], + self.spec, + device=device, ), - ).to(device), - action_batch( - [self.decisions[index].action for index in indices], - self.spec, - device=device, - ), - torch.tensor( - [self.decisions[index].old_log_probability for index in indices], - dtype=torch.float32, - device=device, - ), - torch.tensor( - [self.decisions[index].old_value for index in indices], - dtype=torch.float32, - device=device, - ), - torch.tensor( - [self.advantages[index] for index in indices], - dtype=torch.float32, - device=device, - ), - torch.tensor( - [self.targets[index] for index in indices], - dtype=torch.float32, - device=device, - ), - ) + torch.tensor( + [self.decisions[index].old_log_probability for index in indices], + dtype=torch.float32, + device=device, + ), + torch.tensor( + [self.decisions[index].old_value for index in indices], + dtype=torch.float32, + device=device, + ), + torch.tensor( + [self.advantages[index] for index in indices], + dtype=torch.float32, + device=device, + ), + torch.tensor( + [self.targets[index] for index in indices], + dtype=torch.float32, + device=device, + ), + ) def archive(self, path, *, include_scores): """A separate versioned collection record, never an offline update source.""" @@ -224,7 +226,9 @@ def compute_gae(decisions, gamma, gae_lambda, normalize): class PPOBufferBuilder: - def __init__(self, spec, version, horizon, initial_frames, max_bytes): + def __init__( + self, spec, version, horizon, initial_frames, max_bytes, *, storage=None + ): self.spec = spec.model_copy(deep=True) self.version = version self.horizon = horizon @@ -244,6 +248,14 @@ def __init__(self, spec, version, horizon, initial_frames, max_bytes): self.used_bytes = sum(frame_storage_bytes(frame) for frame in self.frames) self.sealed = False self.decision_bytes = 0 + self.storage = storage + self._account() + + def _account(self): + if self.storage is not None: + self.storage.resize( + max(self.estimated_bytes, self.used_bytes + self.decision_bytes) + ) def append_frame(self, frame): if self.sealed: @@ -255,6 +267,7 @@ def append_frame(self, frame): raise ValueError("PPO history exceeded its bounded frame capacity") self.frames.append(frame) self.used_bytes += cost + self._account() return len(self.frames) - 1 def append(self, decision): @@ -266,6 +279,7 @@ def append(self, decision): json.dumps(asdict(decision), allow_nan=False).encode() ) self.decisions.append(decision) + self._account() if self.used_bytes + self.decision_bytes > self.max_bytes: raise ValueError("PPO decisions exceeded the memory budget") diff --git a/src/sequifier/rl/ppo_collector.py b/src/sequifier/rl/ppo_collector.py index d9920245..ef943b34 100644 --- a/src/sequifier/rl/ppo_collector.py +++ b/src/sequifier/rl/ppo_collector.py @@ -32,7 +32,7 @@ def reset(self): f"train-{self.reset_count}", ) - def collect(self, horizon, policy_version): + def collect(self, horizon, policy_version, *, storage=None): if self.actor.ppo_policy_version != policy_version: raise ValueError("collection policy generation disagrees with the actor") self.actor.eval() @@ -54,6 +54,7 @@ def collect(self, horizon, policy_version): horizon, self.cursor.history.frames, self.phase.max_buffer_bytes, + storage=storage, ) current_reference = len(builder.frames) - 1 completed = [] diff --git a/src/sequifier/rl/ppo_evaluation.py b/src/sequifier/rl/ppo_evaluation.py index 2964d7c7..4debd850 100644 --- a/src/sequifier/rl/ppo_evaluation.py +++ b/src/sequifier/rl/ppo_evaluation.py @@ -111,6 +111,7 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): else None, } ) + cursor.close() except EvaluationTimeout: timed_out = True finally: diff --git a/src/sequifier/rl/ppo_update.py b/src/sequifier/rl/ppo_update.py index 7d9a3d28..82232891 100644 --- a/src/sequifier/rl/ppo_update.py +++ b/src/sequifier/rl/ppo_update.py @@ -53,15 +53,18 @@ def validate_collection_scores(actor, value, buffer, batch_size): actor.eval() value.eval() for start in range(0, len(buffer), batch_size): - history, actions, old_logp, old_v, _, _ = buffer.batch( - range(start, min(start + batch_size, len(buffer))), actor.device - ) - if not torch.allclose( - actor.evaluate_actions(history, actions), old_logp, atol=1e-5, rtol=1e-4 - ): - raise ValueError("PPO reconstructed action scores differ from collection") - if not torch.allclose(value(history), old_v, atol=1e-5, rtol=1e-4): - raise ValueError("PPO reconstructed values differ from collection") + indices = range(start, min(start + batch_size, len(buffer))) + with buffer.batch_scope(indices, actor.device) as batch: + history, actions, old_logp, old_v, *_ = batch + if not torch.allclose( + actor.evaluate_actions(history, actions), old_logp, atol=1e-5, rtol=1e-4 + ): + raise ValueError( + "PPO reconstructed action scores differ from collection" + ) + if not torch.allclose(value(history), old_v, atol=1e-5, rtol=1e-4): + raise ValueError("PPO reconstructed values differ from collection") + del batch, history def update_ppo(session, value, buffer): @@ -91,44 +94,50 @@ def update_ppo(session, value, buffer): kl_sum = 0.0 with torch.no_grad(): for indices in minibatches: - history, actions, old_logp, _, _, _ = buffer.batch( - indices, actor.device - ) - log_ratio = actor.evaluate_actions(history, actions) - old_logp - kl = log_ratio.exp() - 1 - log_ratio - if not torch.isfinite(kl).all(): - raise ValueError("non-finite PPO KL") - kl_sum += float(kl.sum()) + with buffer.batch_scope(indices, actor.device) as batch: + history, actions, old_logp, _, _, _ = batch + log_ratio = ( + actor.evaluate_actions(history, actions) - old_logp + ) + kl = log_ratio.exp() - 1 - log_ratio + if not torch.isfinite(kl).all(): + raise ValueError("non-finite PPO KL") + kl_sum += float(kl.sum()) + del batch, history if kl_sum / len(group) > phase.old_policy_target_kl: stopped = True break session.zero_grad() for indices in minibatches: - history, actions, old_logp, old_v, advantages, target = buffer.batch( - indices, actor.device - ) - distribution = actor.distribution(history) - policy_loss, value_loss, metrics = ppo_objectives( - distribution.log_prob(actions), - old_logp, - advantages, - value(history), - old_v, - target, - distribution.entropy(), - phase, - ) - actor_loss = ( - policy_loss - phase.entropy_coefficient * metrics["entropy"] - ).sum() / len(group) - critic_loss = phase.value_coefficient * value_loss.sum() / len(group) - if not torch.isfinite(actor_loss) or not torch.isfinite(critic_loss): - raise ValueError("non-finite weighted PPO loss") - actor_loss.backward() - critic_loss.backward() - totals.add(metrics) - ratio_chunks.append(metrics["ratio"].detach().cpu()) - count += len(indices) + with buffer.batch_scope(indices, actor.device) as batch: + history, actions, old_logp, old_v, advantages, target = batch + distribution = actor.distribution(history) + policy_loss, value_loss, metrics = ppo_objectives( + distribution.log_prob(actions), + old_logp, + advantages, + value(history), + old_v, + target, + distribution.entropy(), + phase, + ) + actor_loss = ( + policy_loss - phase.entropy_coefficient * metrics["entropy"] + ).sum() / len(group) + critic_loss = ( + phase.value_coefficient * value_loss.sum() / len(group) + ) + if not torch.isfinite(actor_loss) or not torch.isfinite( + critic_loss + ): + raise ValueError("non-finite weighted PPO loss") + actor_loss.backward() + critic_loss.backward() + totals.add(metrics) + ratio_chunks.append(metrics["ratio"].detach().cpu()) + count += len(indices) + del batch, history, distribution for role, norm in session.step().items(): norms[role].append(norm) epochs = epoch + 1 @@ -143,11 +152,10 @@ def update_ppo(session, value, buffer): with torch.no_grad(): predictions = [] for start in range(0, len(buffer), phase.batch_size): - history, *_ = buffer.batch( - range(start, min(start + phase.batch_size, len(buffer))), - actor.device, - ) - predictions.append(value(history).cpu()) + indices = range(start, min(start + phase.batch_size, len(buffer))) + with buffer.batch_scope(indices, actor.device) as batch: + predictions.append(value(batch[0]).cpu()) + del batch residual_variance = (targets - torch.cat(predictions)).var(unbiased=False) target_variance = targets.var(unbiased=False) result: dict[str, Any] = totals.means(count) diff --git a/src/sequifier/rl/prepared_inputs.py b/src/sequifier/rl/prepared_inputs.py index 041a95eb..83e46ccc 100644 --- a/src/sequifier/rl/prepared_inputs.py +++ b/src/sequifier/rl/prepared_inputs.py @@ -1,5 +1,6 @@ """Run-owned source identities, episode catalogs and policy partitions.""" +from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING @@ -7,6 +8,9 @@ from sequifier.config.policy_config import PolicyBinding from sequifier.config.policy_inputs import PolicyFittingSpec from sequifier.fingerprints import fingerprint +from sequifier.io.episode_cache import EpisodeCache +from sequifier.io.memory_budget import resident_bytes +from sequifier.runtime.resource_accounting import ResourceAccounting from sequifier.trajectory.sources import ( ResolvedSequenceSource, file_checksum, @@ -92,6 +96,61 @@ def __init__(self): self._partitions = {} self._catalogs = {} self._preference_manifests = {} + self._episode_caches = [] + self.accounting = ResourceAccounting() + self._prepared_storage = self.accounting.allocation("prepared_data") + self._running = False + + def episode_cache(self, capacity): + cache = EpisodeCache(capacity, self.accounting.allocation("episode_caches")) + self._episode_caches.append(cache) + return cache + + def measure(self): + roots = ( + self._inputs, + self._sources, + self._checksums, + self._partitions, + self._catalogs, + self._preference_manifests, + ) + seen = {id(self), *(id(cache) for cache in self._episode_caches)} + self._prepared_storage.resize(resident_bytes(roots, seen)) + + def _clear_data(self): + for cache in self._episode_caches: + cache.clear() + self._episode_caches.clear() + self._partitions.clear() + self._catalogs.clear() + self._preference_manifests.clear() + + def release_data(self): + self._clear_data() + self.measure() + + def close(self): + self._clear_data() + self._inputs.clear() + self._sources.clear() + # Keep the resolved config's checksum baseline for subsequent runs. + self._prepared_storage.resize(0) + + @contextmanager + def run_scope(self): + if self._running: + yield + return + self._running = True + try: + with self.accounting.activate(): + yield + finally: + self.close() + self._running = False + self.accounting = ResourceAccounting() + self._prepared_storage = self.accounting.allocation("prepared_data") @classmethod def for_run(cls, config): @@ -223,6 +282,7 @@ def prepare_phase(self, config, phase, policy, inputs, *, fitting=False): policy, config.episode_cache_size, preference_manifests=self._preference_manifests, + cache_owner=self, ) self._partitions[key] = _PolicyPartitions(self._catalogs[catalog_key]) partitions = self._partitions[key] @@ -230,6 +290,8 @@ def prepare_phase(self, config, phase, policy, inputs, *, fitting=False): return prepare(config, phase, policy, partitions.catalog, partitions.prepared) def prepare(self, config, policy, inputs): - return [ + data = [ self.prepare_phase(config, phase, policy, inputs) for phase in config.phases ] + self.measure() + return data diff --git a/src/sequifier/rl/scoring.py b/src/sequifier/rl/scoring.py index 2156721e..7f800b55 100644 --- a/src/sequifier/rl/scoring.py +++ b/src/sequifier/rl/scoring.py @@ -1,12 +1,10 @@ """Bounded differentiable scores for causal action sequences.""" -from typing import cast - import torch from sequifier.io.actions import action_batch -from sequifier.io.batch import SequifierInputs, collate_model_inputs from sequifier.io.policy_window import PolicyWindowView +from sequifier.rl.window_storage import collated_windows def sequence_log_probabilities(actor, episode, start, end, chunk_size): @@ -15,22 +13,20 @@ def sequence_log_probabilities(actor, episode, start, end, chunk_size): view = PolicyWindowView(actor.config) for offset in range(start, end, chunk_size): positions = range(offset, min(offset + chunk_size, end)) - inputs = cast( - SequifierInputs, - collate_model_inputs( - [view.episode_inputs(episode, t, None) for t in positions] - ), - ).to(actor.device) - actions = action_batch( - [episode.actions[t] for t in positions], - actor.config, - device=actor.device, - encoded=False, - ) - scores = actor.evaluate_actions(inputs, actions).float() - if not torch.isfinite(scores).all(): - raise ValueError("non-finite sequence action log likelihood") - yield scores + with collated_windows( + (view.episode_inputs(episode, t, None) for t in positions), actor.device + ) as inputs: + actions = action_batch( + [episode.actions[t] for t in positions], + actor.config, + device=actor.device, + encoded=False, + ) + scores = actor.evaluate_actions(inputs, actions).float() + del inputs + if not torch.isfinite(scores).all(): + raise ValueError("non-finite sequence action log likelihood") + yield scores def sequence_score(actor, episode, start, end, chunk_size): diff --git a/src/sequifier/rl/simulator_workers.py b/src/sequifier/rl/simulator_workers.py index c86a29c8..0736ef4c 100644 --- a/src/sequifier/rl/simulator_workers.py +++ b/src/sequifier/rl/simulator_workers.py @@ -1,14 +1,19 @@ """Opt-in spawned simulators with explicit requests, deadlines and bounded cleanup.""" import multiprocessing +import pickle import time import traceback +from contextlib import contextmanager +from multiprocessing.reduction import ForkingPickler +from sequifier.io.memory_budget import resident_bytes from sequifier.rl.environment import ( adapter_identity, load_online_adapter, require_group_start, ) +from sequifier.runtime.resource_accounting import allocation def require_parallel_group_start(adapter): @@ -59,6 +64,7 @@ def __init__(self, spec, count, timeout, threads, expected_identity): self.timeout = timeout self.connections, self.processes = [], [] self.pending, self.sequence = {}, 0 + self.requests = {} self.closed = False try: context = multiprocessing.get_context("spawn") @@ -101,29 +107,52 @@ def send(self, index, method, *args): self.sequence += 1 self.pending[index] = (self.sequence, method, time.monotonic() + self.timeout) try: - self.connections[index].send((self.sequence, method, args)) + payload = ForkingPickler.dumps((self.sequence, method, args)) + storage = self.requests[index] = allocation("worker_payloads") + storage.resize(len(payload)) + self.connections[index].send_bytes(payload) except (OSError, EOFError) as exc: raise RuntimeError( f"simulator {index} failed while sending {method}" ) from exc def receive(self, index): + with self.response(index) as result: + return result + + @contextmanager + def response(self, index): if self.closed or index not in self.pending: raise RuntimeError("simulator has no pending response") request, method, deadline = self.pending[index] connection = self.connections[index] - try: - if not connection.poll(max(0.0, deadline - time.monotonic())): - raise TimeoutError(f"simulator {index} timed out during {method}") - received, ok, result = connection.recv() - except (OSError, EOFError) as exc: - raise RuntimeError(f"simulator {index} exited during {method}") from exc - if received != request: - raise RuntimeError(f"simulator {index} returned an out-of-order response") - del self.pending[index] - if not ok: - raise RuntimeError(f"simulator {index} failed during {method}:\n{result}") - return result + with allocation("worker_payloads") as storage: + try: + if not connection.poll(max(0.0, deadline - time.monotonic())): + raise TimeoutError(f"simulator {index} timed out during {method}") + payload = connection.recv_bytes() + storage.resize(len(payload)) + received, ok, result = pickle.loads(payload) + except (OSError, EOFError) as exc: + raise RuntimeError(f"simulator {index} exited during {method}") from exc + if received != request: + raise RuntimeError( + f"simulator {index} returned an out-of-order response" + ) + del self.pending[index] + request_storage = self.requests.pop(index, None) + if request_storage is not None: + request_storage.close() + if not ok: + raise RuntimeError( + f"simulator {index} failed during {method}:\n{result}" + ) + if storage.accounting is not None: + decoded_size = resident_bytes(result) + storage.resize(len(payload) + decoded_size) + storage.resize(decoded_size) + del payload + yield result def close(self): if self.closed: @@ -154,3 +183,6 @@ def close(self): for connection in self.connections: connection.close() self.pending.clear() + for storage in self.requests.values(): + storage.close() + self.requests.clear() diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index 47f23bb3..bed22d19 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -3,16 +3,19 @@ import torch from sequifier.config.policy_plan import PolicyImitationPhaseConfig -from sequifier.io.batch import collate_sequence_batches from sequifier.io.policy_dataset import imitation_example from sequifier.rl.batch_order import BatchOrder from sequifier.rl.batching import batches as minibatches from sequifier.rl.offline_data import OfflineData from sequifier.rl.run_state import PhaseContext, PhaseSession +from sequifier.rl.window_storage import collated_windows def action_nll(actor, batch): - batch = batch.to(actor.device) + return _action_nll(actor, batch.to(actor.device)) + + +def _action_nll(actor, batch): actions = {name: value[:, -1] for name, value in batch.targets.items()} return -actor.evaluate_actions(batch.model_inputs, actions).float() @@ -22,13 +25,12 @@ def evaluate_imitation(actor, dataset, batch_size): actor.eval() total = 0.0 for start in range(0, len(dataset), batch_size): - batch = collate_sequence_batches( - [ - imitation_example(dataset, i) - for i in range(start, min(start + batch_size, len(dataset))) - ] - ) - total += float(action_nll(actor, batch).sum()) + indices = range(start, min(start + batch_size, len(dataset))) + with collated_windows( + (imitation_example(dataset, i) for i in indices), actor.device + ) as batch: + total += float(_action_nll(actor, batch).sum()) + del batch return total / len(dataset) @@ -72,14 +74,15 @@ def evaluate(**record): session.zero_grad() total = 0.0 for indices in minibatches(group, phase.batch_size): - batch = collate_sequence_batches( - [imitation_example(training, i) for i in indices] - ) - loss = action_nll(actor, batch).sum() / len(group) - if not torch.isfinite(loss): - raise ValueError("non-finite imitation action likelihood") - loss.backward() - total += float(loss.detach()) + with collated_windows( + (imitation_example(training, i) for i in indices), actor.device + ) as batch: + loss = _action_nll(actor, batch).sum() / len(group) + if not torch.isfinite(loss): + raise ValueError("non-finite imitation action likelihood") + loss.backward() + total += float(loss.detach()) + del batch session.step() session.complete_update( final=batches.epoch == phase.epochs, diff --git a/src/sequifier/rl/window_storage.py b/src/sequifier/rl/window_storage.py new file mode 100644 index 00000000..cf194563 --- /dev/null +++ b/src/sequifier/rl/window_storage.py @@ -0,0 +1,59 @@ +"""Account for reconstructed windows through collation, transfer and scoring.""" + +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from dataclasses import fields +from typing import cast + +import torch + +from sequifier.io.tensors import Record, TensorRecord, collate_tensor_records +from sequifier.runtime.resource_accounting import allocation + + +def tensor_storage_bytes(value, seen=None): + if seen is None: + seen = set() + if value is None: + return 0 + if isinstance(value, dict): + return sum(tensor_storage_bytes(item, seen) for item in value.values()) + if isinstance(value, TensorRecord): + return sum( + tensor_storage_bytes(getattr(value, field.name), seen) + for field in fields(value) + ) + storage = value.untyped_storage() + identity = storage._cdata + if identity in seen: + return 0 + seen.add(identity) + return storage.nbytes() + + +@contextmanager +def collated_windows( + windows: Iterable[Record], device: str | torch.device, *, borrowed: bool = False +) -> Iterator[Record]: + with allocation("reconstructed_windows") as batch_storage: + with allocation("reconstructed_windows") as rows_storage: + rows, seen, size = [], set(), 0 + for window in windows: + rows.append(window) + if not borrowed and rows_storage.accounting is not None: + size += tensor_storage_bytes(window, seen) + rows_storage.resize(size) + del window + batch = cast(Record, collate_tensor_records(rows)) + if batch_storage.accounting is not None: + batch_storage.resize(tensor_storage_bytes(batch)) + rows.clear() + + moved = batch.to(device) + if batch_storage.accounting is not None: + seen = set() + moved_size = tensor_storage_bytes(moved, seen) + batch_storage.resize(moved_size + tensor_storage_bytes(batch, seen)) + batch_storage.resize(moved_size) + del batch + yield moved diff --git a/src/sequifier/runtime/resource_accounting.py b/src/sequifier/runtime/resource_accounting.py new file mode 100644 index 00000000..ca3cd9f1 --- /dev/null +++ b/src/sequifier/runtime/resource_accounting.py @@ -0,0 +1,72 @@ +"""Run-local estimates for policy data and working payloads.""" + +from __future__ import annotations + +from collections import Counter +from contextlib import contextmanager +from contextvars import ContextVar + +_active: ContextVar[ResourceAccounting | None] = ContextVar( + "policy_resource_accounting", default=None +) + + +class ResourceAccounting: + def __init__(self): + self.current: Counter[str] = Counter() + self.peaks: Counter[str] = Counter() + self.total = 0 + self.peak_total = 0 + + @contextmanager + def activate(self): + token = _active.set(self) + try: + yield + finally: + _active.reset(token) + + def allocation(self, category): + return Allocation(self, category) + + def snapshot(self): + return { + "current_bytes": {key: self.current[key] for key in sorted(self.peaks)}, + "peak_bytes": dict(sorted(self.peaks.items())), + "current_total_bytes": self.total, + "peak_total_bytes": self.peak_total, + } + + +class Allocation: + def __init__(self, accounting, category): + self.accounting = accounting + self.category = category + self.size = 0 + + def resize(self, size): + if self.accounting is None or size == self.size: + return + accounting, category = self.accounting, self.category + delta = size - self.size + self.size = size + accounting.current[category] += delta + accounting.total += delta + accounting.peaks[category] = max( + accounting.peaks[category], accounting.current[category] + ) + accounting.peak_total = max(accounting.peak_total, accounting.total) + + def close(self): + self.resize(0) + self.accounting = None + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + +def allocation(category): + return Allocation(_active.get(), category) diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index f3b1d3e4..a3d8fe3c 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -129,6 +129,14 @@ def _completed_report(item): def run_learning_plan(config) -> LearningRunResult: + from sequifier.rl.prepared_inputs import PreparedTrainingInputs + + prepared = PreparedTrainingInputs.for_run(config) + with prepared.run_scope(): + return _run_learning_plan(config, prepared) + + +def _run_learning_plan(config, prepared) -> LearningRunResult: phases = config.training_plan if any(phase.optimizer_state == "continue" for phase in phases): raise ValueError("portable phase handoffs cannot continue optimizer state") @@ -196,10 +204,9 @@ def run_learning_plan(config) -> LearningRunResult: prepare_data, validate_output_paths, ) - from sequifier.rl.prepared_inputs import PreparedTrainingInputs - prepared_inputs = PreparedTrainingInputs.for_run(config) dependencies = {} + last_data_phase = -1 protected_inputs = set() for index, phase in enumerate(phases): if index == 0: @@ -231,14 +238,17 @@ def run_learning_plan(config) -> LearningRunResult: destination, False, ) - inputs = prepared_inputs.resolve(run) + inputs = prepared.resolve(run) inputs.revalidate(inputs.identity, f"phases.{phase.name}.source_identity") validate_output_paths(run, inputs) dependencies.update(inputs.identity) preflight_environments(run) - prepare_data(run, run.policy, inputs) + if prepare_data(run, run.policy, inputs)[0] is not None: + last_data_phase = index if run.execution_model is None or run.policy_interface is None: raise ValueError("a policy phase has no resolved model interface") + if last_data_phase < len(state["completed"]): + prepared.release_data() if "dependencies" in state and state["dependencies"] != dependencies: raise ValueError("learning plan source identities changed since its checkpoint") state["dependencies"] = dependencies @@ -382,6 +392,8 @@ def commit_phase(index, phase, outcome): ) state.pop("active_run_id", None) save_boundary() + if index == last_data_phase: + prepared.release_data() migrated = False for index, item in enumerate(state["completed"]): @@ -414,6 +426,7 @@ def commit_phase(index, phase, outcome): result.pop("report_receipt", None) result["report"] = _completed_report(item) completed.append(result) + prepared.close() publish_run_report( run_report( model_path=config.model_path, @@ -428,6 +441,7 @@ def commit_phase(index, phase, outcome): exports=[config.model_path], status="complete", completion_reason=completion_reason, + resource_accounting=prepared.accounting.snapshot(), ), config.report_path, ) From cd7936adfb5840e6512f22045769529f321631f4 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 14:25:11 +0100 Subject: [PATCH 56/71] unify contracts --- .github/workflows/contract-versions.yml | 23 ++ documentation/configs/infer.md | 4 +- documentation/configs/preprocess.md | 5 +- documentation/configs/train.md | 2 +- .../ddconfigs/preprocessing_input.json | 3 +- .../configs/rollout-ppo-toy.yaml | 2 +- .../configs/train-multiphase-control.yaml | 8 +- .../example_inputs/iql_toy_control.py | 7 +- .../example_inputs/policy_control_data.py | 8 +- documentation/policy-learning.md | 11 +- documentation/rollout.md | 4 +- documentation/supervised-policy-learning.md | 4 +- documentation/training/artifacts.md | 17 +- .../training/runtime-architecture.md | 4 +- release-notes/unreleased-contract-versions.md | 33 ++ release-notes/unreleased-phase-ownership.md | 2 +- release-notes/unreleased-policy-training.md | 4 +- ...nreleased-supervised-policy-unification.md | 2 +- .../unreleased-training-artifact-contracts.md | 4 +- .../artifacts/backbone_repository.py | 10 +- .../artifacts/checkpoint_envelope.py | 55 ++-- src/sequifier/artifacts/model_artifact.py | 47 +-- src/sequifier/artifacts/model_config.py | 72 +++-- src/sequifier/artifacts/phase.py | 6 + src/sequifier/artifacts/reports.py | 5 +- src/sequifier/artifacts/run_checkpoint.py | 73 +++-- src/sequifier/config/infer_config.py | 68 ++-- src/sequifier/config/inference_common.py | 19 +- src/sequifier/config/metadata.py | 88 +++++- src/sequifier/config/policy_plan.py | 5 +- src/sequifier/config/rollout_config.py | 3 +- src/sequifier/config/train_config.py | 33 +- src/sequifier/contract_versions.py | 72 +++++ src/sequifier/export/onnx.py | 3 +- src/sequifier/export/preflight.py | 18 +- src/sequifier/helpers.py | 25 +- src/sequifier/hyperparameter_search.py | 18 +- src/sequifier/infer.py | 29 +- src/sequifier/io/depth_preprocess.py | 37 ++- src/sequifier/io/folder_dataset.py | 2 + src/sequifier/io/images.py | 14 +- src/sequifier/io/memory_budget.py | 2 + src/sequifier/io/policy_window.py | 6 +- src/sequifier/io/pt_payload.py | 90 ++++-- src/sequifier/model/execution_schema.py | 19 +- src/sequifier/model/hash_codec.py | 9 +- src/sequifier/model/initialization_seed.py | 7 +- src/sequifier/preprocess.py | 54 ++-- src/sequifier/rl/collection.py | 3 +- src/sequifier/rl/control.py | 7 +- src/sequifier/rl/environment.py | 8 +- src/sequifier/rl/grpo.py | 3 +- src/sequifier/rl/learner.py | 18 +- src/sequifier/rl/ppo_buffer.py | 9 +- src/sequifier/rl/run_checkpoint.py | 87 ++--- src/sequifier/rl/run_state.py | 94 +----- src/sequifier/rl/runtime.py | 3 +- src/sequifier/rollout/coordinator.py | 2 + src/sequifier/rollout/factories.py | 12 +- src/sequifier/runtime/builder.py | 2 - src/sequifier/templates/train-grpo.yaml | 4 +- src/sequifier/templates/train-ppo.yaml | 4 +- src/sequifier/training/checkpoint_service.py | 4 +- src/sequifier/training/coordinator.py | 88 ++---- src/sequifier/training/metrics.py | 42 ++- src/sequifier/training/phase_runners.py | 7 +- src/sequifier/trajectory/archives.py | 51 +-- src/sequifier/trajectory/export.py | 3 +- src/sequifier/trajectory/preferences.py | 18 +- src/sequifier/trajectory/schema.py | 27 +- src/sequifier/trajectory/sources.py | 14 +- src/sequifier/visualize_training.py | 13 +- tools/check_contract_versions.py | 291 +++++++++++++++++ tools/convert_to_pt.py | 143 ++++----- tools/resize_pt_files.py | 298 ++++++------------ 75 files changed, 1353 insertions(+), 938 deletions(-) create mode 100644 .github/workflows/contract-versions.yml create mode 100644 release-notes/unreleased-contract-versions.md create mode 100644 src/sequifier/contract_versions.py create mode 100644 tools/check_contract_versions.py diff --git a/.github/workflows/contract-versions.yml b/.github/workflows/contract-versions.yml new file mode 100644 index 00000000..05412695 --- /dev/null +++ b/.github/workflows/contract-versions.yml @@ -0,0 +1,23 @@ +name: Contract versions + +on: + pull_request: + merge_group: + +permissions: + contents: read + +jobs: + contract-versions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Validate allocations and shipped examples on the merge result + env: + TARGET_SHA: ${{ github.event.merge_group.base_sha || github.event.pull_request.base.sha }} + run: python tools/check_contract_versions.py --base "$TARGET_SHA" diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index 74ea90a3..cfab48e1 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -304,7 +304,7 @@ selection, read/write formats, device, seed, `enforce_determinism` and batch siz | `previous_action_columns` | {}; recorded lagged-action column mappings for fragments | | `sequence_id_column`, `item_position_column` | `sequenceId`, `itemPosition`; raw history ordering | | `output_probabilities` | false; include joint log probability and write separate target-specific categorical probability files | -| `rollout` | null; execute in a version-3 online environment instead of writing prediction rows | +| `rollout` | null; execute in an online policy environment instead of writing prediction rows | Optional assertions `metadata_config_path`, `input_columns`, `static_columns`, `target_columns`, `column_data_types`, `target_column_types`, `training_objective`, @@ -318,7 +318,7 @@ reject generative sampling, decoding, probabilities, autoregression and rollout. Policy rollout declares `environment: {factory: module:factory, config: {}, implementation_id: ..., units: {}}`, required `max_steps`, `seed_row_index` and `trajectory_path`, plus optional `max_episode_steps`. It rejects forecast `bindings`. -Forecast rollout uses the distinct version-1 environment, requires bindings and +Forecast rollout uses the forecast rollout adapter protocol, requires bindings and selects `seed_window_index`. Neither accepts `rollout.type` or `seed_index`. Both indices are zero-based and validated before environment creation. Trajectory metadata uses the same route-specific index name. See the [rollout guide](../rollout.md) and diff --git a/documentation/configs/preprocess.md b/documentation/configs/preprocess.md index 2546ee0a..6c1001be 100644 --- a/documentation/configs/preprocess.md +++ b/documentation/configs/preprocess.md @@ -505,7 +505,7 @@ always allowed. With gaps enabled, physical slots remain unchanged. Outer item positions must be continuous within each selected sequence. An item in this raw format must have at least one child; null child rows do not encode emptiness. -All PT files use version 5 of the `sequifier_tensor_batch` envelope. The payload +All PT files use the current `ContractVersions.PT_TENSOR_BATCH` version of the `sequifier_tensor_batch` envelope. The payload stores shallow tensors as `[N,W]`, deep tensors as `[N,W,D]`, boolean masks under `metadata.depth_valid_masks.`, and the absolute window and split-boundary positions needed to enforce split ownership. Optional signed Int64 curriculum @@ -513,8 +513,7 @@ values are stored under `metadata.sample_positions`; their source names are stored under `metadata.curriculum_columns`, and multiple preserved columns use shape `[N,C]`. Public metadata records the model-facing `tensor_payload_version` separately from this internal storage envelope. Readers -reject legacy tuple payloads and envelope versions 2 through 4; re-run -preprocessing to migrate them. +require the current envelope version. Re-run preprocessing to migrate them. Categorical padding is the existing unknown-token ID (zero); real padding is finite zero after normalization. Temporal padding has false depth masks. diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 99456e84..80295355 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -1293,7 +1293,7 @@ now take effect; exact resume of those older runs is rejected. `initialization_seed` accepts integers from zero through `2**63-1`, excluding booleans. An explicit seed isolates constructor and custom-initializer randomness -from the run stream. Descendants derive seeds using SHA-256 over version 1, +from the run stream. Descendants derive seeds using SHA-256 over `ContractVersions.INITIALIZATION_SEED_DERIVATION`, namespace seed, relative branch path, and phase; explicit child seeds start a new namespace. Omitted seeds preserve the existing RNG stream and initialization traversal. Initialization seeds do not control runtime dropout. Dropout continues diff --git a/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json b/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json index aa546b6d..6f43198c 100644 --- a/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json +++ b/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json @@ -63,5 +63,6 @@ }, "window_length": 11, "max_target_offset": 1, - "stored_window_layout_version": 2 + "tensor_payload_version": 113, + "stored_window_layout_version": 115 } diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index 283e3da0..ef98214a 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -10,7 +10,7 @@ rollout: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 action_selection: deterministic task: policy model_paths: diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index 9e212492..11136817 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -121,13 +121,13 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 evaluation: environment: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 seeds: - 17 - 29 @@ -154,7 +154,7 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 total_env_steps: 8192 rollout_env_steps: 256 optimization_epochs: 4 @@ -167,7 +167,7 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 seeds: - 17 - 29 diff --git a/documentation/example_inputs/iql_toy_control.py b/documentation/example_inputs/iql_toy_control.py index e5e53983..26679d24 100644 --- a/documentation/example_inputs/iql_toy_control.py +++ b/documentation/example_inputs/iql_toy_control.py @@ -13,6 +13,7 @@ from pathlib import Path from sequifier.config.policy_infer_config import PolicyRolloutConfig +from sequifier.contract_versions import ContractVersions from sequifier.rl.checkpoint import load_policy from sequifier.rl.environment import ( EnvironmentCapabilities, @@ -26,8 +27,8 @@ class ToyControlAdapter: - protocol_version = 3 - implementation_id = "sequifier.toy-control.v1" + protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER + implementation_id = f"sequifier.toy-control.v{ContractVersions.TOY_CONTROL}" capabilities = EnvironmentCapabilities(seeded_reset=True) def reset(self, seed): @@ -124,7 +125,7 @@ def evaluate(paths): { "environment": { "factory": "iql_toy_control:ToyControlAdapter", - "implementation_id": "sequifier.toy-control.v1", + "implementation_id": ToyControlAdapter.implementation_id, "units": {}, }, "max_steps": HORIZON, diff --git a/documentation/example_inputs/policy_control_data.py b/documentation/example_inputs/policy_control_data.py index 257dc582..0ac120ea 100644 --- a/documentation/example_inputs/policy_control_data.py +++ b/documentation/example_inputs/policy_control_data.py @@ -10,6 +10,8 @@ import sys from pathlib import Path +from sequifier.contract_versions import ContractVersions + def generate(directory): directory = Path(directory) @@ -77,7 +79,11 @@ def generate(directory): ) (directory / "control-preferences.json").write_text( json.dumps( - {"format": "sequifier.preference-pairs", "version": 1, "pairs": pairs}, + { + "format": "sequifier.preference-pairs", + "version": ContractVersions.PREFERENCE_PAIR_MANIFEST, + "pairs": pairs, + }, indent=2, ) + "\n", diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 3fd4e753..3ecff5b1 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -81,7 +81,7 @@ critic diagnostics are separate. Critics are fresh per phase and are not exporte ## DPO preferences -A preference manifest declares `format: sequifier.preference-pairs`, `version: 1` +A preference manifest declares `format: sequifier.preference-pairs`, `version: 126` (`ContractVersions.PREFERENCE_PAIR_MANIFEST`) and `pairs`. Each pair has `pair_id`, `context_id`, optional `group_id`, and `chosen`/`rejected` spans `{source, episode_id, start, end}`. Bounds are zero-based, start-inclusive/end-exclusive within loaded episode rows. Initial executable @@ -102,7 +102,7 @@ a still-present changed reference is rejected. ## Online adapters -PPO, GRPO, evaluation and policy rollout use the version-3 environment protocol: +PPO, GRPO, evaluation and policy rollout use `ContractVersions.ONLINE_POLICY_ADAPTER`: `environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. Factories receive `config` as keyword arguments. Implementation identity and units must match the declaration. `OnlineEnvironmentAdapter`, `EnvironmentCapabilities`, @@ -243,14 +243,13 @@ Online intervals count completed collection/update iterations. ## Checkpoints and inference -Policy runs use the shared version-4 run-checkpoint envelope with a version-8 -native recovery payload. They retain representation, plan/source/partition/adapter +Policy runs use the shared `RUN_CHECKPOINT` envelope with a `POLICY_RECOVERY` +payload and a `POLICY_PHASE_STATE` record for the live phase. They retain representation, plan/source/partition/adapter identities, selected phase summaries and checksums, RNG and live phase networks/optimization/sampler/selection state. Training and evaluation histories are journaled under each native phase directory. Keep the journal and committed phase models with the checkpoint so reports can be regenerated from the recorded -positions. Mixed plans also retain their phase completion receipts. Version-7 -policy checkpoints import their embedded histories on resume. See +positions. Mixed plans also retain their phase completion receipts. See [Training artifacts](training/artifacts.md). Source files and output-path conflicts are rechecked at phase boundaries. diff --git a/documentation/rollout.md b/documentation/rollout.md index 3bf60ff5..986e9d27 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -26,7 +26,7 @@ one prepared window in dataset/chunk order, following normal stride/window rules before first-subsequence filtering. Each temporal input needs exactly one producer; static inputs cannot have bindings. Extra/missing environment fields fail validation. -Forecast adapters declare `protocol_version = 1` and implement `reset(ResetRequest)`, +Forecast adapters declare `protocol_version = ContractVersions.FORECAST_ROLLOUT_ADAPTER` and implement `reset(ResetRequest)`, `step(StepRequest) -> StepResult`, and `close()`, exported by `sequifier.rollout_api`. Factories receive `environment.config` as keyword arguments. Reset initializes from the supplied history without executing a transition; step receives copied current @@ -40,7 +40,7 @@ together. Unsupported shapes, overflow and non-finite transformed inputs fail. ## Policy rollout -Policy rollout uses the [version-3 online adapter](policy-learning.md#online-adapters), +Policy rollout uses the [online policy adapter](policy-learning.md#online-adapters), matching `implementation_id`/`units` and seeded `reset_from_observation`. It selects raw CSV/Parquet rows with `seed_row_index`, rejects forecast bindings, and executes named policy actions. A seed supplies exactly one current observation; model history, diff --git a/documentation/supervised-policy-learning.md b/documentation/supervised-policy-learning.md index bf96d32f..711c8b43 100644 --- a/documentation/supervised-policy-learning.md +++ b/documentation/supervised-policy-learning.md @@ -88,5 +88,5 @@ and `GaussianNLLLoss` opt into the shared distribution implementation; learned s belong to the model artifact. `task: forecast` selects point prediction from a policy-bound artifact; `task: policy` selects policy execution. See [inference routing](configs/infer.md#policy-model-interfaces). -Model artifacts write version 3 and read versions 1/2/3; joint trajectories and PPO -collection records use version 2. ONNX remains a prediction export. +Model artifacts, joint trajectories, and PPO collection records each use their +current version from `ContractVersions`. ONNX remains a prediction export. diff --git a/documentation/training/artifacts.md b/documentation/training/artifacts.md index 6a54a3e0..a1ce4f31 100644 --- a/documentation/training/artifacts.md +++ b/documentation/training/artifacts.md @@ -1,9 +1,8 @@ # Training artifacts Supervised learning, policy imitation, IQL, PPO, DPO, GRPO and mixed plans use the -same public output, checkpoint, report and scalar metric contracts. These are -new major-version contracts. Previous native checkpoint formats remain readable; -earlier metric schemas are not migrated. +same public output, checkpoint, report and scalar metric contracts. Each reader requires the current version for its contract. See +[ContractVersions](../../src/sequifier/contract_versions.py) for the IDs. ## Configuration and layout @@ -36,7 +35,7 @@ publication, including generated phase destinations and symlink escapes. ## Portable models and recovery -Portable models use `artifact_type: sequifier_model`, `format_version: 3`. +Portable models use `artifact_type: sequifier_model`, `format_version: 105`. Their execution contract embeds interfaces, codecs, normalization, policy bindings and units. Policy execution reads units from the selected interface; provenance is not a second source of execution configuration. @@ -45,7 +44,7 @@ All newly written run checkpoints have the same envelope: ```text artifact_type: sequifier_run_checkpoint -format_version: 4 +format_version: 100 model: recovery_type: supervised | policy | learning recovery: @@ -63,13 +62,13 @@ checkpoint contains in-phase progress. ## Reports and discovery All final reports use `artifact_type: sequifier_run_manifest`, -`format_version: 3`, at the configured report path and at `manifest.json` beside +`format_version: 108`, at the configured report path and at `manifest.json` beside the rolling checkpoint. Common fields identify the logical run, current session, status, completion reason, final model, checkpoint, exports and completed phases. Additional fields contain runner-specific provenance. Each phase report uses `artifact_type: sequifier_phase_report`, -`format_version: 1`. Its common fields include phase index, name, algorithm, +`format_version: 107`. Its common fields include phase index, name, algorithm, model and checkpoint paths, monitor/direction, handoff, selected metric and transaction, selected weight hash, clocks, metrics, provenance and stop scope. Unavailable measurements are null. Algorithm-specific metrics keep their names @@ -80,7 +79,7 @@ and units; a reward monitor is not represented as a loss. Every native runner configures the model-scoped logger under `logs/`. Structured scalar observations use the supervised writer's `*-training.csv`, `*-training-full.csv`, `*-validation.csv` and `*-validation-full.csv` tables. -All tables carry `schema_version: 3`. Full tables include phase, algorithm and +All tables carry `schema_version: 109`. Full tables include phase, algorithm and iteration identity, run/session IDs, UTC timestamps and progress. Named scalar observations use `metric`, `target`, `value` and `is_primary`; policy observations use `target: __total__`. Nested diagnostic records remain in phase reports. @@ -115,7 +114,7 @@ files keep their canonical target-specific destinations. Forecast and policy rollout use the shared validated trajectory record. Training-time PPO and GRPO archives instead use the common -`sequifier.collection-archive` version-1 envelope with `algorithm` and +`sequifier.collection-archive` envelope (version 122) with `algorithm` and `collection` fields. The collection payload contains algorithm-specific replay and diagnostic information. `read_collection_archive` reads the envelope; `load_collection_record` additionally validates PPO collections. These archives diff --git a/documentation/training/runtime-architecture.md b/documentation/training/runtime-architecture.md index 7f56b201..7e7a5a0a 100644 --- a/documentation/training/runtime-architecture.md +++ b/documentation/training/runtime-architecture.md @@ -58,8 +58,8 @@ precedes deep feature arithmetic in eager, compiled, and exported forwards. Unique layers in all temporal and depth stacks are compiled once with their shared aliases restored. Warm-up preserves the run RNG, including on failure. -New portable model artifacts use format version 2; version 1 flat models remain -readable. Tensor payload versions and stored-window versions remain separate. +Portable model artifacts, tensor payloads and stored-window layouts each use +their own current version from `ContractVersions`. Run checkpoints record initialization-policy and seed-derivation versions in addition to resolved branch policies and the existing optimizer/scaler/RNG state. Resume comparisons use the current layout and payload fields. diff --git a/release-notes/unreleased-contract-versions.md b/release-notes/unreleased-contract-versions.md new file mode 100644 index 00000000..a9ba1883 --- /dev/null +++ b/release-notes/unreleased-contract-versions.md @@ -0,0 +1,33 @@ +# Consolidated Sequifier contract versions + +[ContractVersions](../src/sequifier/contract_versions.py) is the source of current +IDs for Sequifier artifacts, metadata, adapter protocols, and implementation +identifiers. Checkpoint and collection archive envelopes have their own IDs, as +do the payloads they contain. + +- Re-run preprocessing from raw data to rebuild dataset metadata, PT batches, + stored-window metadata, preprocessing manifests, depth ledgers, and video + frame indexes. +- Start a fresh training run. Pre-cutover checkpoints cannot be resumed. Use the + previous Sequifier release for any run or portable model you still need to + load from that release. Produce fresh portable models, backbone repository + revisions, reports, metrics, and ONNX exports for the current release. +- Re-export sequence data and regenerate trajectories and collection archives. + Update authored preference-pair and trajectory-source manifests to declare + the current version and point to current source artifacts. +- Update external forecast adapters to declare + `ContractVersions.FORECAST_ROLLOUT_ADAPTER` and external online policy + adapters to declare `ContractVersions.ONLINE_POLICY_ADAPTER`. Use the + current implementation IDs in configurations that select the built-in or toy + control. +- Initialization and policy collection use new seed namespaces. The same inputs + are deterministic within the new version and can produce different seeds from + earlier releases. Versioned identities change fingerprints and export + preflight cache keys. The ONNX operator set remains 18. + +To change a contract, allocate an ID above all active and retired IDs on the +target branch, retire the previous ID in the same change, and update the affected +boundary checks. For a pure rename, record the old and new names in +`tools/check_contract_versions.py`. Bump an envelope only when its own contract +changes. Require the Contract versions check on an up-to-date merge result, +using the merge queue when enabled. diff --git a/release-notes/unreleased-phase-ownership.md b/release-notes/unreleased-phase-ownership.md index 90a2335a..9d7939d6 100644 --- a/release-notes/unreleased-phase-ownership.md +++ b/release-notes/unreleased-phase-ownership.md @@ -15,7 +15,7 @@ inference/evaluation selection to `action_selection`. Seeded sampling and deterministic execution remain separate controls. - Use `rollout.environment` and select rollout schemas from the artifact interface and `task`. - Forecast version-1 bindings and policy version-3 environments remain distinct. + Forecast rollout and online policy environments use distinct adapter protocols. Document forecast `seed_window_index` versus policy `seed_row_index` units. - Update shipped examples, scaffolding, Python examples and consolidated docs. CPU smoke coverage is required before release for native supervised and policy algorithms, mixed diff --git a/release-notes/unreleased-policy-training.md b/release-notes/unreleased-policy-training.md index d36d350a..6b68237b 100644 --- a/release-notes/unreleased-policy-training.md +++ b/release-notes/unreleased-policy-training.md @@ -13,11 +13,11 @@ - GRPO adds explicit reproducible group-start capabilities, independent RNG streams, bounded complete groups, population-normalized advantages, clipped ratios and a separately named sampled frozen-reference penalty, without a critic. -- Version-3 local online adapters serve PPO, GRPO and policy rollout. Explicit +- Versioned local online adapters serve PPO, GRPO and policy rollout. Explicit seeded reset preserves supplied initial observations during inference. Exact continuation requires validated snapshots or documented reproducible group starts; non-exact restarts report a discontinuity. Failed/partial groups are never reused. -- One atomic version-6 (with version-5 resume support) `sequifier_policy_run` checkpoint owns phase progress, +- One atomic policy recovery checkpoint owns phase progress, role-keyed optimizer/network state, references, source identities and RNG. Phase actors are durable exports referenced by checksum; reports are derived and regenerated on resume. No phase manifests are needed. diff --git a/release-notes/unreleased-supervised-policy-unification.md b/release-notes/unreleased-supervised-policy-unification.md index c598bb96..2a3f19eb 100644 --- a/release-notes/unreleased-supervised-policy-unification.md +++ b/release-notes/unreleased-supervised-policy-unification.md @@ -14,7 +14,7 @@ - Separate persisted forward attention from optimization objectives and allow explicit prediction or policy inference views. - Add shared real prediction distributions and supervised Gaussian NLL support. -- Write model artifact version 3 and typed policy trajectory/collection version 2; +- Write versioned model artifacts, typed policy trajectories, and collection payloads; retain readers for earlier model artifacts and single-categorical interfaces. Focused CPU tests now cover phase-owned optimization, all policy algorithms, diff --git a/release-notes/unreleased-training-artifact-contracts.md b/release-notes/unreleased-training-artifact-contracts.md index c22b7a31..ca59bb4e 100644 --- a/release-notes/unreleased-training-artifact-contracts.md +++ b/release-notes/unreleased-training-artifact-contracts.md @@ -4,9 +4,9 @@ a canonical final portable model and per-phase model/report records. - Reject overlapping output files, reserved phase/log/archive trees, protected inputs and generated destinations escaping through symlinks before publication. -- Store exact recovery behind a shared version-4 `sequifier_run_checkpoint` +- Store exact recovery behind a shared `sequifier_run_checkpoint` envelope with an embedded executable model and runner-owned recovery payload. -- Publish common version-3 run manifests and version-1 phase reports, retaining +- Publish versioned run manifests and phase reports, retaining supervised manifest discovery beside the rolling checkpoint. - Read policy units from the model interface across every export/loading route. - Write policy observations through the supervised scalar metric tables with diff --git a/src/sequifier/artifacts/backbone_repository.py b/src/sequifier/artifacts/backbone_repository.py index 8ae40fc8..1a3201ba 100644 --- a/src/sequifier/artifacts/backbone_repository.py +++ b/src/sequifier/artifacts/backbone_repository.py @@ -10,11 +10,11 @@ import torch from torch import Tensor, nn +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.io.atomic import atomic_save, write_json_atomic from sequifier.typechecking import beartype ARTIFACT_TYPE = "sequifier_backbone" -FORMAT_VERSION = 1 SUPPORTED_DTYPES = { torch.bool, torch.int8, @@ -118,6 +118,11 @@ def select_revision(backbone_config: Any, project_root: str) -> dict[str, Any] | f"Backbone revision {revision_id!r} is incomplete in {revision_path}." ) manifest = _read_json(manifest_path) + require_contract_version( + "backbone repository manifest", + manifest.get("format_version"), + ContractVersions.BACKBONE_MANIFEST, + ) expected_fingerprint = architecture_fingerprint(backbone_config.architecture) manifest_architecture = manifest.get("architecture") try: @@ -128,7 +133,6 @@ def select_revision(backbone_config: Any, project_root: str) -> dict[str, Any] | ) from exc checks = { "artifact_type": (manifest.get("artifact_type"), ARTIFACT_TYPE), - "format_version": (manifest.get("format_version"), FORMAT_VERSION), "backbone_id": (manifest.get("backbone_id"), repository.backbone_id), "revision_id": (manifest.get("revision_id"), revision_id), "architecture_fingerprint": ( @@ -272,7 +276,7 @@ def publish_revision( architecture = canonical_architecture(backbone_config.architecture) manifest = { "artifact_type": ARTIFACT_TYPE, - "format_version": FORMAT_VERSION, + "format_version": ContractVersions.BACKBONE_MANIFEST, "backbone_id": repository.backbone_id, "revision_id": revision_id, "parent_revision_id": parent_revision_id, diff --git a/src/sequifier/artifacts/checkpoint_envelope.py b/src/sequifier/artifacts/checkpoint_envelope.py index fbca0038..2f5d58a2 100644 --- a/src/sequifier/artifacts/checkpoint_envelope.py +++ b/src/sequifier/artifacts/checkpoint_envelope.py @@ -1,43 +1,44 @@ """Shared execution envelope around runner-owned exact recovery state.""" -from sequifier.artifacts.model_artifact import ModelArtifact +from collections.abc import Mapping -CHECKPOINT_FORMAT_VERSION = 4 +from sequifier.artifacts.model_artifact import ModelArtifact +from sequifier.contract_versions import ContractVersions, require_contract_version def checkpoint_envelope(recovery_type, recovery, model): return { "artifact_type": "sequifier_run_checkpoint", - "format_version": CHECKPOINT_FORMAT_VERSION, + "format_version": ContractVersions.RUN_CHECKPOINT, "model": model.state_dict() if isinstance(model, ModelArtifact) else model, "recovery_type": recovery_type, "recovery": recovery, } +def validate_checkpoint_envelope(payload): + if not isinstance(payload, Mapping): + raise ValueError("Expected a Sequifier run checkpoint envelope mapping") + require_contract_version( + "run checkpoint envelope", + payload.get("format_version"), + ContractVersions.RUN_CHECKPOINT, + ) + if payload.get("artifact_type") != "sequifier_run_checkpoint": + raise ValueError("Expected a Sequifier run checkpoint envelope") + if payload.get("recovery_type") not in ("supervised", "policy", "learning"): + raise ValueError( + f"Unsupported checkpoint recovery kind {payload.get('recovery_type')!r}" + ) + if not isinstance(payload.get("recovery"), dict): + raise ValueError("Run checkpoint requires a recovery mapping") + + def recovery_payload(payload, recovery_type): """Validate the shared envelope before restoring runner-owned state.""" - legacy = { - "supervised": ("sequifier_run_checkpoint", 3), - "policy": ("sequifier_policy_run", 7), - "learning": ("sequifier_learning_run", 2), - } - if isinstance(payload, dict) and ( - payload.get("artifact_type"), - payload.get("format_version"), - ) == legacy.get(recovery_type): - return payload - if ( - isinstance(payload, dict) - and payload.get("artifact_type") == "sequifier_run_checkpoint" - ): - if payload.get("format_version") != CHECKPOINT_FORMAT_VERSION: - raise ValueError("Unsupported run checkpoint envelope version") - if payload.get("recovery_type", "supervised") != recovery_type: - raise ValueError( - f"Checkpoint does not contain {recovery_type} recovery state" - ) - if not isinstance(payload.get("recovery"), dict): - raise ValueError("Run checkpoint requires a recovery mapping") - return payload["recovery"] - raise ValueError("Expected a Sequifier run checkpoint envelope") + validate_checkpoint_envelope(payload) + if payload["recovery_type"] != recovery_type: + raise ValueError( + f"Checkpoint recovery kind {payload['recovery_type']!r}; expected {recovery_type!r}" + ) + return payload["recovery"] diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index 7b3b45bb..a29fcb63 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -12,18 +12,18 @@ from sequifier.artifacts.model_config import ( model_execution_from_model_config, resolved_training_config, + validate_interface_versions, ) from sequifier.artifacts.model_export import model_execution_config from sequifier.artifacts.state_dict import ( canonicalize_state_dict, validate_model_state_contract, ) +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.io.atomic import atomic_save from sequifier.model.execution import ModelExecution from sequifier.model.factory import build_transformer_network -MODEL_ARTIFACT_FORMAT_VERSION = 3 - @dataclass(frozen=True) class ModelExecutionConfig: @@ -50,16 +50,13 @@ class ModelArtifact: metadata: ModelArtifactMetadata def validate(self) -> None: - if self.format_version != MODEL_ARTIFACT_FORMAT_VERSION: - raise ValueError( - f"Unsupported model artifact format {self.format_version}; " - f"expected {MODEL_ARTIFACT_FORMAT_VERSION}." - ) - for name, interface in self.model_config.values.get("interfaces", {}).items(): - if interface.get("depth_layouts") and "execution_schema" not in interface: - raise ValueError( - f"Depth model interface {name!r} requires an execution schema" - ) + require_contract_version( + "portable model artifact", + self.format_version, + ContractVersions.MODEL_ARTIFACT, + ) + for interface in self.model_config.values.get("interfaces", {}).values(): + validate_interface_versions(interface) validate_model_state_contract(self.model_state_dict) def state_dict(self) -> dict[str, Any]: @@ -76,8 +73,13 @@ def state_dict(self) -> dict[str, Any]: def from_state_dict(cls, payload: Mapping[str, Any]) -> "ModelArtifact": if payload.get("artifact_type") != "sequifier_model": raise ValueError("Artifact is not a portable Sequifier model.") + require_contract_version( + "portable model artifact", + payload.get("format_version"), + ContractVersions.MODEL_ARTIFACT, + ) artifact = cls( - format_version=int(payload["format_version"]), + format_version=payload["format_version"], model_config=ModelExecutionConfig(dict(payload["model_config"])), model_state_dict=canonicalize_state_dict(payload["model_state_dict"]), metadata=ModelArtifactMetadata(**dict(payload.get("metadata", {}))), @@ -109,7 +111,7 @@ def build_model_artifact( phase.model_dump(mode="json") for phase in phases ] artifact = ModelArtifact( - format_version=MODEL_ARTIFACT_FORMAT_VERSION, + format_version=ContractVersions.MODEL_ARTIFACT, model_config=ModelExecutionConfig.from_training_config(config), model_state_dict=state, metadata=ModelArtifactMetadata( @@ -171,16 +173,19 @@ def read_artifact( Path(path).expanduser().resolve(), map_location="cpu", weights_only=False ) artifact_type = payload.get("artifact_type") if isinstance(payload, dict) else None - if expected_type is not None and artifact_type != expected_type: - raise ValueError(f"Expected {expected_type} artifact, received {artifact_type}") - if artifact_type == "sequifier_run_checkpoint": - from sequifier.artifacts.checkpoint_envelope import CHECKPOINT_FORMAT_VERSION - - if payload.get("format_version") not in {3, CHECKPOINT_FORMAT_VERSION}: - raise ValueError("Unsupported run checkpoint envelope version") + if artifact_type in { + "sequifier_run_checkpoint", + "sequifier_policy_run", + "sequifier_learning_run", + }: + from sequifier.artifacts.checkpoint_envelope import validate_checkpoint_envelope + + validate_checkpoint_envelope(payload) payload = payload.get("model") if payload is None: raise ValueError("Checkpoint precedes the first executable model") + if expected_type is not None and artifact_type != expected_type: + raise ValueError(f"Expected {expected_type} artifact, received {artifact_type}") if not isinstance(payload, dict) or not isinstance(artifact_type, str): raise ValueError( "Sequifier model artifact must be a mapping with an artifact_type" diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index fd414b33..3b81f778 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.train_config import ( BackboneComponentConfig, @@ -11,8 +13,10 @@ ResolvedSequifierConfig, TrainingRuntimeSettings, ) +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view from sequifier.model.execution import ModelExecution +from sequifier.model.execution_schema import ExecutionSchema from sequifier.special_tokens import SPECIAL_TOKEN_IDS from sequifier.typechecking import beartype @@ -98,6 +102,40 @@ def resolved_config_from_model_config( return resolved_training_config(execution), selected +def validate_interface_storage_versions(interface: dict) -> None: + storage = interface.get("storage_layout") + require_contract_version( + "stored-window layout", + storage.get("version") if isinstance(storage, dict) else None, + ContractVersions.STORED_WINDOW_LAYOUT, + ) + # The interface stores only the depth columns it consumes. A shallow projection + # can still refer to a dataset that uses the depth metadata version. + expected = ( + ContractVersions.DEPTH_TENSOR_METADATA + if interface.get("depth_layouts") + else ( + ContractVersions.DATASET_TENSOR_METADATA, + ContractVersions.DEPTH_TENSOR_METADATA, + ) + ) + require_contract_version( + "model dataset tensor metadata", + interface.get("tensor_payload_version"), + expected, + ) + + +def validate_interface_versions(interface: dict) -> None: + validate_interface_storage_versions(interface) + schema = interface.get("execution_schema") + require_contract_version( + "model execution schema", + schema.get("version") if isinstance(schema, dict) else None, + ContractVersions.MODEL_EXECUTION_SCHEMA, + ) + + @beartype def model_execution_from_model_config( values: dict, @@ -132,11 +170,6 @@ def model_execution_from_model_config( ) authored_interfaces = {} resolved_interfaces = {} - fallback_storage_layout = StoredWindowLayout( - window_length=context_length + max(1, target_offset), - max_target_offset=max(1, target_offset), - version=2, - ) window_view = ModelWindowView( context_length=context_length, objective=objective, @@ -147,12 +180,8 @@ def model_execution_from_model_config( ] for name in interface_order: interface = interface_values[name] - storage_layout_values = interface.get("storage_layout") - storage_layout = ( - StoredWindowLayout(**storage_layout_values) - if storage_layout_values is not None - else fallback_storage_layout - ) + validate_interface_versions(interface) + storage_layout = StoredWindowLayout(**interface["storage_layout"]) resolve_window_view(storage_layout, window_view) authored = { key: value @@ -194,22 +223,13 @@ def model_execution_from_model_config( "window_view": window_view, } ) - if resolved.depth_layouts and resolved.tensor_payload_version not in {2, 3}: - raise ValueError( - "Depth model execution requires tensor_payload_version 2 or 3" - ) persisted_schema = interface.get("execution_schema") - if persisted_schema is not None: - from sequifier.model.execution_schema import ExecutionSchema - - actual_schema = ExecutionSchema.from_dict(persisted_schema) - expected_schema = ExecutionSchema.from_interface(resolved, context_length) - import json - - if json.dumps(actual_schema.to_dict(), sort_keys=True) != json.dumps( - expected_schema.to_dict(), sort_keys=True - ): - raise ValueError(f"Execution schema contradicts interface {name!r}") + actual_schema = ExecutionSchema.from_dict(persisted_schema) + expected_schema = ExecutionSchema.from_interface(resolved, context_length) + if json.dumps(actual_schema.to_dict(), sort_keys=True) != json.dumps( + expected_schema.to_dict(), sort_keys=True + ): + raise ValueError(f"Execution schema contradicts interface {name!r}") resolved_interfaces[name] = resolved config = ModelExecution( ModelSpecModel(backbone=backbone, interfaces=authored_interfaces), diff --git a/src/sequifier/artifacts/phase.py b/src/sequifier/artifacts/phase.py index 55f331d1..ce084777 100644 --- a/src/sequifier/artifacts/phase.py +++ b/src/sequifier/artifacts/phase.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from pathlib import Path +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.fingerprints import fingerprint from sequifier.io.atomic import write_json_atomic from sequifier.trajectory.sources import file_checksum @@ -56,6 +57,11 @@ def read_receipt(self, identity, *, run_id=None): self.model ): raise ValueError("completed phase artifact or receipt changed") + require_contract_version( + "phase report", + receipt["report"].get("format_version"), + ContractVersions.PHASE_REPORT, + ) return receipt def write_receipt(self, identity, report, *, run_id=None): diff --git a/src/sequifier/artifacts/reports.py b/src/sequifier/artifacts/reports.py index 74a24bcd..26464789 100644 --- a/src/sequifier/artifacts/reports.py +++ b/src/sequifier/artifacts/reports.py @@ -2,13 +2,13 @@ from pathlib import Path +from sequifier.contract_versions import ContractVersions from sequifier.io.atomic import write_json_atomic def phase_report(*, index, name, algorithm, model_path, checkpoint_path, **details): return { "artifact_type": "sequifier_phase_report", - "format_version": 1, "index": index, "name": name, "algorithm": algorithm, @@ -26,13 +26,13 @@ def phase_report(*, index, name, algorithm, model_path, checkpoint_path, **detai "stop_scope": None, "completion_reason": "completed", **details, + "format_version": ContractVersions.PHASE_REPORT, } def run_report(*, model_path, checkpoint_path, phases, **details): return { "artifact_type": "sequifier_run_manifest", - "format_version": 3, "model_path": str(model_path), "checkpoint_path": str(checkpoint_path), "phases": list(phases), @@ -45,6 +45,7 @@ def run_report(*, model_path, checkpoint_path, phases, **details): "initialization": {}, "resume": {}, **details, + "format_version": ContractVersions.RUN_MANIFEST, } diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index fd67e5fe..2571ba0a 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -10,10 +10,9 @@ import torch from sequifier.artifacts.model_artifact import ModelArtifact +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.io.atomic import atomic_save -RUN_CHECKPOINT_FORMAT_VERSION = 3 - @dataclass(frozen=True) class OptimizationState: @@ -51,11 +50,11 @@ class RunCheckpoint: training_config: Any def validate(self) -> None: - if self.format_version != RUN_CHECKPOINT_FORMAT_VERSION: - raise ValueError( - f"Unsupported run checkpoint format {self.format_version}; " - f"expected {RUN_CHECKPOINT_FORMAT_VERSION}." - ) + require_contract_version( + "supervised recovery", + self.format_version, + ContractVersions.SUPERVISED_RECOVERY, + ) self.model.validate() def state_dict(self) -> dict[str, Any]: @@ -72,8 +71,8 @@ def state_dict(self) -> dict[str, Any]: "random_state": {"states": self.random_state.states}, "loader_state": {"parts": self.loader_state.parts}, "integration_state": {"values": self.integration_state.values}, - "initialization_policy_version": 1, - "initialization_seed_derivation_version": 1, + "initialization_policy_version": ContractVersions.INITIALIZATION_POLICY, + "initialization_seed_derivation_version": ContractVersions.INITIALIZATION_SEED_DERIVATION, "training_config": self.training_config.model_dump(mode="python"), } return checkpoint_envelope("supervised", recovery, self.model) @@ -83,45 +82,49 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": from sequifier.artifacts.checkpoint_envelope import recovery_payload payload = recovery_payload(payload, "supervised") + require_contract_version( + "supervised recovery", + payload.get("format_version"), + ContractVersions.SUPERVISED_RECOVERY, + ) if payload.get("artifact_type") != "sequifier_run_checkpoint": - raise ValueError("Unsupported checkpoint: expected new Sequifier format.") - if payload.get("format_version") != RUN_CHECKPOINT_FORMAT_VERSION: - raise ValueError( - f"unsupported native checkpoint format {payload.get('format_version')}; expected {RUN_CHECKPOINT_FORMAT_VERSION}. Initialize from its portable model artifact to start a new run." - ) - if payload.get("recovery_type", "supervised") != "supervised": + raise ValueError("Expected a supervised checkpoint recovery payload") + if payload.get("recovery_type") != "supervised": raise ValueError("Checkpoint belongs to another training runner") + from sequifier.artifacts.model_config import validate_interface_storage_versions + from sequifier.config.metadata import validate_persisted_metadata_versions from sequifier.config.train_config import ResolvedSequifierConfig - if ( - payload.get("initialization_policy_version") != 1 - or payload.get("initialization_seed_derivation_version") != 1 - ): - raise ValueError("Unsupported initialization provenance version") + require_contract_version( + "initialization policy", + payload.get("initialization_policy_version"), + ContractVersions.INITIALIZATION_POLICY, + ) + require_contract_version( + "initialization seed derivation", + payload.get("initialization_seed_derivation_version"), + ContractVersions.INITIALIZATION_SEED_DERIVATION, + ) + model = ModelArtifact.from_state_dict(payload["model"]) saved_config = payload["training_config"] + for interface in saved_config["interfaces"].values(): + validate_interface_storage_versions(interface) + for dataset in saved_config["dataset_training"].values(): + if "interface" in dataset: + validate_interface_storage_versions(dataset["interface"]) + for part in dataset["parts"].values(): + validate_persisted_metadata_versions(part["metadata"]) training_config = ResolvedSequifierConfig.model_validate(saved_config) - removed_datasets = ( - saved_config["dataset_training"].keys() - - training_config.dataset_training.keys() - if "interfaces" not in saved_config - else set() - ) loader_parts = payload["loader_state"]["parts"] checkpoint = cls( - format_version=int(payload["format_version"]), - model=ModelArtifact.from_state_dict(payload["model"]), + format_version=payload["format_version"], + model=model, optimization=OptimizationState(**dict(payload["optimization"])), run_state=dict(payload["run_state"]), random_state=DistributedRandomState( states=tuple(payload["random_state"]["states"]) ), - loader_state=LoaderState( - parts={ - name: parts - for name, parts in loader_parts.items() - if parts or name not in removed_datasets - } - ), + loader_state=LoaderState(parts=dict(loader_parts)), integration_state=IntegrationState( values=dict(payload["integration_state"]["values"]) ), diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 99c87b79..2c97507f 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -19,6 +19,7 @@ DatasetMetadata, extract_inline_metadata, load_dataset_metadata, + validate_persisted_metadata_versions, ) from sequifier.config.policy_infer_config import ( ResolvedPolicyInferenceConfig, @@ -154,36 +155,39 @@ def _execution_source( "prediction_length": interface.decoder.prediction_length, "decoder_support_length": interface.decoder.support_length, }, - DatasetMetadata( - depth_layouts=interface.depth_layouts, - tensor_payload_version=interface.tensor_payload_version, - split_paths=interface.split_paths, - column_data_types=dict(interface.column_data_types), - static_columns=( - list(interface.static_conditioning.columns) - if interface.static_conditioning - else [] + DatasetMetadata.model_validate( + dict( + depth_layouts=interface.depth_layouts, + tensor_payload_version=interface.tensor_payload_version, + split_paths=interface.split_paths, + column_data_types=dict(interface.column_data_types), + static_columns=( + list(interface.static_conditioning.columns) + if interface.static_conditioning + else [] + ), + static_features_path=interface.static_features_path, + image_sources=interface.image_sources, + preconditioning_image=interface.preconditioning_image, + n_classes=dict(interface.n_classes), + id_maps=dict(interface.id_maps), + special_token_ids=dict(interface.special_token_ids), + selected_columns_statistics=dict(interface.selected_columns_statistics), + normalize_real_columns=interface.normalize_real_columns, + prediction_aligned_splits=interface.prediction_aligned_splits, + prediction_alignment_steps=interface.prediction_alignment_steps, + window_lengths=interface.window_lengths, + prediction_length=interface.prediction_length, + target_offset=interface.target_offset, + window_length=( + interface.window_lengths[0] + if interface.window_lengths + else layout.window_length + ), + max_target_offset=layout.max_target_offset, + stored_window_layout_version=layout.version, ), - static_features_path=interface.static_features_path, - image_sources=interface.image_sources, - preconditioning_image=interface.preconditioning_image, - n_classes=dict(interface.n_classes), - id_maps=dict(interface.id_maps), - special_token_ids=dict(interface.special_token_ids), - selected_columns_statistics=dict(interface.selected_columns_statistics), - normalize_real_columns=interface.normalize_real_columns, - prediction_aligned_splits=interface.prediction_aligned_splits, - prediction_alignment_steps=interface.prediction_alignment_steps, - window_lengths=interface.window_lengths, - prediction_length=interface.prediction_length, - target_offset=interface.target_offset, - window_length=( - interface.window_lengths[0] - if interface.window_lengths - else layout.window_length - ), - max_target_offset=layout.max_target_offset, - stored_window_layout_version=layout.version, + context={"projected_depth_layouts": True}, ), ) @@ -257,6 +261,7 @@ def _assert_folder_metadata_matches( ) with open(metadata_path) as metadata_file: folder_values = json.load(metadata_file) + validate_persisted_metadata_versions(folder_values) required_semantics = { "id_maps", "selected_columns_statistics", @@ -635,11 +640,6 @@ def resolve_inference_config( f"metadata {metadata.prediction_alignment_steps}." ) storage_layout = metadata.storage_layout - if storage_layout.version != 2: - raise ValueError( - "Inference requires metadata stored_window_layout_version=2, " - f"got {storage_layout.version}." - ) available_column_data_types = config.column_data_types or metadata.column_data_types if image_targets := set(config.target_columns) & set(metadata.image_sources): raise ValueError( diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index 1e2ee3b6..54d0a201 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -132,14 +132,25 @@ def inference_model_config(path, project_root, artifacts): return None import onnx - from sequifier.model.execution_schema import EXECUTION_SCHEMA_KEY, MODEL_CONFIG_KEY + from sequifier.model.execution_schema import ( + DROPOUT_MODE_KEY, + EXECUTION_SCHEMA_KEY, + MODEL_CONFIG_KEY, + ExecutionSchema, + ) graph = onnx.load(normalized, load_external_data=False) properties = {item.key: item.value for item in graph.metadata_props} - if EXECUTION_SCHEMA_KEY not in properties and any( - len(value.type.tensor_type.shape.dim) == 3 for value in graph.graph.input + schema = properties.get(EXECUTION_SCHEMA_KEY) + if ( + schema is not None + or MODEL_CONFIG_KEY in properties + or DROPOUT_MODE_KEY in properties + or any( + len(value.type.tensor_type.shape.dim) == 3 for value in graph.graph.input + ) ): - raise ValueError("Depth ONNX graphs require execution-schema metadata") + ExecutionSchema.from_dict(json.loads(schema) if schema is not None else None) return ( json.loads(properties[MODEL_CONFIG_KEY]) if MODEL_CONFIG_KEY in properties diff --git a/src/sequifier/config/metadata.py b/src/sequifier/config/metadata.py index a45090dc..b6696969 100644 --- a/src/sequifier/config/metadata.py +++ b/src/sequifier/config/metadata.py @@ -6,11 +6,12 @@ import json from typing import Any -from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic import ConfigDict, Field, ValidationInfo, field_validator, model_validator from sequifier.config.depth_layout import DepthLayoutRegistryModel from sequifier.config.image_config import ImageSourceConfig, PreconditioningImageConfig from sequifier.config.naming import ConfigModel +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.helpers import ModelWindowView, StoredWindowLayout from sequifier.special_tokens import SPECIAL_TOKEN_IDS, validate_special_token_ids from sequifier.typechecking import beartype @@ -38,6 +39,29 @@ } +def dataset_tensor_version(depth_layouts) -> int: + return ( + ContractVersions.DEPTH_TENSOR_METADATA + if depth_layouts + else ContractVersions.DATASET_TENSOR_METADATA + ) + + +def validate_persisted_metadata_versions(values: dict) -> None: + if not isinstance(values, dict): + raise ValueError("Dataset metadata must contain a mapping") + require_contract_version( + "dataset tensor metadata", + values.get("tensor_payload_version"), + dataset_tensor_version(values.get("depth_layouts")), + ) + require_contract_version( + "stored-window layout", + values.get("stored_window_layout_version"), + ContractVersions.STORED_WINDOW_LAYOUT, + ) + + class DatasetMetadata(ConfigModel): """The stable subset of preprocessing metadata consumed by other commands.""" @@ -45,7 +69,7 @@ class DatasetMetadata(ConfigModel): @model_validator(mode="before") @classmethod - def reject_legacy_placement(cls, values: Any) -> Any: + def normalize_inline_metadata(cls, values: Any) -> Any: if isinstance(values, dict) and { "window_strides", "window_placement", @@ -54,12 +78,28 @@ def reject_legacy_placement(cls, values: Any) -> Any: raise ValueError( "Preprocessing metadata uses obsolete placement settings; re-run preprocessing" ) + if isinstance(values, dict) and "tensor_payload_version" not in values: + values = { + **values, + "tensor_payload_version": dataset_tensor_version( + values.get("depth_layouts") + ), + } return values depth_layouts: DepthLayoutRegistryModel = Field( default_factory=DepthLayoutRegistryModel ) - tensor_payload_version: int = Field(default=1, ge=1, le=3) + tensor_payload_version: int = Field( + default=ContractVersions.DATASET_TENSOR_METADATA, + strict=True, + json_schema_extra={ + "enum": [ + ContractVersions.DATASET_TENSOR_METADATA, + ContractVersions.DEPTH_TENSOR_METADATA, + ] + }, + ) split_paths: list[str] = Field(default_factory=list) split_index: int | None = Field(default=None, ge=0) column_data_types: dict[str, str] = Field( @@ -88,12 +128,33 @@ def reject_legacy_placement(cls, values: Any) -> Any: window_length: int = Field(gt=0) window_lengths: list[int] | None = None max_target_offset: int = Field(default=1, ge=0) - stored_window_layout_version: int = Field(default=2, ge=2, le=2) + stored_window_layout_version: int = Field( + default=ContractVersions.STORED_WINDOW_LAYOUT, + strict=True, + json_schema_extra={"const": ContractVersions.STORED_WINDOW_LAYOUT}, + ) @model_validator(mode="after") - def validate_layout_metadata(self): - if self.depth_layouts and self.tensor_payload_version not in {2, 3}: - raise ValueError("Depth datasets require tensor_payload_version 2 or 3") + def validate_layout_metadata(self, info: ValidationInfo): + expected: int | tuple[int, ...] = dataset_tensor_version(self.depth_layouts) + if not self.depth_layouts and (info.context or {}).get( + "projected_depth_layouts" + ): + # A model interface can project only shallow columns from a depth dataset. + expected = ( + ContractVersions.DATASET_TENSOR_METADATA, + ContractVersions.DEPTH_TENSOR_METADATA, + ) + require_contract_version( + "dataset tensor metadata", + self.tensor_payload_version, + expected, + ) + require_contract_version( + "stored-window layout", + self.stored_window_layout_version, + ContractVersions.STORED_WINDOW_LAYOUT, + ) if self.window_lengths is not None and ( (self.split_paths and len(self.window_lengths) != len(self.split_paths)) or ( @@ -184,6 +245,7 @@ def load_dataset_metadata(path: str) -> DatasetMetadata: values: Any = json.load(file) if not isinstance(values, dict): raise ValueError(f"Metadata config '{path}' must contain a JSON object.") + validate_persisted_metadata_versions(values) return DatasetMetadata.model_validate(values) @@ -214,7 +276,10 @@ def extract_inline_metadata( if path is not None ], "depth_layouts": authored.get("depth_layouts", {}), - "tensor_payload_version": authored.get("tensor_payload_version", 1), + "tensor_payload_version": authored.get( + "tensor_payload_version", + dataset_tensor_version(authored.get("depth_layouts")), + ), "column_data_types": authored.get("column_data_types", {}), "static_columns": authored.get("static_columns", []), "static_features_path": authored.get("static_features_path"), @@ -252,7 +317,9 @@ def extract_inline_metadata( { "window_length": storage_layout.get("window_length"), "max_target_offset": storage_layout.get("max_target_offset", 1), - "stored_window_layout_version": storage_layout.get("version", 2), + "stored_window_layout_version": storage_layout.get( + "version", ContractVersions.STORED_WINDOW_LAYOUT + ), } ) else: @@ -261,7 +328,8 @@ def extract_inline_metadata( "window_length": authored.get("window_length"), "max_target_offset": authored.get("max_target_offset", 1), "stored_window_layout_version": authored.get( - "stored_window_layout_version", 2 + "stored_window_layout_version", + ContractVersions.STORED_WINDOW_LAYOUT, ), } ) diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index 3f68a64c..eb56463f 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -24,6 +24,7 @@ ) from sequifier.config.policy_optimization import OptimizationConfig from sequifier.config.sources import TrainingSourceSpecModel +from sequifier.contract_versions import ContractVersions class OfflineEvaluationMonitor(ConfigModel): @@ -159,7 +160,7 @@ class OnlineEnvironmentConfig(ConfigModel): @property def protocol_version(self): - return 3 + return ContractVersions.ONLINE_POLICY_ADAPTER implementation_id: str = Field(min_length=1) # Empty means unitless; adapters must report this exact declaration. @@ -318,7 +319,7 @@ class GRPOPhaseConfig(BatchedPhaseConfig): @model_serializer(mode="wrap") def serialize_execution_options(self, handler): values = handler(self) - # Disabled/default options must not change legacy checkpoint plan hashes. + # Omit default options from the serialized execution plan. for name in ( "simulator_workers", "rollout_inference_batch_size", diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index f8992f09..0603ea55 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -5,6 +5,7 @@ from pydantic import ConfigDict, Field, field_validator, model_validator from sequifier.config.naming import ConfigModel +from sequifier.contract_versions import ContractVersions class InputBinding(ConfigModel): @@ -23,7 +24,7 @@ class AdapterConfig(ConfigModel): @property def protocol_version(self): - return 1 + return ContractVersions.FORECAST_ROLLOUT_ADAPTER @field_validator("factory") @classmethod diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 804a1e72..c051f8a3 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -24,7 +24,6 @@ model_validator, ) -# Reusable schemas are independent of the canonical root configuration. from sequifier.config.components import ( BackboneComponentConfig, BERTSpecModel, @@ -46,7 +45,11 @@ image_views, ) from sequifier.config.learning_config import SupervisedLearningConfig -from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata +from sequifier.config.metadata import ( + DatasetMetadata, + dataset_tensor_version, + load_dataset_metadata, +) from sequifier.config.naming import ConfigModel from sequifier.config.normalization_config import FrozenNormalizationConfig from sequifier.config.parameter_selection import TrainabilityConfig @@ -69,6 +72,9 @@ from sequifier.config.prediction_config import PredictionDistributionConfig from sequifier.config.sources import TrainingSourceSpecModel, phase_sources from sequifier.config.sources import validate_source as _validate_source + +# Reusable schemas are independent of the canonical root configuration. +from sequifier.contract_versions import ContractVersions from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -728,7 +734,7 @@ class ResolvedDatasetPart(ConfigModel): class ResolvedInterfaceStorage(ConfigModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - tensor_payload_version: int = 1 + tensor_payload_version: int = ContractVersions.DATASET_TENSOR_METADATA split_paths: list[str] = Field(default_factory=list) prediction_aligned_splits: list[int] = Field(default_factory=list) prediction_alignment_steps: int = 1 @@ -949,8 +955,6 @@ class ResolvedSequifierConfig(ConfigModel): preference_sources: dict[str, str] = Field(default_factory=dict) policy_bindings: dict[str, ResolvedPolicyBinding] = Field(default_factory=dict) training_plan: list[ResolvedTrainingPhase | PolicyPhase] - # Retained in the serialized schema for older training checkpoints. - active_phase_index: int = 0 curriculum_training: bool = False curriculum_column: Optional[str] = None @@ -962,8 +966,6 @@ def read_dataset_interfaces(cls, values): values.get("dataset_training"), dict ): return values - interfaces = dict(values.get("interfaces", {})) - legacy = "interfaces" not in values datasets = {} for name, dataset in values["dataset_training"].items(): if isinstance(dataset, dict) and "interface" in dataset: @@ -971,13 +973,9 @@ def read_dataset_interfaces(cls, values): interface = ResolvedModelInterface.model_validate( dataset.pop("interface") ) - if legacy: - interfaces.setdefault(dataset["model_interface"], interface) - if not dataset["parts"]: - continue dataset["storage"] = ResolvedInterfaceStorage.from_interface(interface) datasets[name] = dataset - return {**values, "interfaces": interfaces, "dataset_training": datasets} + return {**values, "dataset_training": datasets} @model_serializer(mode="wrap") def serialize_training_config(self, handler, info): @@ -1930,11 +1928,13 @@ def _inline_metadata( elif isinstance(layout, dict): window_length = layout.get("window_length") max_target_offset = layout.get("max_target_offset", 1) - layout_version = layout.get("version", 2) + layout_version = layout.get("version", ContractVersions.STORED_WINDOW_LAYOUT) else: window_length = values.get("window_length") max_target_offset = values.get("max_target_offset", 1) - layout_version = values.get("stored_window_layout_version", 2) + layout_version = values.get( + "stored_window_layout_version", ContractVersions.STORED_WINDOW_LAYOUT + ) if window_length is None: if global_spec.context_length is None: raise ValueError("inline metadata requires context_length or window_length") @@ -1942,7 +1942,10 @@ def _inline_metadata( return DatasetMetadata.model_validate( { "depth_layouts": values.get("depth_layouts", {}), - "tensor_payload_version": values.get("tensor_payload_version", 1), + "tensor_payload_version": values.get( + "tensor_payload_version", + dataset_tensor_version(values.get("depth_layouts")), + ), "split_paths": values.get("split_paths", []), "column_data_types": values.get("column_data_types", {}), "static_columns": values.get("static_columns", []), diff --git a/src/sequifier/contract_versions.py b/src/sequifier/contract_versions.py new file mode 100644 index 00000000..3e07c30f --- /dev/null +++ b/src/sequifier/contract_versions.py @@ -0,0 +1,72 @@ +"""Sequifier's global allocation of persisted contract and protocol versions.""" + + +class ContractVersions: + HISTORICAL_MIN = 0 + HISTORICAL_MAX = 99 + RETIRED = () + + # Checkpoint envelope and runner-owned payloads. + RUN_CHECKPOINT = 100 + SUPERVISED_RECOVERY = 101 + POLICY_RECOVERY = 102 + POLICY_PHASE_STATE = 103 + LEARNING_RUN_STATE = 104 + + # Portable artifacts and training telemetry. + MODEL_ARTIFACT = 105 + BACKBONE_MANIFEST = 106 + PHASE_REPORT = 107 + RUN_MANIFEST = 108 + METRICS_SCHEMA = 109 + + INITIALIZATION_POLICY = 110 + INITIALIZATION_SEED_DERIVATION = 111 + + # Dataset storage and preprocessing resume records. + PT_TENSOR_BATCH = 112 + DATASET_TENSOR_METADATA = 113 + DEPTH_TENSOR_METADATA = 114 + STORED_WINDOW_LAYOUT = 115 + PREPROCESSING_MANIFEST = 116 + VIDEO_FRAME_INDEX = 117 + SERIAL_DEPTH_LEDGER = 118 + PARALLEL_DEPTH_LEDGER = 119 + + # Rollout records, algorithm archives, and offline source manifests. + FORECAST_TRAJECTORY = 120 + JOINT_POLICY_TRAJECTORY = 121 + COLLECTION_ARCHIVE = 122 + PPO_COLLECTION = 123 + GRPO_ARCHIVE = 124 + SEQUENCE_DATA_MANIFEST = 125 + PREFERENCE_PAIR_MANIFEST = 126 + SCALAR_POLICY_TRAJECTORY = 138 + TRAJECTORY_SOURCE_MANIFEST = 139 # Authored manifest of episode files. + + FORECAST_ROLLOUT_ADAPTER = 127 + ONLINE_POLICY_ADAPTER = 128 + MODEL_EXECUTION_SCHEMA = 129 + EXPORT_PREFLIGHT = 130 + + # Algorithm and implementation identities used in fingerprints and seeds. + HASH_CODEC = 131 + RESERVED_CODE_SPECIAL_TOKENS = 132 + INJECTIVE_SPECIAL_TOKENS = 133 + POLICY_COLLECTION_SEED = 134 + EXPORT_ATTENTION_LOWERING = 135 + CATEGORICAL_CONTROL = 136 + TOY_CONTROL = 137 + + # External ONNX operator set, outside the Sequifier allocation. + ONNX_OPSET = 18 + + +def require_contract_version( + contract: str, actual, expected: int | tuple[int, ...] +) -> None: + allowed = (expected,) if isinstance(expected, int) else expected + if not isinstance(actual, int) or isinstance(actual, bool) or actual not in allowed: + raise ValueError( + f"Unsupported {contract} version {actual!r}, expected {expected}" + ) diff --git a/src/sequifier/export/onnx.py b/src/sequifier/export/onnx.py index c6d281ea..ab38d618 100644 --- a/src/sequifier/export/onnx.py +++ b/src/sequifier/export/onnx.py @@ -17,6 +17,7 @@ import torch from torch import Tensor, nn +from sequifier.contract_versions import ContractVersions from sequifier.model.execution_schema import ( DROPOUT_MODE_KEY, EXECUTION_SCHEMA_KEY, @@ -356,7 +357,7 @@ def export( inputs, staged, export_params=True, - opset_version=18, + opset_version=ContractVersions.ONNX_OPSET, dynamo=True, external_data=False, optimize=False, diff --git a/src/sequifier/export/preflight.py b/src/sequifier/export/preflight.py index 5e05cc61..aad929f0 100644 --- a/src/sequifier/export/preflight.py +++ b/src/sequifier/export/preflight.py @@ -15,9 +15,9 @@ from sequifier.artifacts.manifests import write_manifest from sequifier.artifacts.model_export import model_execution_config +from sequifier.contract_versions import ContractVersions from sequifier.model.execution import ModelExecution -PROTOCOL_VERSION = 1 PREFLIGHT_SEED = 20260910 @@ -50,7 +50,7 @@ def graph_only(value): return value return { - "protocol_version": PROTOCOL_VERSION, + "protocol_version": ContractVersions.EXPORT_PREFLIGHT, "representative_batch_policy": "publication_clamped_2_to_4_preflight_2", "execution": graph_only(execution), "generative": config.export_generative_model, @@ -59,11 +59,11 @@ def graph_only(value): "dropout_mode": "stochastic" if config.export_with_dropout else "evaluation", "export_policy": { "dynamo": True, - "opset": 18, + "opset": ContractVersions.ONNX_OPSET, "dtype": "float32", "external_data": "unique_external_files_graph_committed_last", "optimize": False, - "attention_lowering": "explicit_matmul_softmax_dropout_v1", + "attention_lowering": f"explicit_matmul_softmax_dropout_v{ContractVersions.EXPORT_ATTENTION_LOWERING}", "dynamic_shapes": "shared_batch_fixed_capacities", }, "versions": versions, @@ -120,9 +120,17 @@ def ensure_export_preflight(config, distributed): record_path = cache / f"{fingerprint}.json" if record_path.exists(): record = json.loads(record_path.read_text()) + cached_description = record.get("description") + cached_protocol = ( + cached_description.get("protocol_version") + if isinstance(cached_description, dict) + else None + ) if ( record.get("success") is True - and record.get("description") == description + and isinstance(cached_protocol, int) + and not isinstance(cached_protocol, bool) + and cached_description == description ): return_record = True else: diff --git a/src/sequifier/helpers.py b/src/sequifier/helpers.py index 06021ed9..6c332dec 100644 --- a/src/sequifier/helpers.py +++ b/src/sequifier/helpers.py @@ -16,6 +16,7 @@ from pydantic import ValidationError from torch import Tensor +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.logging_paths import ( dataset_artifact_prefix, model_artifact_path, @@ -227,10 +228,9 @@ class StoredWindowLayout: @beartype def __post_init__(self) -> None: - if self.version != 2: - raise ValueError( - f"stored_window_layout_version must be 2, got {self.version}" - ) + require_contract_version( + "stored-window layout", self.version, ContractVersions.STORED_WINDOW_LAYOUT + ) if self.window_length < 1: raise ValueError("window_length must be a positive integer") if self.max_target_offset < 0: @@ -608,10 +608,15 @@ def validate_stored_window_width(tensor: Tensor, window_length: int) -> None: @beartype def stored_window_layout_from_metadata(metadata: dict) -> StoredWindowLayout: + require_contract_version( + "stored-window layout", + metadata.get("stored_window_layout_version"), + ContractVersions.STORED_WINDOW_LAYOUT, + ) return StoredWindowLayout( window_length=int(metadata["window_length"]), max_target_offset=int(metadata["max_target_offset"]), - version=int(metadata["stored_window_layout_version"]), + version=metadata["stored_window_layout_version"], ) @@ -1027,9 +1032,17 @@ def get_last_training_batch_timedelta( ) if os.path.exists(metrics_path): + from sequifier.training.metrics import ( + validate_metrics_csv_header, + validate_metrics_csv_row, + ) + observations = [] with open(metrics_path, "r", encoding="utf-8", newline="") as file: - for row in csv.DictReader(file): + reader = csv.DictReader(file) + validate_metrics_csv_header(reader.fieldnames, metrics_path) + for row in reader: + validate_metrics_csv_row(row, metrics_path) if row.get("metric") != "loss" or row.get("target") != "__total__": continue timestamp = row.get("timestamp_utc") diff --git a/src/sequifier/hyperparameter_search.py b/src/sequifier/hyperparameter_search.py index 337e7251..6f876cc9 100644 --- a/src/sequifier/hyperparameter_search.py +++ b/src/sequifier/hyperparameter_search.py @@ -29,7 +29,11 @@ dataset_artifact_prefix, model_log_directory, ) -from sequifier.training.metrics import VALIDATION_FIELDS # noqa: E402 +from sequifier.training.metrics import ( # noqa: E402 + VALIDATION_FIELDS, + validate_metrics_csv_header, + validate_metrics_csv_row, +) _DUPLICATE_OF_USER_ATTR = "sequifier_duplicate_of" _MAX_CONSECUTIVE_DUPLICATE_PROPOSALS = 1000 @@ -140,14 +144,16 @@ def objective( if os.path.exists(validation_path): with open(validation_path, "r", encoding="utf-8", newline="") as file: reader = csv.DictReader(file) + validate_metrics_csv_header(reader.fieldnames, validation_path) if reader.fieldnames != VALIDATION_FIELDS: raise ValueError( f"Unexpected validation metrics schema in {validation_path}: " f"{reader.fieldnames!r}" ) - consumed_evaluation_ids.update( - row["evaluation_id"] for row in reader if row.get("evaluation_id") - ) + for row in reader: + validate_metrics_csv_row(row, validation_path) + if row.get("evaluation_id"): + consumed_evaluation_ids.add(row["evaluation_id"]) env = os.environ.copy() env["SEQUIFIER_HYPERPARAMETER_SEARCH_RUN"] = "1" @@ -169,12 +175,16 @@ def consume_metrics( if os.path.exists(validation_path): with open(validation_path, "r", encoding="utf-8", newline="") as file: reader = csv.DictReader(file) + validate_metrics_csv_header(reader.fieldnames, validation_path) if reader.fieldnames != VALIDATION_FIELDS: raise ValueError( f"Unexpected validation metrics schema in " f"{validation_path}: {reader.fieldnames!r}" ) for data in reader: + if None in data.values(): + continue # The training process may still be writing this row. + validate_metrics_csv_row(data, validation_path) evaluation_id = data.get("evaluation_id") if ( not evaluation_id diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index 1dd3d13f..07115076 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -216,12 +216,15 @@ def load_pt_dataset(data_path: str, start_pct: float, end_pct: float) -> Iterato for pt_file in pt_files[start_idx:end_idx]: from sequifier.config.depth_layout import DepthLayoutRegistryModel + from sequifier.config.metadata import validate_persisted_metadata_versions from sequifier.io.pt_payload import load_pt_payload metadata_path = Path(data_path) / "metadata.json" folder_metadata = ( json.loads(metadata_path.read_text()) if metadata_path.exists() else {} ) + if metadata_path.exists(): + validate_persisted_metadata_versions(folder_metadata) yield load_pt_payload( pt_file, layouts=DepthLayoutRegistryModel.model_validate( @@ -1600,23 +1603,19 @@ def __init__( raise ValueError( f"ONNX graph provides {mode} dropout mode, but inference requests {requested}; export a graph for the requested mode" ) - self.execution_schema = ( - ExecutionSchema.from_dict(json.loads(properties[EXECUTION_SCHEMA_KEY])) - if EXECUTION_SCHEMA_KEY in properties - else None - ) - if self.execution_schema is not None: - import onnx + import onnx - from sequifier.export.onnx import validate_graph_contract + from sequifier.export.onnx import validate_graph_contract - graph = onnx.load( - normalize_path(model_path, project_root), - load_external_data=False, - ) - validate_graph_contract(graph, self.execution_schema) - elif any(len(value.shape) == 3 for value in self.ort_session.get_inputs()): - raise ValueError("Depth ONNX inputs require execution schema metadata") + schema = properties.get(EXECUTION_SCHEMA_KEY) + self.execution_schema = ExecutionSchema.from_dict( + json.loads(schema) if schema is not None else None + ) + graph = onnx.load( + normalize_path(model_path, project_root), + load_external_data=False, + ) + validate_graph_contract(graph, self.execution_schema) self.symbolic_batch = all( not isinstance(value.shape[0], int) for value in self.ort_session.get_inputs() diff --git a/src/sequifier/io/depth_preprocess.py b/src/sequifier/io/depth_preprocess.py index e834bddc..c53cbbf2 100644 --- a/src/sequifier/io/depth_preprocess.py +++ b/src/sequifier/io/depth_preprocess.py @@ -21,6 +21,8 @@ from loguru import logger from sequifier.config.depth_layout import DepthLayoutRegistryModel +from sequifier.config.metadata import validate_persisted_metadata_versions +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.helpers import PANDAS_TO_TORCH_TYPES from sequifier.io.pt_payload import ( StoredTensorBatch, @@ -843,10 +845,9 @@ def preprocess_depth(owner, selected_columns): raise ValueError(f"{filename}: missing columns {sorted(missing)}") for coordinate in ("sequenceId", "itemPosition"): dtype = schema[coordinate] - assert dtype.is_integer(), ( - f"{coordinate} must have an integer dtype in {filename}; " - f"found {dtype}." - ) + assert ( + dtype.is_integer() + ), f"{coordinate} must have an integer dtype in {filename}; found {dtype}." types = {c: schema[c] for c in columns} if schema_types is not None and types != schema_types: raise ValueError( @@ -989,6 +990,7 @@ def item_rows(item): existing = json.loads( (Path(owner.project_root) / owner.metadata_config_path).read_text() ) + validate_persisted_metadata_versions(existing) owner.metadata_fitted_on_all_data = existing.get("normalize_on_all_data", True) if ( DepthLayoutRegistryModel.model_validate(existing.get("depth_layouts", {})) @@ -1211,7 +1213,6 @@ def fit_jobs(): owner._write_or_validate_resume_manifest( selected_columns, "pt", columns, id_maps, n_classes, col_types, stats ) - owner._export_metadata(id_maps, n_classes, col_types, stats) # A shard is reusable only after both its payload and its ledger entry have # been committed. The ledger also ties completed shards to this source set. ledger_path = scratch_root / "depth-shards.json" @@ -1219,26 +1220,38 @@ def fit_jobs(): parallel_render = parallel_groups is not None if owner.continue_preprocessing and ledger_path.exists(): ledger = json.loads(ledger_path.read_text()) - if ( - ledger.get("version") not in (1, 2) - or ledger.get("sources") != source_identity - ): + require_contract_version( + "depth ledger", + ledger.get("version"), + ( + ContractVersions.SERIAL_DEPTH_LEDGER, + ContractVersions.PARALLEL_DEPTH_LEDGER, + ), + ) + if ledger.get("sources") != source_identity: raise ValueError( "Depth resume source files differ from the completed shards" ) - parallel_render = parallel_render and ledger["version"] == 2 - if ledger["version"] == 2 and not parallel_render: + was_parallel = ledger["version"] == ContractVersions.PARALLEL_DEPTH_LEDGER + if was_parallel and not parallel_render: raise ValueError( "Depth parallel resume requires the original Parquet row groups " "and num_workers > 1" ) + parallel_render = was_parallel else: ledger = { - "version": 2 if parallel_render else 1, + "version": ( + ContractVersions.PARALLEL_DEPTH_LEDGER + if parallel_render + else ContractVersions.SERIAL_DEPTH_LEDGER + ), "sources": source_identity, "shards": {}, } + owner._export_metadata(id_maps, n_classes, col_types, stats) + def shard_path(split, index): return Path(owner.split_paths[split]).stem + f"-0-{index:08d}.pt" diff --git a/src/sequifier/io/folder_dataset.py b/src/sequifier/io/folder_dataset.py index 29c5f97b..c381ef0f 100644 --- a/src/sequifier/io/folder_dataset.py +++ b/src/sequifier/io/folder_dataset.py @@ -12,6 +12,7 @@ from torch import Tensor from torch.utils.data import IterableDataset, get_worker_info +from sequifier.config.metadata import validate_persisted_metadata_versions from sequifier.helpers import ( WindowSampleIndex, configured_window_stride, @@ -76,6 +77,7 @@ def __init__(self, data_path: str, config: Any, shuffle: bool, format_hint: str) with open(metadata_path, "r") as f: metadata = json.load(f) + validate_persisted_metadata_versions(metadata) validate_folder_curriculum(config, metadata, self.data_dir) self.metadata = metadata diff --git a/src/sequifier/io/images.py b/src/sequifier/io/images.py index 2d8929e9..4364e618 100644 --- a/src/sequifier/io/images.py +++ b/src/sequifier/io/images.py @@ -16,6 +16,7 @@ import torch from sequifier.config.image_config import ImageSourceConfig, PreconditioningImageConfig +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.helpers import normalize_path IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"} @@ -32,9 +33,6 @@ class _VideoIndex: by_pts: dict[int, int] -_VIDEO_INDEX_VERSION = 1 - - def _media_signature(path: Path) -> dict[str, int]: stat = path.stat() return {"size": stat.st_size, "mtime_ns": stat.st_mtime_ns} @@ -247,7 +245,7 @@ def _scan_video(path: Path) -> dict[str, Any]: else None ) return { - "version": _VIDEO_INDEX_VERSION, + "version": ContractVersions.VIDEO_FRAME_INDEX, "media": _media_signature(path), "time_base": [base.numerator, base.denominator] if base else None, "frame_count": len(stamps), @@ -629,6 +627,11 @@ def _video_record(self, path: Path) -> dict[str, Any]: try: with index_path.open() as file: record = json.load(file) + require_contract_version( + "video frame index", + record.get("version") if isinstance(record, dict) else None, + ContractVersions.VIDEO_FRAME_INDEX, + ) import av with av.open(str(path)) as container: @@ -636,7 +639,6 @@ def _video_record(self, path: Path) -> dict[str, Any]: time_base = [base.numerator, base.denominator] if base else None if ( not isinstance(record, dict) - or record.get("version") != _VIDEO_INDEX_VERSION or record.get("media") != _media_signature(path) or record.get("time_base") != time_base or not isinstance(record.get("pts"), list) @@ -681,7 +683,7 @@ def _video_record(self, path: Path) -> dict[str, Any]: ) ) ): - raise ValueError("invalid version, signature, or frame data") + raise ValueError("invalid signature or frame data") except (OSError, ValueError, TypeError, KeyError) as error: raise ValueError( f"Video index for {path} is missing, stale, or invalid: {error}" diff --git a/src/sequifier/io/memory_budget.py b/src/sequifier/io/memory_budget.py index b94f0340..62a1bace 100644 --- a/src/sequifier/io/memory_budget.py +++ b/src/sequifier/io/memory_budget.py @@ -20,6 +20,7 @@ import torch from loguru import logger +from sequifier.config.metadata import validate_persisted_metadata_versions from sequifier.helpers import PANDAS_TO_TORCH_TYPES, normalize_path @@ -149,6 +150,7 @@ def _estimate_loader(path: str, config: Any, eager: bool) -> LoaderMemoryEstimat if source.is_dir(): with (source / "metadata.json").open() as stream: metadata = json.load(stream) + validate_persisted_metadata_versions(metadata) metadata_bytes = resident_bytes(metadata) width = metadata["window_length"] infos = metadata["batch_files"] diff --git a/src/sequifier/io/policy_window.py b/src/sequifier/io/policy_window.py index d93fe066..6417affd 100644 --- a/src/sequifier/io/policy_window.py +++ b/src/sequifier/io/policy_window.py @@ -4,6 +4,7 @@ from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_inputs import PolicyEpisodeSpec +from sequifier.contract_versions import ContractVersions from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view from sequifier.io.batch import SequifierInputs from sequifier.io.preprocessing import normalize_values @@ -56,7 +57,10 @@ def __init__(self, config: ResolvedPolicyBinding): self.config = config self.view = ModelWindowView(config.context_length, "policy", 0) self.resolved = resolve_window_view( - StoredWindowLayout(config.context_length, 0, 2), self.view + StoredWindowLayout( + config.context_length, 0, ContractVersions.STORED_WINDOW_LAYOUT + ), + self.view, ) def episode_inputs( diff --git a/src/sequifier/io/pt_payload.py b/src/sequifier/io/pt_payload.py index b3a86e00..69bd18e6 100644 --- a/src/sequifier/io/pt_payload.py +++ b/src/sequifier/io/pt_payload.py @@ -6,6 +6,7 @@ from torch import Tensor from sequifier.config.depth_layout import DepthLayoutRegistryModel +from sequifier.contract_versions import ContractVersions, require_contract_version def validate_tensor_inputs( @@ -87,6 +88,31 @@ class StoredTensorBatch: sample_positions: Tensor | None = None curriculum_columns: tuple[str, ...] = () + def slice_rows(self, start: int, stop: int) -> "StoredTensorBatch": + # Own just the selected storage so saving a shard cannot serialize its parent. + rows = slice(start, stop) + return StoredTensorBatch( + sequences={ + name: value[rows].clone() for name, value in self.sequences.items() + }, + depth_valid_masks={ + name: value[rows].clone() + for name, value in self.depth_valid_masks.items() + }, + sample_positions=( + self.sample_positions[rows].clone() + if self.sample_positions is not None + else None + ), + curriculum_columns=self.curriculum_columns, + sequence_ids=self.sequence_ids[rows].clone(), + subsequence_ids=self.subsequence_ids[rows].clone(), + start_item_positions=self.start_item_positions[rows].clone(), + left_pad_lengths=self.left_pad_lengths[rows].clone(), + split_start_item_positions=self.split_start_item_positions[rows].clone(), + split_end_item_positions=self.split_end_item_positions[rows].clone(), + ) + def validate(self, layouts=None, *, n_classes=None): layouts = DepthLayoutRegistryModel.model_validate(layouts or {}) missing = set(layouts.deep_columns) - set(self.sequences) @@ -200,36 +226,38 @@ def tabular_tensor_batch( def load_pt_payload(path, *, layouts=None, n_classes=None) -> StoredTensorBatch: value = torch.load(path, map_location="cpu", weights_only=True) - if isinstance(value, dict): - version = value.get("version") - if value.get("format") != "sequifier_tensor_batch" or version != 5: - raise ValueError(f"Unsupported PT tensor payload format/version in {path}") - expected = { - "format", - "version", - "sequences", - "metadata", - "sequence_ids", - "subsequence_ids", - "start_item_positions", - "left_pad_lengths", - } - expected.update({"split_start_item_positions", "split_end_item_positions"}) - expected_metadata = { - "depth_valid_masks", - "sample_positions", - "curriculum_columns", - } - if set(value) != expected or set(value["metadata"]) != expected_metadata: - raise ValueError(f"Inconsistent version {version} tensor payload schema") - batch = StoredTensorBatch( - **{key: value[key] for key in expected - {"format", "version", "metadata"}}, - depth_valid_masks=value["metadata"]["depth_valid_masks"], - sample_positions=value["metadata"].get("sample_positions"), - curriculum_columns=tuple(value["metadata"].get("curriculum_columns", ())), - ) - else: - raise ValueError(f"Unsupported PT tensor payload in {path}") + version = value.get("version") if isinstance(value, dict) else None + require_contract_version( + "PT tensor-batch envelope", version, ContractVersions.PT_TENSOR_BATCH + ) + if value.get("format") != "sequifier_tensor_batch": + raise ValueError(f"Unsupported PT tensor payload format in {path}") + expected = { + "format", + "version", + "sequences", + "metadata", + "sequence_ids", + "subsequence_ids", + "start_item_positions", + "left_pad_lengths", + "split_start_item_positions", + "split_end_item_positions", + } + expected_metadata = {"depth_valid_masks", "sample_positions", "curriculum_columns"} + metadata = value.get("metadata") + if ( + set(value) != expected + or not isinstance(metadata, dict) + or set(metadata) != expected_metadata + ): + raise ValueError(f"Inconsistent version {version} tensor payload schema") + batch = StoredTensorBatch( + **{key: value[key] for key in expected - {"format", "version", "metadata"}}, + depth_valid_masks=metadata["depth_valid_masks"], + sample_positions=metadata["sample_positions"], + curriculum_columns=tuple(metadata["curriculum_columns"]), + ) return batch.validate(layouts, n_classes=n_classes) @@ -238,7 +266,7 @@ def save_pt_payload(batch: StoredTensorBatch, path, *, layouts=None, n_classes=N torch.save( { "format": "sequifier_tensor_batch", - "version": 5, + "version": ContractVersions.PT_TENSOR_BATCH, "sequences": batch.sequences, "metadata": { "depth_valid_masks": batch.depth_valid_masks, diff --git a/src/sequifier/model/execution_schema.py b/src/sequifier/model/execution_schema.py index 3217c179..34006263 100644 --- a/src/sequifier/model/execution_schema.py +++ b/src/sequifier/model/execution_schema.py @@ -9,6 +9,7 @@ depth_mask_metadata_key, ) from sequifier.config.image_config import image_views +from sequifier.contract_versions import ContractVersions, require_contract_version EXECUTION_SCHEMA_KEY = "sequifier.execution_schema" MODEL_CONFIG_KEY = "sequifier.model_config" @@ -30,9 +31,16 @@ class ExecutionSchema: outputs: tuple[dict, ...] depth_layouts: dict n_classes: dict - version: int = 1 + version: int = ContractVersions.MODEL_EXECUTION_SCHEMA batch_policy: str = "symbolic" + def __post_init__(self): + require_contract_version( + "model execution schema", + self.version, + ContractVersions.MODEL_EXECUTION_SCHEMA, + ) + @classmethod def from_interface( cls, interface, context_length, *, embedding=False, embedding_width=None @@ -179,8 +187,13 @@ def to_dict(self): @classmethod def from_dict(cls, values): - if values.get("version") != 1 or values.get("batch_policy") != "symbolic": - raise ValueError("Unsupported execution schema version or batch policy") + require_contract_version( + "model execution schema", + values.get("version") if isinstance(values, dict) else None, + ContractVersions.MODEL_EXECUTION_SCHEMA, + ) + if values.get("batch_policy") != "symbolic": + raise ValueError("Unsupported execution schema batch policy") descriptors = tuple( InputDescriptor(**{**value, "shape": tuple(value["shape"])}) for value in values["inputs"] diff --git a/src/sequifier/model/hash_codec.py b/src/sequifier/model/hash_codec.py index 0036d1f3..2e15f6b8 100644 --- a/src/sequifier/model/hash_codec.py +++ b/src/sequifier/model/hash_codec.py @@ -5,7 +5,8 @@ import torch from torch import Tensor, nn -HASH_CODEC_VERSION = 1 +from sequifier.contract_versions import ContractVersions + HASH_PRIME = 2_147_483_647 @@ -78,7 +79,7 @@ def __init__(self, config, n_classes: int, decoder_ids: list[int] | None = None) self.num_buckets = config.num_buckets self.num_hashes = config.num_hashes self.seed = getattr(config, "seed", 0) - self.version = HASH_CODEC_VERSION + self.version = ContractVersions.HASH_CODEC self.widths = ( [config.num_buckets] * config.num_hashes if config.type == "multi_hash" @@ -182,7 +183,7 @@ def contract(self) -> dict: "offsets": self.offsets.tolist(), "codebook": self.codebook.tolist(), "decoder_ids": self.decoder_ids.tolist(), - "special_token_policy": "reserve_complete_codes_0_1_2_v1" + "special_token_policy": f"reserve_complete_codes_0_1_2_v{ContractVersions.RESERVED_CODE_SPECIAL_TOKENS}" if self.kind == "multi_hash" - else "qr_injective_v1", + else f"qr_injective_v{ContractVersions.INJECTIVE_SPECIAL_TOKENS}", } diff --git a/src/sequifier/model/initialization_seed.py b/src/sequifier/model/initialization_seed.py index 8a2bff91..bbe2f2f5 100644 --- a/src/sequifier/model/initialization_seed.py +++ b/src/sequifier/model/initialization_seed.py @@ -1,4 +1,4 @@ -"""Version 1 stable initialization namespaces, independent of runtime dropout.""" +"""Stable initialization namespaces, independent of runtime dropout.""" import hashlib import json @@ -8,14 +8,15 @@ import numpy as np import torch -SEED_DERIVATION_VERSION = 1 +from sequifier.contract_versions import ContractVersions def derived_seed(seed: int | None, path: tuple[str, ...], phase: str) -> int | None: if seed is None: return None value = json.dumps( - [SEED_DERIVATION_VERSION, seed, path, phase], separators=(",", ":") + [ContractVersions.INITIALIZATION_SEED_DERIVATION, seed, path, phase], + separators=(",", ":"), ) return int.from_bytes(hashlib.sha256(value.encode()).digest()[:8], "big") % (2**63) diff --git a/src/sequifier/preprocess.py b/src/sequifier/preprocess.py index aad7feda..0704aa97 100644 --- a/src/sequifier/preprocess.py +++ b/src/sequifier/preprocess.py @@ -22,10 +22,15 @@ from sequifier.config.depth_layout import DepthLayoutRegistryModel from sequifier.config.image_config import ImageSourceConfig, PreconditioningImageConfig +from sequifier.config.metadata import ( + dataset_tensor_version, + validate_persisted_metadata_versions, +) from sequifier.config.preprocess_config import ( CardinalityLimitModel, load_preprocessor_config, ) +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.helpers import ( FLOAT_TYPE_ORDER, PANDAS_TO_TORCH_TYPES, @@ -71,7 +76,6 @@ INPUT_METADATA_COLUMNS = ("sequenceId", "itemPosition") SPLIT_VALUE_COLUMN = "__sequifier_split_value" REAL_MASK_VALUE = 0.0 -CURRENT_STORED_WINDOW_LAYOUT_VERSION = 2 MAX_WINDOW_BUFFER_BYTES = 256 * 1024 * 1024 HASH_BUCKET_LABEL_PREFIX = "[hash_bucket:" STABLE_HASH_OFFSET_BASIS = 14695981039346656037 @@ -1216,7 +1220,7 @@ def __init__( self.window_lengths: list[int] = resolved_window_lengths self.split_layouts = [ StoredWindowLayout( - width, max_target_offset, CURRENT_STORED_WINDOW_LAYOUT_VERSION + width, max_target_offset, ContractVersions.STORED_WINDOW_LAYOUT ) for width in resolved_window_lengths ] @@ -1258,7 +1262,7 @@ def __init__( self.storage_layout = StoredWindowLayout( window_length=resolved_window_lengths[0], max_target_offset=max_target_offset, - version=CURRENT_STORED_WINDOW_LAYOUT_VERSION, + version=ContractVersions.STORED_WINDOW_LAYOUT, ) self.has_sample_positions = False self.image_manifest = self._validate_image_sources() @@ -1410,6 +1414,7 @@ def __init__( with open(metadata_path, "r") as f: preexisting_metadata = json.load(f) + validate_persisted_metadata_versions(preexisting_metadata) self.metadata_fitted_on_all_data = preexisting_metadata.get( "normalize_on_all_data", True @@ -1630,6 +1635,7 @@ def __init__( with open(metadata_path, "r") as f: preexisting_metadata = json.load(f) + validate_persisted_metadata_versions(preexisting_metadata) self.metadata_fitted_on_all_data = preexisting_metadata.get( "normalize_on_all_data", True @@ -2882,7 +2888,7 @@ def _cleanup(self, write_format: str) -> None: def _layout_metadata(self) -> dict[str, Any]: return { "depth_layouts": self.depth_layouts.model_dump(mode="json"), - "tensor_payload_version": 2 if self.depth_layouts else 1, + "tensor_payload_version": dataset_tensor_version(self.depth_layouts), "window_length": self.storage_layout.window_length, "window_lengths": self.window_lengths, "prediction_alignment_steps": self.prediction_alignment_steps, @@ -2905,7 +2911,7 @@ def _write_or_validate_resume_manifest( selected_columns_statistics: dict[str, dict[str, float]], ) -> None: manifest = { - "manifest_version": 1, + "manifest_version": ContractVersions.PREPROCESSING_MANIFEST, "preprocessing_config": { **self._layout_metadata(), "read_format": self.read_format, @@ -2978,31 +2984,13 @@ def _write_or_validate_resume_manifest( ) with open(manifest_path, "r") as f: previous_manifest = json.load(f) - previous_manifest.get("preprocessing_config", {}).setdefault( - "depth_layouts", {} - ) - previous_manifest.get("preprocessing_config", {}).setdefault( - "tensor_payload_version", 1 - ) - previous_manifest.get("preprocessing_config", {}).setdefault( - "normalize_on_all_data", True - ) - previous_manifest.get("preprocessing_config", {}).setdefault( - "categorical_columns", None + require_contract_version( + "preprocessing manifest", + previous_manifest.get("manifest_version"), + ContractVersions.PREPROCESSING_MANIFEST, ) - previous_manifest.get("preprocessing_config", {}).setdefault( - "real_columns", None - ) - previous_manifest.get("preprocessing_config", {}).setdefault( - "cardinality_config", {} - ) - previous_manifest.get("preprocessing_config", {}).setdefault( - "window_lengths", - [previous_manifest["preprocessing_config"]["window_length"]] - * self.n_splits, - ) - previous_manifest.get("preprocessing_config", {}).setdefault( - "prediction_alignment_steps", 1 + validate_persisted_metadata_versions( + previous_manifest["preprocessing_config"] ) if _stable_json_value(previous_manifest) != _stable_json_value(manifest): raise ValueError( @@ -3033,7 +3021,13 @@ def _restore_resume_manifest_state( ) with open(manifest_path, "r") as f: manifest = json.load(f) - previous = manifest.get("preprocessing_config", {}) + require_contract_version( + "preprocessing manifest", + manifest.get("manifest_version"), + ContractVersions.PREPROCESSING_MANIFEST, + ) + previous = manifest["preprocessing_config"] + validate_persisted_metadata_versions(previous) required = { "data_columns", "id_maps", diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index c7f74bea..c4729017 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -6,6 +6,7 @@ import torch +from sequifier.contract_versions import ContractVersions from sequifier.io.actions import action_record, decode_action from sequifier.rl.distributions import action_generator from sequifier.rl.environment import validate_reset, validate_step @@ -16,7 +17,7 @@ def stream_seed(seed, stream, reset_count): - value = f"sequifier-policy-v1:{seed}:{stream}:{reset_count}".encode() + value = f"sequifier-policy-v{ContractVersions.POLICY_COLLECTION_SEED}:{seed}:{stream}:{reset_count}".encode() return int.from_bytes(hashlib.blake2b(value, digest_size=8).digest(), "big") % ( 2**63 ) diff --git a/src/sequifier/rl/control.py b/src/sequifier/rl/control.py index 813248b8..fc992326 100644 --- a/src/sequifier/rl/control.py +++ b/src/sequifier/rl/control.py @@ -7,6 +7,7 @@ import random +from sequifier.contract_versions import ContractVersions from sequifier.rl.environment import ( EnvironmentCapabilities, OnlineResetResult, @@ -15,8 +16,10 @@ class CategoricalControlAdapter: - protocol_version = 3 - implementation_id = "sequifier.categorical-control.v1" + protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER + implementation_id = ( + f"sequifier.categorical-control.v{ContractVersions.CATEGORICAL_CONTROL}" + ) capabilities = EnvironmentCapabilities( seeded_reset=True, exact_snapshot=True, diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py index 269798e3..61c97849 100644 --- a/src/sequifier/rl/environment.py +++ b/src/sequifier/rl/environment.py @@ -7,6 +7,7 @@ import numpy as np +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.io.values import ( DomainValue, numeric_scalar, @@ -96,8 +97,11 @@ def load_online_adapter(spec): def validate_online_adapter(adapter, spec): - if getattr(adapter, "protocol_version", None) != 3: - raise ValueError("online adapters must explicitly declare protocol_version = 3") + require_contract_version( + "online policy environment adapter protocol", + getattr(adapter, "protocol_version", None), + ContractVersions.ONLINE_POLICY_ADAPTER, + ) if getattr(adapter, "implementation_id", None) != spec.implementation_id: raise ValueError( "environment.implementation_id differs from adapter implementation" diff --git a/src/sequifier/rl/grpo.py b/src/sequifier/rl/grpo.py index 0631e8fe..1e1f700f 100644 --- a/src/sequifier/rl/grpo.py +++ b/src/sequifier/rl/grpo.py @@ -9,6 +9,7 @@ from loguru import logger from sequifier.config.policy_plan import GRPOPhaseConfig +from sequifier.contract_versions import ContractVersions from sequifier.io.atomic import write_json_atomic from sequifier.model.frozen_image_cache import ( frozen_image_cache_scope, @@ -130,7 +131,7 @@ def run_grpo_phase(context: PhaseContext[GRPOPhaseConfig, None]): "grpo", { "format": "sequifier.grpo-archive", - "version": 1, + "version": ContractVersions.GRPO_ARCHIVE, "reference_identity": context.reference_identity, "generation": buffer.generation, "trajectories": [ diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index b1d2d34f..25aa74cd 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -172,28 +172,24 @@ def run_policy_training( def run_policy_phase( - config: ResolvedPolicyRun, identity: RunIdentity, *, legacy_recovery: bool = False + config: ResolvedPolicyRun, identity: RunIdentity ) -> PolicyRunResult: """Run one prepared policy phase for a portable handoff.""" if len(config.phases) != 1: raise ValueError("a portable policy phase requires one resolved phase") - return _run_policy( - config, phase_only=True, identity=identity, legacy_recovery=legacy_recovery - ) + return _run_policy(config, phase_only=True, identity=identity) -def _run_policy(config, *, phase_only=False, identity=None, legacy_recovery=False): +def _run_policy(config, *, phase_only=False, identity=None): prepared = PreparedTrainingInputs.for_run(config) with prepared.run_scope(): try: - return _execute_policy( - config, prepared, phase_only, identity, legacy_recovery - ) + return _execute_policy(config, prepared, phase_only, identity) finally: config._metrics_service = None -def _execute_policy(config, prepared, phase_only, identity, legacy_recovery): +def _execute_policy(config, prepared, phase_only, identity): torch.set_float32_matmul_precision("highest") configure_determinism(config.seed, config.enforce_determinism) plan = _plan_contract(config) @@ -208,9 +204,7 @@ def _execute_policy(config, prepared, phase_only, identity, legacy_recovery): config, plan, data_identity, - run_id=identity.run_id - if identity is not None and not legacy_recovery - else None, + run_id=identity.run_id if identity is not None else None, ) resuming = recovery is not None adapter_contract = preflight_environments(config) diff --git a/src/sequifier/rl/ppo_buffer.py b/src/sequifier/rl/ppo_buffer.py index 1b80ff12..c294977d 100644 --- a/src/sequifier/rl/ppo_buffer.py +++ b/src/sequifier/rl/ppo_buffer.py @@ -7,6 +7,7 @@ import torch from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.contract_versions import ContractVersions from sequifier.io.actions import action_batch, decode_action from sequifier.io.atomic import write_json_atomic from sequifier.rl.history import frame_storage_bytes, history_inputs @@ -137,7 +138,7 @@ def archive(self, path, *, include_scores): decisions.append(record) payload = { "format": "sequifier.ppo-collection", - "version": 2, + "version": ContractVersions.PPO_COLLECTION, "policy_version": self.policy_version, "policy": self.spec.model_dump(mode="json"), "frames": [frame.state_dict() for frame in self.frames], @@ -149,14 +150,10 @@ def archive(self, path, *, include_scores): def load_collection_record(path): - """Read the independent archival format without changing v1 episode readers.""" + """Read and validate a current PPO collection record.""" from sequifier.trajectory.archives import read_collection_archive payload = read_collection_archive(path, "ppo") - if payload.get("format") != "sequifier.ppo-collection" or payload.get( - "version" - ) not in {1, 2}: - raise ValueError("expected version-1 PPO collection record") from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.rl.history import PolicyFrame diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 7abea3ef..eb47e22a 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -16,12 +16,12 @@ ModelExecutionConfig, ) from sequifier.artifacts.phase import PhaseArtifacts +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save from sequifier.rl.checkpoint import phase_actor_path, policy_model_artifact from sequifier.rl.references import validate_reference_state from sequifier.rl.run_state import ( - PHASE_STATE_VERSION, ActorRepresentation, PolicyPhaseResult, clone_weights, @@ -33,7 +33,6 @@ from sequifier.runtime.random_state import capture_rng_state from sequifier.trajectory.sources import file_checksum -POLICY_RECOVERY_VERSION = 8 ONLINE_TELEMETRY = ("evaluations", "continuations") @@ -45,13 +44,13 @@ class CompletedPhase: telemetry_position: str | None = None @classmethod - def from_dict(cls, values, *, legacy): + def from_dict(cls, values): result = values["result"] return cls( PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}), values["sha256"], values["result_fingerprint"], - None if legacy else values["telemetry_position"], + values["telemetry_position"], ) def as_dict(self): @@ -107,31 +106,27 @@ def load(cls, path, config, plan, data_identity, *, run_id=None): payload = recovery_payload( torch.load(path, map_location="cpu", weights_only=False), "policy" ) - if ( - not isinstance(payload, dict) - or payload.get("artifact_type") != "sequifier_policy_run" - or payload.get("format_version") not in {7, POLICY_RECOVERY_VERSION} - ): - raise ValueError( - "policy resume requires a version-7 or version-8 policy run checkpoint; initialize from its model artifact to start a new run" - ) + require_contract_version( + "policy recovery", + payload.get("format_version"), + ContractVersions.POLICY_RECOVERY, + ) + if payload.get("artifact_type") != "sequifier_policy_run": + raise ValueError("Expected a policy checkpoint recovery payload") if run_id is not None and payload.get("run_id") != run_id: return None try: - legacy = payload["format_version"] == 7 saved_plan = payload["plan"] require_equal(plan, saved_plan, "resume.plan") require_equal(data_identity, payload["data_identity"], "resume.data") completed = [ - CompletedPhase.from_dict(record, legacy=legacy) - for record in payload["completed"] + CompletedPhase.from_dict(record) for record in payload["completed"] ] checkpoint = cls( **{ f.name: payload[f.name] for f in fields(cls) if f.name not in {"representation", "completed"} - and f.name in payload }, representation=ActorRepresentation(payload["representation"]), completed=completed, @@ -141,14 +136,12 @@ def load(cls, path, config, plan, data_identity, *, run_id=None): payload["representation_fingerprint"], "resume.representation_fingerprint", ) - checkpoint.validate(config, legacy=legacy) - if legacy: - checkpoint.migrate_telemetry(config) + checkpoint.validate(config) return checkpoint except (KeyError, TypeError, AttributeError, IndexError) as exc: raise ValueError(f"malformed policy run checkpoint: {exc}") from exc - def validate(self, config, *, legacy=False): + def validate(self, config): if self.phase_index > len(config.phases): raise ValueError("checkpoint completed handoffs disagree with the plan") validate_finite_state(self.incoming_actor, "incoming_actor") @@ -175,8 +168,7 @@ def validate(self, config, *, legacy=False): record.sha256, file_checksum(path), "resume.phase_actor.sha256" ) result_data = result.as_dict() - if not legacy: - result_data["telemetry_position"] = record.telemetry_position + result_data["telemetry_position"] = record.telemetry_position require_equal( record.result_fingerprint, fingerprint(result_data), @@ -196,9 +188,12 @@ def validate(self, config, *, legacy=False): return if self.phase_index == len(config.phases): raise ValueError("completed run cannot have a live phase state") - if state["version"] != (3 if legacy else PHASE_STATE_VERSION): - raise ValueError("unsupported phase checkpoint state version") - if not legacy and not isinstance(state["telemetry_position"], str): + require_contract_version( + "policy phase state", + state.get("version"), + ContractVersions.POLICY_PHASE_STATE, + ) + if not isinstance(state["telemetry_position"], str): raise ValueError("phase checkpoint requires a committed telemetry position") transaction = state["transaction"] if ( @@ -225,36 +220,6 @@ def validate(self, config, *, legacy=False): validate_reference_state(config.phases[self.phase_index], self) validate_finite_state(state, "phase") - def migrate_telemetry(self, config): - for index, record in enumerate(self.completed): - result = record.result - artifacts = PhaseArtifacts(Path(result.provenance["actor_path"])) - provenance = dict(result.provenance) - histories = {key: provenance.pop(key, ()) for key in ONLINE_TELEMETRY} - position = import_telemetry( - artifacts.telemetry, {"metrics": result.metrics, **histories} - ) - compact = replace(result, metrics=(), provenance=provenance) - self.completed[index] = replace( - record, - result=compact, - result_fingerprint=fingerprint( - {**compact.as_dict(), "telemetry_position": position} - ), - telemetry_position=position, - ) - if self.phase_state is not None: - state = self.phase_state - phase = config.phases[self.phase_index] - artifacts = PhaseArtifacts.for_phase( - config.phase_directory, self.phase_index, phase.name - ) - state["telemetry_position"] = import_telemetry( - artifacts.telemetry, - {key: state.pop(key, ()) for key in ("metrics", *ONLINE_TELEMETRY)}, - ) - state["version"] = PHASE_STATE_VERSION - def validate_actor(self, actor, units): self.representation.validate_actor(actor, units, "resume.representation") expected = actor.network.state_dict() @@ -283,7 +248,7 @@ def save(self, path, device, state=None): self.random = capture_rng_state(device) self.phase_state = state model = ModelArtifact( - 3, + ContractVersions.MODEL_ARTIFACT, ModelExecutionConfig(self.representation.values["execution"]), state["networks"]["actor"] if state else self.incoming_actor, ModelArtifactMetadata(), @@ -294,7 +259,7 @@ def save(self, path, device, state=None): "policy", { "artifact_type": "sequifier_policy_run", - "format_version": POLICY_RECOVERY_VERSION, + "format_version": ContractVersions.POLICY_RECOVERY, **{ f.name: getattr(self, f.name) for f in fields(self) @@ -333,14 +298,6 @@ def finish_phase(self, actor, result, telemetry_position): self.phase_state = None -def import_telemetry(path, histories): - with TelemetryJournal.open(path) as journal: - for kind, records in histories.items(): - for record in records: - journal.append(kind, record) - return journal.commit() - - def validate_finite_state(value, field): """Check algorithm-owned optimizer/network trees before restoring them.""" if isinstance(value, torch.Tensor): diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index de89aac3..e36e904a 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -11,7 +11,7 @@ from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_plan import PhaseConfig -from sequifier.rl.models import HistoryCritic +from sequifier.contract_versions import ContractVersions from sequifier.rl.policy import Policy from sequifier.rl.telemetry import TelemetryJournal from sequifier.runtime.plan import metric_improved, select_handoff, stop_scope @@ -23,7 +23,6 @@ PhaseType = TypeVar("PhaseType", bound=PhaseConfig) DataType = TypeVar("DataType") -PHASE_STATE_VERSION = 4 def clone_weights(module): @@ -56,53 +55,6 @@ def require_equal(expected, actual, field): raise ValueError(f"{field}: expected {expected!r}, received {actual!r}") -def _legacy_critic_prefixes(module, interface_names): - prefixes = [] - for path, critic in module.named_modules(): - if isinstance(critic, HistoryCritic): - root = f"{path + '.' if path else ''}network.interfaces." - active = root + critic.interface_name - prefixes.extend((f"{active}.decoder.", f"{active}.prediction_scales.")) - prefixes.extend( - f"{root}{name}." - for name in interface_names - if name != critic.interface_name - ) - return tuple(prefixes) - - -def _prune_legacy_optimizer(state, optimizer, module, weights, saved_mask, mask): - """Restore positional optimizer slots after dropping unused critic parameters.""" - groups = state["param_groups"] - if len(groups) != 1 or len(optimizer.param_groups) != 1: - raise ValueError( - "legacy critic optimizer recovery cannot remove parameters from a multi-group plan" - ) - names = set(saved_mask) - saved_order = [name for name in weights if name in names] - group = groups[0] - slots = group["params"] - require_equal(len(names), len(saved_order), "legacy critic optimizer names") - require_equal(len(saved_order), len(slots), "legacy critic optimizer slots") - current_names = {id(p): name for name, p in module.named_parameters()} - keep = set(mask) - require_equal( - [current_names[id(p)] for p in optimizer.param_groups[0]["params"]], - [name for name in saved_order if name in keep], - "legacy critic optimizer order", - ) - dropped = {slot for name, slot in zip(saved_order, slots) if name not in keep} - return { - **state, - "param_groups": [ - {**group, "params": [slot for slot in slots if slot not in dropped]} - ], - "state": { - slot: value for slot, value in state["state"].items() if slot not in dropped - }, - } - - @dataclass(frozen=True) class ActorRepresentation: # Canonical JSON provides an immutable owned value, even for nested schemas. @@ -377,40 +329,18 @@ def _restore_networks_and_optimizers(self, state, field): sorted(state["optimization"]), f"{field}.optimization", ) - interface_names = self.context.actor.execution_config.interfaces for role, module in self.networks.items(): - weights = state["networks"][role] - prefixes = ( - _legacy_critic_prefixes(module, interface_names) - if role != "actor" - else () - ) - obsolete = ( - {name for name in weights if name.startswith(prefixes)} - if prefixes - else set() - ) - module.load_state_dict( - {name: value for name, value in weights.items() if name not in obsolete} - if obsolete - else weights, - strict=True, - ) - if role not in self.optimization: - continue - mask = self.mask_ids[role] - saved_mask = state["trainable_parameters"][role] - retained = [name for name in saved_mask if name not in obsolete] - require_equal(mask, retained, f"{field}.trainability.{role}") - optimizer, scheduler = self.optimization[role] - saved = state["optimization"][role] - optimizer_state = saved["optimizer"] - if retained != saved_mask: - optimizer_state = _prune_legacy_optimizer( - optimizer_state, optimizer, module, weights, saved_mask, mask + module.load_state_dict(state["networks"][role], strict=True) + if role in self.optimization: + require_equal( + self.mask_ids[role], + state["trainable_parameters"][role], + f"{field}.trainability.{role}", ) - optimizer.load_state_dict(optimizer_state) - scheduler.load_state_dict(saved["scheduler"]) + optimizer, scheduler = self.optimization[role] + saved = state["optimization"][role] + optimizer.load_state_dict(saved["optimizer"]) + scheduler.load_state_dict(saved["scheduler"]) def restore_random(self): if self.state: @@ -591,7 +521,7 @@ def checkpoint(self, final, **extra): self.context.commit( { **extra, - "version": PHASE_STATE_VERSION, + "version": ContractVersions.POLICY_PHASE_STATE, "networks": { name: clone_weights(module) for name, module in self.networks.items() diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index f701082c..e8d24c71 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -6,6 +6,7 @@ from sequifier.config.policy_infer_config import PolicyRolloutConfig from sequifier.config.rollout_config import InputBinding +from sequifier.contract_versions import ContractVersions from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.values import numeric_scalar from sequifier.rl.collection import PolicyCursor @@ -130,7 +131,7 @@ def collect_policy_rollout( break return recorder.finish( format="sequifier.policy-trajectory", - version=2, + version=ContractVersions.JOINT_POLICY_TRAJECTORY, seed=seed, bindings=[ InputBinding(input=name, source="environment", key=name) diff --git a/src/sequifier/rollout/coordinator.py b/src/sequifier/rollout/coordinator.py index 2c5ba3ad..817c6250 100644 --- a/src/sequifier/rollout/coordinator.py +++ b/src/sequifier/rollout/coordinator.py @@ -5,6 +5,7 @@ from uuid import uuid4 from sequifier.config.rollout_config import RolloutConfig +from sequifier.contract_versions import ContractVersions from sequifier.io.values import DomainValue from sequifier.rollout.contracts import ( EnvironmentAdapter, @@ -136,6 +137,7 @@ def collect_rollout( break return recorder.finish( format="sequifier.numeric-forecast-trajectory", + version=ContractVersions.FORECAST_TRAJECTORY, seed=initial_history, bindings=config.bindings, prediction_columns=prediction_columns, diff --git a/src/sequifier/rollout/factories.py b/src/sequifier/rollout/factories.py index 4bd812ec..8bf7d9c5 100644 --- a/src/sequifier/rollout/factories.py +++ b/src/sequifier/rollout/factories.py @@ -4,17 +4,17 @@ from typing import cast from sequifier.config.rollout_config import AdapterConfig +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.rollout.contracts import EnvironmentAdapter def validate_adapter(adapter): protocol_version = getattr(adapter, "protocol_version", None) - if ( - not isinstance(protocol_version, int) - or isinstance(protocol_version, bool) - or protocol_version != 1 - ): - raise ValueError("rollout adapter must declare protocol_version = 1") + require_contract_version( + "forecast rollout adapter protocol", + protocol_version, + ContractVersions.FORECAST_ROLLOUT_ADAPTER, + ) for method in ("reset", "step", "close"): if not callable(getattr(adapter, method, None)): raise ValueError(f"rollout adapter requires a callable {method} method") diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index ee49af2c..274d18c6 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -170,7 +170,6 @@ def build( *, initial_artifact=None, identity: RunIdentity | None = None, - legacy_recovery: bool = False, ) -> TrainingRun: from sequifier.artifacts.run_checkpoint import ( checkpoint_path as output_checkpoint_path, @@ -202,7 +201,6 @@ def build( if ( loaded is not None and identity is not None - and not legacy_recovery and loaded.checkpoint.run_state["run_id"] != identity.run_id ): loaded = None diff --git a/src/sequifier/templates/train-grpo.yaml b/src/sequifier/templates/train-grpo.yaml index e3bcee12..e5772faf 100644 --- a/src/sequifier/templates/train-grpo.yaml +++ b/src/sequifier/templates/train-grpo.yaml @@ -46,13 +46,13 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 evaluation: environment: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 seeds: - 17 - 29 diff --git a/src/sequifier/templates/train-ppo.yaml b/src/sequifier/templates/train-ppo.yaml index 96e3a5d4..d95f8c3a 100644 --- a/src/sequifier/templates/train-ppo.yaml +++ b/src/sequifier/templates/train-ppo.yaml @@ -42,7 +42,7 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 total_env_steps: 8192 rollout_env_steps: 256 optimization_epochs: 4 @@ -55,7 +55,7 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v1 + implementation_id: sequifier.categorical-control.v136 seeds: - 17 - 29 diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index 5029c10c..66c0aee0 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -9,12 +9,12 @@ from sequifier.artifacts.model_artifact import build_model_artifact from sequifier.artifacts.model_export import model_execution_config from sequifier.artifacts.run_checkpoint import ( - RUN_CHECKPOINT_FORMAT_VERSION, IntegrationState, RunCheckpoint, RunCheckpointStore, load_run_checkpoint, ) +from sequifier.contract_versions import ContractVersions from sequifier.integration.contexts import CheckpointSaved, CheckpointSaving from sequifier.training.loader_state import LoaderStateService @@ -205,7 +205,7 @@ def save(self, request: CheckpointRequest, run: Any) -> Path | None: if self.distributed.rank != 0: return None checkpoint = RunCheckpoint( - format_version=RUN_CHECKPOINT_FORMAT_VERSION, + format_version=ContractVersions.SUPERVISED_RECOVERY, model=build_model_artifact( run.network, run.config, diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index a3d8fe3c..ad2bdaa8 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -5,7 +5,7 @@ Completing a phase publishes its selected artifact before advancing the envelope. """ -from dataclasses import dataclass, replace +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -22,6 +22,7 @@ ) from sequifier.artifacts.phase import PhaseArtifacts, verify_artifact from sequifier.artifacts.reports import phase_report, publish_run_report, run_report +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save from sequifier.runtime.plan import PhaseOutcome, RunIdentity, new_run_id, run_phases @@ -93,32 +94,13 @@ def run_policy_plan(config): return run_learning_plan(config) -def _legacy_learning_plan_identity(plan): - """Recover the manifest identity used before runs serialized an interface registry.""" - legacy = dict(plan) - legacy["dataset_training"] = { - name: { - "name": name, - "model_interface": name, - "interface": interface, - "parts": {}, - "criterion": { - target: "CrossEntropyLoss" if kind == "categorical" else "MSELoss" - for target, kind in interface["target_column_types"].items() - }, - "class_weights": None, - "loss_weights": None, - "class_share_log_columns": [], - } - for name, interface in legacy.pop("interfaces").items() - } - legacy.update(curriculum_training=False, curriculum_column=None) - return fingerprint(legacy) - - def _completed_report(item): - if "report" in item: - return item["report"] + if item["algorithm"] == "supervised": + report = item["report"] + require_contract_version( + "phase report", report.get("format_version"), ContractVersions.PHASE_REPORT + ) + return report artifacts = PhaseArtifacts(Path(item["model_path"])) receipt = artifacts.read_receipt(**item["report_receipt"]) if receipt is None: @@ -153,7 +135,7 @@ def _run_learning_plan(config, prepared) -> LearningRunResult: identity = fingerprint(plan) state: dict[str, Any] = { "artifact_type": "sequifier_learning_run", - "format_version": 3, + "format_version": ContractVersions.LEARNING_RUN_STATE, "plan": identity, "completed": [], "incoming": config.initialize_from, @@ -167,19 +149,23 @@ def _run_learning_plan(config, prepared) -> LearningRunResult: state = recovery_payload( torch.load(checkpoint, map_location="cpu", weights_only=False), "learning" ) + require_contract_version( + "learning run coordinator state", + state.get("format_version"), + ContractVersions.LEARNING_RUN_STATE, + ) if ( state.get("artifact_type") != "sequifier_learning_run" - or state.get("format_version") not in {2, 3} - or ( - state.get("plan") != identity - and state.get("plan") != _legacy_learning_plan_identity(plan) - ) + or state.get("plan") != identity ): raise ValueError("learning run checkpoint differs from the requested plan") - state["plan"] = identity + if state["active"] and state.get("active_run_id") != state["run_id"]: + raise ValueError( + "learning run checkpoint requires the active phase run identity" + ) elif resume.policy == "required": raise FileNotFoundError(checkpoint) - identity_scope = RunIdentity.start(state.get("run_id")) + identity_scope = RunIdentity.start(state["run_id"]) state["run_id"] = identity_scope.run_id if not 0 <= len(state["completed"]) <= len(phases): raise ValueError("learning run checkpoint exceeds the phase plan") @@ -264,27 +250,9 @@ def _run_learning_plan(config, prepared) -> LearningRunResult: validate_training_destinations(config, checkpoint, protected) def save_boundary(): - state["format_version"] = 3 model = ( read_execution_artifact(state["incoming"]) if state["incoming"] else None ) - if ( - model is not None - and state["completed"] - and (state["completed"][-1]["algorithm"] != "supervised") - ): - # Older phase models may still embed their policy histories. - provenance = dict(model.metadata.provenance) - provenance.pop("metrics", None) - if "provenance" in provenance: - provenance["provenance"] = { - key: value - for key, value in provenance["provenance"].items() - if key not in {"evaluations", "continuations"} - } - model = replace( - model, metadata=replace(model.metadata, provenance=provenance) - ) atomic_save(checkpoint, checkpoint_envelope("learning", state, model)) def run_phase(index, phase): @@ -296,7 +264,6 @@ def run_phase(index, phase): ) artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) resuming = state["active"] and resume.policy != "never" - legacy_recovery = resuming and "active_run_id" not in state if not state["active"]: state.update(active=True, active_run_id=identity_scope.run_id) save_boundary() @@ -316,7 +283,7 @@ def run_phase(index, phase): report = receipt["report"] else: request = PortablePhaseRequest( - index, incoming, artifacts, resuming, legacy_recovery, identity_scope + index, incoming, artifacts, resuming, identity_scope ) runner = ( run_supervised_phase @@ -395,22 +362,11 @@ def commit_phase(index, phase, outcome): if index == last_data_phase: prepared.release_data() - migrated = False for index, item in enumerate(state["completed"]): artifacts = PhaseArtifacts.for_phase( config.phase_directory, index, phases[index].name ) - report = _completed_report(item) - if item["algorithm"] != "supervised" and "report" in item: - receipt = {"identity": fingerprint(report), "run_id": identity_scope.run_id} - artifacts.write_receipt(report=report, **receipt) - item["report_receipt"] = receipt - del item["report"] - migrated = True - artifacts.publish_report(report) - del report - if migrated: - save_boundary() + artifacts.publish_report(_completed_report(item)) completion_reason = run_phases( phases, run_phase, diff --git a/src/sequifier/training/metrics.py b/src/sequifier/training/metrics.py index f361115c..a67aa257 100644 --- a/src/sequifier/training/metrics.py +++ b/src/sequifier/training/metrics.py @@ -13,13 +13,12 @@ import numpy as np from loguru import logger +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.logging_paths import dataset_artifact_prefix from sequifier.typechecking import beartype TOTAL_TARGET = "__total__" -METRICS_SCHEMA_VERSION = 3 - COMMON_FIELDS = [ "schema_version", "phase", @@ -124,6 +123,23 @@ def _format_number(number: float) -> str: return f"{adjusted:5.2f}e{order_of_magnitude}" +def validate_metrics_csv_header(fields, path): + if fields is None or "schema_version" not in fields: + raise ValueError( + f"Unsupported metrics schema in {path}: missing schema_version, " + f"expected {ContractVersions.METRICS_SCHEMA}" + ) + + +def validate_metrics_csv_row(row, path): + value = row.get("schema_version") + if value != str(ContractVersions.METRICS_SCHEMA): + raise ValueError( + f"Unsupported metrics schema version {value!r} in {path}; " + f"expected {ContractVersions.METRICS_SCHEMA}" + ) + + class _CsvAppender: """Validate and append complete row groups to one CSV file.""" @@ -138,7 +154,12 @@ def __init__(self, path: Path, fields: list[str]) -> None: def _ensure_header(self) -> None: if self.path.exists() and self.path.stat().st_size > 0: with self.path.open("r", encoding="utf-8", newline="") as file: - existing = next(csv.reader(file), None) + reader = csv.DictReader(file) + existing = reader.fieldnames + if "schema_version" in self.fields: + validate_metrics_csv_header(existing, self.path) + for row in reader: + validate_metrics_csv_row(row, self.path) if existing != self.fields: raise ValueError( f"Structured metric schema mismatch in {self.path}: " @@ -152,15 +173,20 @@ def _ensure_header(self) -> None: @beartype def append(self, rows: Iterable[Mapping[str, Any]], *, durable: bool) -> None: - row_group = list(rows) + row_group = [dict(row) for row in rows] + if "schema_version" in self.fields: + for row in row_group: + version = row.setdefault( + "schema_version", ContractVersions.METRICS_SCHEMA + ) + require_contract_version( + "metrics schema", version, ContractVersions.METRICS_SCHEMA + ) if not row_group: return with self.path.open("a", encoding="utf-8", newline="") as file: writer = csv.DictWriter(file, fieldnames=self.fields, extrasaction="raise") for row in row_group: - row = dict(row) - if "schema_version" in self.fields: - row.setdefault("schema_version", METRICS_SCHEMA_VERSION) if "metric" in self.fields: row.setdefault("metric", "loss") if "is_primary" in self.fields: @@ -242,7 +268,7 @@ def _common( part: str | None = None, ) -> dict[str, Any]: return { - "schema_version": METRICS_SCHEMA_VERSION, + "schema_version": ContractVersions.METRICS_SCHEMA, "phase": self.phase, "algorithm": self.algorithm, "iteration": self.iteration, diff --git a/src/sequifier/training/phase_runners.py b/src/sequifier/training/phase_runners.py index a2ef59e4..ad73d99c 100644 --- a/src/sequifier/training/phase_runners.py +++ b/src/sequifier/training/phase_runners.py @@ -15,7 +15,6 @@ class PortablePhaseRequest: incoming: str | None artifacts: PhaseArtifacts resuming: bool - legacy_recovery: bool identity: RunIdentity @@ -86,7 +85,6 @@ def supervised_phase_config(config, phase, index, incoming, resume): "interfaces": interfaces, "dataset_training": datasets, "training_plan": [phase.model_copy(deep=True)], - "active_phase_index": 0, "runtime_settings": training, "seed": config.seed + index, "model_name": f"{config.model_name}-phase-{index:03d}-{phase.name}", @@ -134,7 +132,6 @@ def run_supervised_phase( integrations, initial_artifact=request.incoming, identity=request.identity, - legacy_recovery=request.legacy_recovery, ) try: report = TrainingEngine().run_phase(run) @@ -176,9 +173,7 @@ def run_portable_policy_phase( request.artifacts.model, request.resuming, ) - result = run_policy_phase( - run, request.identity, legacy_recovery=request.legacy_recovery - ).phases[0] + result = run_policy_phase(run, request.identity).phases[0] return PortablePhaseResult( read_execution_artifact(result.provenance["actor_path"]), result.as_dict() ) diff --git a/src/sequifier/trajectory/archives.py b/src/sequifier/trajectory/archives.py index c264f1c5..a0f9d5dc 100644 --- a/src/sequifier/trajectory/archives.py +++ b/src/sequifier/trajectory/archives.py @@ -3,11 +3,13 @@ import json from pathlib import Path +from sequifier.contract_versions import ContractVersions, require_contract_version + def collection_archive(algorithm, payload): return { "format": "sequifier.collection-archive", - "version": 1, + "version": ContractVersions.COLLECTION_ARCHIVE, "algorithm": algorithm, "collection": payload, } @@ -15,24 +17,31 @@ def collection_archive(algorithm, payload): def read_collection_archive(path, algorithm=None): payload = json.loads(Path(path).read_text(encoding="utf-8")) - legacy_formats = { - "ppo": "sequifier.ppo-collection", - "grpo": "sequifier.grpo-archive", + require_contract_version( + "collection archive envelope", + payload.get("version") if isinstance(payload, dict) else None, + ContractVersions.COLLECTION_ARCHIVE, + ) + if payload.get("format") != "sequifier.collection-archive": + raise ValueError("Expected a Sequifier collection archive envelope") + actual = payload.get("algorithm") + if algorithm is not None and actual != algorithm: + raise ValueError(f"Expected {algorithm} collection archive") + contracts = { + "ppo": ("sequifier.ppo-collection", ContractVersions.PPO_COLLECTION), + "grpo": ("sequifier.grpo-archive", ContractVersions.GRPO_ARCHIVE), } - if isinstance(payload, dict) and payload.get("format") in legacy_formats.values(): - if algorithm is not None and payload["format"] != legacy_formats.get(algorithm): - raise ValueError(f"Expected {algorithm} collection archive") - return payload - if payload.get("format") == "sequifier.collection-archive": - if payload.get("version") != 1: - raise ValueError("Unsupported collection archive envelope version") - if algorithm is not None and payload.get("algorithm") != algorithm: - raise ValueError(f"Expected {algorithm} collection archive") - if payload.get("algorithm") not in {"ppo", "grpo"} or not isinstance( - payload.get("collection"), dict - ): - raise ValueError( - "Collection archive requires a known algorithm and collection mapping" - ) - return payload["collection"] - raise ValueError("Expected a Sequifier collection archive envelope") + collection = payload.get("collection") + if ( + not isinstance(actual, str) + or actual not in contracts + or not isinstance(collection, dict) + ): + raise ValueError( + "Collection archive requires a known algorithm and collection mapping" + ) + kind, version = contracts[actual] + require_contract_version(kind, collection.get("version"), version) + if collection.get("format") != kind: + raise ValueError(f"Expected {kind} collection payload") + return collection diff --git a/src/sequifier/trajectory/export.py b/src/sequifier/trajectory/export.py index 77c81faf..beeb4deb 100644 --- a/src/sequifier/trajectory/export.py +++ b/src/sequifier/trajectory/export.py @@ -7,6 +7,7 @@ from pathlib import Path from sequifier.config.policy_train_config import PolicyTableColumns +from sequifier.contract_versions import ContractVersions from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.sequence import TransitionColumns from sequifier.trajectory.importers import source_episodes @@ -147,7 +148,7 @@ def flush(): flush() manifest = { "format": "sequifier.sequence-data", - "version": 1, + "version": ContractVersions.SEQUENCE_DATA_MANIFEST, "episodes": len(seen), "decisions": sum(shard["decisions"] for shard in shards), "columns": columns.model_dump(), diff --git a/src/sequifier/trajectory/preferences.py b/src/sequifier/trajectory/preferences.py index aa97c2cb..d30b84f7 100644 --- a/src/sequifier/trajectory/preferences.py +++ b/src/sequifier/trajectory/preferences.py @@ -6,6 +6,8 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from sequifier.contract_versions import ContractVersions, require_contract_version + class PreferenceSpan(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) @@ -35,9 +37,23 @@ class PreferencePair(BaseModel): class PreferenceManifest(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) format: Literal["sequifier.preference-pairs"] - version: Literal[1] + version: int = Field( + strict=True, + json_schema_extra={"const": ContractVersions.PREFERENCE_PAIR_MANIFEST}, + ) pairs: list[PreferencePair] = Field(min_length=1) + @model_validator(mode="before") + @classmethod + def validate_version(cls, values): + if isinstance(values, dict): + require_contract_version( + "preference-pair manifest", + values.get("version"), + ContractVersions.PREFERENCE_PAIR_MANIFEST, + ) + return values + @model_validator(mode="after") def unique_ids(self): ids = [pair.pair_id for pair in self.pairs] diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index 38f159bb..8235ef28 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -1,4 +1,4 @@ -"""Inspectable version-1 forecast and policy collection records.""" +"""Inspectable forecast and policy collection records.""" import json import math @@ -16,6 +16,7 @@ ) from sequifier.config.rollout_config import InputBinding +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.io.values import DomainValue from sequifier.trajectory.returns import remaining_return @@ -64,13 +65,22 @@ class Transition(Record): bootstrap_allowed: bool +_TRAJECTORY_VERSIONS = { + "sequifier.numeric-forecast-trajectory": ContractVersions.FORECAST_TRAJECTORY, + "sequifier.scalar-policy-trajectory": ContractVersions.SCALAR_POLICY_TRAJECTORY, + "sequifier.policy-trajectory": ContractVersions.JOINT_POLICY_TRAJECTORY, +} + + class Trajectory(Record): format: Literal[ "sequifier.numeric-forecast-trajectory", "sequifier.scalar-policy-trajectory", "sequifier.policy-trajectory", ] = "sequifier.numeric-forecast-trajectory" - version: Literal[1, 2] = 1 + version: int = Field( + json_schema_extra={"enum": sorted(_TRAJECTORY_VERSIONS.values())} + ) episode_id: str = Field(min_length=1) seed: SeedHistory bindings: list[InputBinding] @@ -80,6 +90,17 @@ class Trajectory(Record): total_return: FiniteFloat metadata: dict[str, Any] = Field(default_factory=dict) + @model_validator(mode="before") + @classmethod + def validate_version(cls, values): + if isinstance(values, dict): + kind = values.get("format", "sequifier.numeric-forecast-trajectory") + if isinstance(kind, str) and kind in _TRAJECTORY_VERSIONS: + require_contract_version( + kind, values.get("version"), _TRAJECTORY_VERSIONS[kind] + ) + return values + @model_validator(mode="after") def validate_temporal_contract(self): # Metadata must round-trip through the same portable JSON representation. @@ -167,8 +188,6 @@ def validate_temporal_contract(self): ): raise ValueError("total_return does not equal the sum of rewards") if self.format == "sequifier.policy-trajectory": - if self.version != 2: - raise ValueError("joint policy trajectories require version 2") _validate_joint_execution(self) return self diff --git a/src/sequifier/trajectory/sources.py b/src/sequifier/trajectory/sources.py index 91dbfb82..92fcc83d 100644 --- a/src/sequifier/trajectory/sources.py +++ b/src/sequifier/trajectory/sources.py @@ -10,6 +10,7 @@ SequenceSourceConfig, resolve_sequence_source, ) +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.trajectory.store import load_trajectory @@ -42,9 +43,18 @@ def _source_files(source, checksums): ) checksums[manifest_path] = checksum manifest = json.loads(content.decode("utf-8")) - if not isinstance(manifest, dict) or manifest.get("version") != 1: - raise ValueError("source manifest must have version 1") + if not isinstance(manifest, dict): + raise ValueError("Source manifest must be a mapping") sequence_manifest = manifest.get("format") == "sequifier.sequence-data" + require_contract_version( + "sequence data manifest" + if sequence_manifest + else "trajectory source manifest", + manifest.get("version"), + ContractVersions.SEQUENCE_DATA_MANIFEST + if sequence_manifest + else ContractVersions.TRAJECTORY_SOURCE_MANIFEST, + ) entries = manifest.get("shards" if sequence_manifest else "episodes") if not isinstance(entries, list) or not entries: raise ValueError("source manifest requires a nonempty episodes list") diff --git a/src/sequifier/visualize_training.py b/src/sequifier/visualize_training.py index 641ce48a..ec6ec141 100644 --- a/src/sequifier/visualize_training.py +++ b/src/sequifier/visualize_training.py @@ -14,7 +14,11 @@ from sequifier.helpers import configure_logger from sequifier.logging_paths import dataset_artifact_prefix -from sequifier.training.metrics import TOTAL_TARGET +from sequifier.training.metrics import ( + TOTAL_TARGET, + validate_metrics_csv_header, + validate_metrics_csv_row, +) from sequifier.typechecking import beartype @@ -156,7 +160,12 @@ def parse_files(self, training_file: str, validation_file: str) -> TrainingMetri @beartype def _read_rows(path: str) -> list[dict[str, str]]: with open(path, "r", encoding="utf-8", newline="") as file: - return list(csv.DictReader(file)) + reader = csv.DictReader(file) + validate_metrics_csv_header(reader.fieldnames, path) + rows = list(reader) + for row in rows: + validate_metrics_csv_row(row, path) + return rows @staticmethod @beartype diff --git a/tools/check_contract_versions.py b/tools/check_contract_versions.py new file mode 100644 index 00000000..8e829704 --- /dev/null +++ b/tools/check_contract_versions.py @@ -0,0 +1,291 @@ +"""Check global contract allocations against the target commit of a merge.""" + +import argparse +import ast +import json +import re +import subprocess +from pathlib import Path + +REGISTRY = "src/sequifier/contract_versions.py" +NON_CONTRACTS = {"HISTORICAL_MIN", "HISTORICAL_MAX", "ONNX_OPSET", "RETIRED"} + +# Put explicit old-name to new-name mappings here when preserving a contract ID. +RENAMES: dict[str, str] = {} + + +def registry(source): + definition = next( + node + for node in ast.parse(source).body + if isinstance(node, ast.ClassDef) and node.name == "ContractVersions" + ) + values = {} + for node in definition.body: + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant): + continue + if ( + not isinstance(node, ast.Assign) + or len(node.targets) != 1 + or not isinstance(node.targets[0], ast.Name) + ): + raise ValueError("ContractVersions requires plain class assignments") + name = node.targets[0].id + if name in values: + raise ValueError(f"Duplicate ContractVersions attribute {name}") + values[name] = ast.literal_eval(node.value) + return values + + +def validate_allocations(current, base, renames=RENAMES): + if current.get("HISTORICAL_MIN") != 0 or current.get("HISTORICAL_MAX") != 99: + raise ValueError("The historical contract reservation must remain 0–99") + active = { + name: value for name, value in current.items() if name not in NON_CONTRACTS + } + retired = current["RETIRED"] + if not isinstance(retired, tuple) or any( + not isinstance(value, int) or isinstance(value, bool) for value in retired + ): + raise ValueError("RETIRED must be a tuple of integer IDs") + if not active or any( + not isinstance(value, int) + or isinstance(value, bool) + or value <= current["HISTORICAL_MAX"] + for value in (*active.values(), *retired) + ): + raise ValueError( + "Active and retired IDs must be integers above the historical range" + ) + if len(set(active.values())) != len(active): + raise ValueError("Duplicate active contract IDs") + if len(set(retired)) != len(retired) or set(retired) & set(active.values()): + raise ValueError("Duplicate or reused retired contract IDs") + + old_active = { + name: value for name, value in base.items() if name not in NON_CONTRACTS + } + old_retired = set(base.get("RETIRED", ())) + if not old_retired <= set(retired): + raise ValueError("Retirement records may only be added") + high_watermark = max(current["HISTORICAL_MAX"], *old_active.values(), *old_retired) + for value in set(retired) - old_retired - set(old_active.values()): + if value <= high_watermark: + raise ValueError( + f"Retired ID {value} was never allocated on the target branch" + ) + preserved: set[str] = set() + for name, old_id in old_active.items(): + replacement = renames.get(name) + if active.get(name) == old_id: + preserved.add(name) + elif ( + replacement is not None + and name not in active + and replacement not in old_active + and active.get(replacement) == old_id + ): + preserved.add(replacement) + elif old_id not in retired: + raise ValueError(f"{name} must retire ID {old_id}") + for name, value in active.items(): + if name not in preserved and value <= high_watermark: + raise ValueError( + f"{name} allocates ID {value} at or below target high-water mark {high_watermark}" + ) + + +VERSION_FIELDS = { + "version", + "format_version", + "schema_version", + "manifest_version", + "stored_window_layout_version", + "tensor_payload_version", + "protocol_version", + "initialization_policy_version", + "initialization_seed_derivation_version", + "opset_version", +} + + +POSITIONAL_VERSION_FIELDS = { + "StoredWindowLayout": (2, "version"), + "ModelArtifact": (0, "format_version"), +} + + +def assignment_name(target): + if isinstance(target, ast.Name): + return target.id + if isinstance(target, ast.Attribute): + return target.attr + if isinstance(target, ast.Subscript) and isinstance(target.slice, ast.Constant): + return target.slice.value + return None + + +def has_integer_literal(value): + if isinstance(value, ast.IfExp): + return has_integer_literal(value.body) or has_integer_literal(value.orelse) + return isinstance(value, ast.Constant) and ( + isinstance(value.value, int) + and not isinstance(value.value, bool) + or isinstance(value.value, str) + and value.value.isdecimal() + ) + + +def hardcoded_versions(tree): + """Find literal versions at Python serialization and declaration sites.""" + for node in ast.walk(tree): + assignments = () + if isinstance(node, ast.Dict): + assignments = ( + (getattr(key, "value", None), value) + for key, value in zip(node.keys, node.values) + ) + elif isinstance(node, ast.Assign): + assignments = ( + (assignment_name(target), node.value) for target in node.targets + ) + elif isinstance(node, ast.AnnAssign): + name = assignment_name(node.target) + assignments = ((name, node.value),) + if name in VERSION_FIELDS and isinstance(node.value, ast.Call): + assignments += tuple( + (name, argument.value) + for argument in node.value.keywords + if argument.arg == "default" + ) + if ( + isinstance(name, str) + and name in VERSION_FIELDS + and isinstance(node.annotation, ast.Subscript) + and isinstance(node.annotation.value, ast.Name) + and node.annotation.value.id == "Literal" + ): + annotation = node.annotation.slice + candidates = ( + annotation.elts + if isinstance(annotation, ast.Tuple) + else [annotation] + ) + assignments += tuple((name, candidate) for candidate in candidates) + elif ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in POSITIONAL_VERSION_FIELDS + ): + index, name = POSITIONAL_VERSION_FIELDS[node.func.id] + if index < len(node.args): + assignments = ((name, node.args[index]),) + elif isinstance(node, ast.keyword): + assignments = ((node.arg, node.value),) + for name, value in assignments: + if ( + isinstance(node, (ast.stmt, ast.expr, ast.keyword)) + and ( + name in VERSION_FIELDS + or isinstance(name, str) + and name.endswith("_VERSION") + ) + and has_integer_literal(value) + ): + yield node.lineno + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and re.search(r"^sequifier[.-][\w.-]+[.-]v[0-9]+(?:$|:)", node.value) + ): + yield node.lineno + if isinstance(node, ast.FunctionDef) and node.name == "protocol_version": + for statement in ast.walk(node): + if ( + isinstance(statement, ast.Return) + and isinstance(statement.value, ast.Constant) + and isinstance(statement.value.value, int) + and not isinstance(statement.value.value, bool) + ): + yield statement.lineno + + +def validate_boundaries(root, values): + errors = [] + for directory in ("src/sequifier", "documentation/example_inputs", "tools"): + for path in (root / directory).rglob("*.py"): + if path.relative_to(root).as_posix() in { + REGISTRY, + "tools/check_contract_versions.py", + }: + continue + tree = ast.parse(path.read_text(), filename=str(path)) + errors.extend( + f"{path.relative_to(root)}:{line}: reference ContractVersions" + for line in hardcoded_versions(tree) + ) + + identities = { + "sequifier.categorical-control": values["CATEGORICAL_CONTROL"], + "sequifier.toy-control": values["TOY_CONTROL"], + } + for directory in ("src/sequifier/templates", "documentation/example_inputs"): + for path in (root / directory).rglob("*"): + if not path.is_file() or path.suffix not in (".yaml", ".yml", ".json"): + continue + content = path.read_text() + for prefix, version in identities.items(): + expected = f"{prefix}.v{version}" + for match in re.finditer(rf"{re.escape(prefix)}\.v[\w.-]+", content): + if match[0] != expected: + errors.append( + f"{path.relative_to(root)}: found {match[0]}, expected {expected}" + ) + metadata_path = ( + "documentation/example_inputs/configs/ddconfigs/preprocessing_input.json" + ) + metadata = json.loads((root / metadata_path).read_text()) + for field, name in ( + ("tensor_payload_version", "DATASET_TENSOR_METADATA"), + ("stored_window_layout_version", "STORED_WINDOW_LAYOUT"), + ): + actual = metadata.get(field) + if ( + not isinstance(actual, int) + or isinstance(actual, bool) + or actual != values[name] + ): + errors.append( + f"{metadata_path}: {field} must match ContractVersions.{name}" + ) + if errors: + raise ValueError("\n".join(errors)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True, help="Target commit included in HEAD") + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + subprocess.run( + ["git", "merge-base", "--is-ancestor", args.base, "HEAD"], cwd=root, check=True + ) + base_paths = subprocess.check_output( + ["git", "ls-tree", "--name-only", args.base, REGISTRY], text=True, cwd=root + ) + base = ( + registry( + subprocess.check_output( + ["git", "show", f"{args.base}:{REGISTRY}"], text=True, cwd=root + ) + ) + if base_paths + else {} + ) + current = registry((root / REGISTRY).read_text()) + validate_allocations(current, base) + validate_boundaries(root, current) + + +if __name__ == "__main__": + main() diff --git a/tools/convert_to_pt.py b/tools/convert_to_pt.py index bb42f2ee..9b27a98b 100644 --- a/tools/convert_to_pt.py +++ b/tools/convert_to_pt.py @@ -1,102 +1,69 @@ -import os -import sys +"""Convert current tabular windows to Sequifier tensor batches.""" + +import argparse +from pathlib import Path -import numpy as np import polars as pl import torch +from sequifier.io.pt_payload import save_pt_payload, tabular_tensor_batch +from sequifier.io.sample_order import ( + curriculum_storage_columns, + source_curriculum_column, +) -def main(): - assert ( - len(sys.argv) >= 2 - ), "Usage: python convert_to_pt.py [output_path] [chunk_size]" - in_path = sys.argv[1] - out_path = ( - sys.argv[2] if len(sys.argv) > 2 else os.path.splitext(in_path)[0] + ".pt" - ) - chunk_size = int(sys.argv[3]) if len(sys.argv) > 3 else None - - assert os.path.exists(in_path), f"Input file not found: {in_path}" - if in_path.endswith(".csv"): - df = pl.read_csv(in_path) - elif in_path.endswith(".parquet"): - df = pl.read_parquet(in_path) +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input_file", type=Path) + parser.add_argument("output_path", type=Path, nargs="?") + parser.add_argument("chunk_size", type=int, nargs="?") + args = parser.parse_args() + source = args.input_file + output = args.output_path or source.with_suffix(".pt") + if args.chunk_size is not None and args.chunk_size <= 0: + parser.error("chunk_size must be positive") + if source.suffix == ".csv": + data = pl.read_csv(source) + elif source.suffix == ".parquet": + data = pl.read_parquet(source) else: - raise ValueError("Input must be .csv or .parquet") - - required_meta = {"sequenceId", "subsequenceId", "startItemPosition", "inputCol"} - assert required_meta.issubset( - df.columns - ), f"Missing columns: {required_meta - set(df.columns)}" - - seq_cols = sorted([c for c in df.columns if c.isdigit()], key=int) - assert len(seq_cols) > 1, "No numbered sequence columns found" - - feature_names = df["inputCol"].unique().to_list() - aggs = [ - pl.concat_list(seq_cols).filter(pl.col("inputCol") == f).flatten().alias(f) - for f in feature_names - ] - aggs.append(pl.col("startItemPosition").first()) - - grouped = df.group_by(["sequenceId", "subsequenceId"], maintain_order=True).agg( - aggs + parser.error("input must be CSV or Parquet") + + sequence_columns = [name for name in data.columns if name.isdigit()] + if not sequence_columns: + parser.error("input must contain numbered sequence columns") + features = data.get_column("inputCol").unique().sort().to_list() + dtype = ( + torch.float32 + if any(data.schema[name].is_float() for name in sequence_columns) + else torch.int64 ) - grouped = grouped.sort(["sequenceId", "subsequenceId"]) - - seq_ids = torch.from_numpy(grouped["sequenceId"].to_numpy().astype(np.int64)) - sub_ids = torch.from_numpy(grouped["subsequenceId"].to_numpy().astype(np.int64)) - start_pos = torch.from_numpy( - grouped["startItemPosition"].to_numpy().astype(np.int64) + batch = tabular_tensor_batch( + data, dict.fromkeys(features, dtype), features, len(sequence_columns) ) + curriculum = curriculum_storage_columns(data.columns) + if curriculum: + positions = ( + data.group_by("sequenceId", "subsequenceId") + .agg(pl.col(name).first() for name in curriculum) + .sort("sequenceId", "subsequenceId") + ) + batch.sample_positions = torch.tensor(positions.to_numpy(), dtype=torch.int64) + batch.curriculum_columns = tuple( + source_curriculum_column(name) for name in curriculum + ) - sequences_dict = {} - targets_dict = {} - - for f in feature_names: - is_float = df.schema[seq_cols[0]] in (pl.Float32, pl.Float64) - dtype = torch.float32 if is_float else torch.int64 - - data_np = np.stack(grouped[f].to_list()) - tensor = torch.tensor(data_np, dtype=dtype) - - sequences_dict[f] = tensor[:, :-1] - targets_dict[f] = tensor[:, 1:] - - total_rows = len(seq_ids) - - if chunk_size is not None and total_rows > chunk_size: - assert not os.path.isfile( - out_path - ), f"Output path '{out_path}' exists as a file, but chunking requires a directory." - os.makedirs(out_path, exist_ok=True) - - print(f"Converting {in_path} -> {out_path}/ (chunked by {chunk_size})") - dirname = os.path.split(out_path)[-1] - - for i in range(0, total_rows, chunk_size): - end = i + chunk_size - - s_ids_chunk = seq_ids[i:end] - sub_ids_chunk = sub_ids[i:end] - start_pos_chunk = start_pos[i:end] - seq_chunk = {k: v[i:end] for k, v in sequences_dict.items()} - tgt_chunk = {k: v[i:end] for k, v in targets_dict.items()} - - file_name = f"{dirname}-{i // chunk_size}.pt" - full_name = os.path.join(out_path, file_name) - - torch.save( - (seq_chunk, tgt_chunk, s_ids_chunk, sub_ids_chunk, start_pos_chunk), - full_name, - ) + total = len(batch.sequence_ids) + if args.chunk_size is None or total <= args.chunk_size: + save_pt_payload(batch, output) else: - print(f"Converting {in_path} -> {out_path}") - print(f"Samples: {total_rows} | Features: {feature_names}") - torch.save( - (sequences_dict, targets_dict, seq_ids, sub_ids, start_pos), out_path - ) + output.mkdir(parents=True, exist_ok=True) + for index, start in enumerate(range(0, total, args.chunk_size)): + save_pt_payload( + batch.slice_rows(start, start + args.chunk_size), + output / f"{output.name}-{index}.pt", + ) if __name__ == "__main__": diff --git a/tools/resize_pt_files.py b/tools/resize_pt_files.py index ae7ff89e..b16f9df6 100644 --- a/tools/resize_pt_files.py +++ b/tools/resize_pt_files.py @@ -1,225 +1,103 @@ +"""Rechunk current Sequifier tensor batches, retaining split metadata.""" + import argparse import json import os import sys -from typing import Any, Dict, List, Tuple +from collections import Counter +from pathlib import Path import torch - -def unpack_dataset_tuple(data_tuple: Tuple) -> Dict[str, Any]: - """Unpack the standard Sequifier PT tuple.""" - return { - "sequences": data_tuple[0], - "targets": data_tuple[1], - "seq_ids": data_tuple[2], - "sub_ids": data_tuple[3], - "start_pos": data_tuple[4], - } - - -def pack_dataset_tuple(data_dict: Dict[str, Any]) -> Tuple: - """Pack a dataset dict into the standard Sequifier PT tuple.""" - return ( - data_dict["sequences"], - data_dict["targets"], - data_dict["seq_ids"], - data_dict["sub_ids"], - data_dict["start_pos"], - ) - - -def concat_dataset_list(data_list: List[Dict[str, Any]]) -> Dict[str, Any]: - """Concatenate dataset dictionaries along dim 0.""" - if not data_list: - return {} - - if len(data_list) == 1: - return data_list[0] - - combined = {} - first = data_list[0] - - combined["sequences"] = { - k: torch.cat([d["sequences"][k] for d in data_list], dim=0) - for k in first["sequences"] - } - - combined["targets"] = { - k: torch.cat([d["targets"][k] for d in data_list], dim=0) - for k in first["targets"] - } - - combined["seq_ids"] = torch.cat([d["seq_ids"] for d in data_list], dim=0) - combined["sub_ids"] = torch.cat([d["sub_ids"] for d in data_list], dim=0) - combined["start_pos"] = torch.cat([d["start_pos"] for d in data_list], dim=0) - - return combined - - -def slice_dataset(data: Dict[str, Any], start: int, end: int) -> Dict[str, Any]: - """Slice a dataset dictionary without cloning.""" - sliced = {} - sliced["sequences"] = {k: v[start:end] for k, v in data["sequences"].items()} - sliced["targets"] = {k: v[start:end] for k, v in data["targets"].items()} - sliced["seq_ids"] = data["seq_ids"][start:end] - sliced["sub_ids"] = data["sub_ids"][start:end] - sliced["start_pos"] = data["start_pos"][start:end] - return sliced - - -def clone_dataset(data: Dict[str, Any]) -> Dict[str, Any]: - """Clone all tensors in a dataset dictionary.""" - cloned = {} - cloned["sequences"] = {k: v.clone() for k, v in data["sequences"].items()} - cloned["targets"] = {k: v.clone() for k, v in data["targets"].items()} - cloned["seq_ids"] = data["seq_ids"].clone() - cloned["sub_ids"] = data["sub_ids"].clone() - cloned["start_pos"] = data["start_pos"].clone() - return cloned - - -def get_row_size_bytes(data: Dict[str, Any]) -> float: - """Return bytes per row across sequence/target tensors.""" - total_bytes = 0 - for t in data["sequences"].values(): - total_bytes += t.element_size() * t.shape[1] - for t in data["targets"].values(): - total_bytes += t.element_size() * t.shape[1] - - # Add metadata sizes (1 element each) - total_bytes += data["seq_ids"].element_size() - total_bytes += data["sub_ids"].element_size() - total_bytes += data["start_pos"].element_size() - - return total_bytes - - -def process_split( - input_dir: str, - output_dir: str, - dataset_name: str, - target_size_mb: float, - split_suffix: str, -): - os.makedirs(output_dir, exist_ok=True) - - meta_path = os.path.join(input_dir, "metadata.json") - if os.path.exists(meta_path): - with open(meta_path, "r") as f: - old_metadata = json.load(f) - input_files = [entry["path"] for entry in old_metadata.get("batch_files", [])] - expected_total_samples = old_metadata.get("total_samples", 0) - else: - input_files = sorted([f for f in os.listdir(input_dir) if f.endswith(".pt")]) - expected_total_samples = None - print( - f"Warning: No metadata.json found in {input_dir}. Using alphabetical sort." - ) - - data_buffer: List[Dict[str, Any]] = [] - buffer_row_count = 0 - - output_batch_idx = 0 - new_batch_files_metadata = [] - total_samples_processed = 0 - - target_bytes = target_size_mb * 1024 * 1024 +from sequifier.config.metadata import load_dataset_metadata +from sequifier.io.atomic import write_json_atomic +from sequifier.io.pt_payload import ( + concatenate_pt_batches, + load_pt_payload, + save_pt_payload, +) + + +def process_split(input_dir, output_dir, dataset_name, target_size_mb, split_suffix): + source = Path(input_dir) + destination = Path(output_dir) + metadata = load_dataset_metadata(str(source / "metadata.json")) + manifest = json.loads((source / "metadata.json").read_text()) + options = {"layouts": metadata.depth_layouts, "n_classes": metadata.n_classes} + destination.mkdir(parents=True, exist_ok=True) + entries = [] + batches = [] + buffered_rows = 0 target_rows = None - print(f"Processing {input_dir} -> {output_dir}") - - for file_name in input_files: - file_path = os.path.join(input_dir, file_name) - if not os.path.exists(file_path): - continue - - try: - loaded_tuple = torch.load(file_path, map_location="cpu", weights_only=False) - current_data = unpack_dataset_tuple(loaded_tuple) - - current_rows = current_data["seq_ids"].shape[0] - if current_rows == 0: - continue - - if target_rows is None: - bytes_per_row = get_row_size_bytes(current_data) - target_rows = max(1, int(target_bytes / bytes_per_row)) - - data_buffer.append(current_data) - buffer_row_count += current_rows - - if buffer_row_count >= target_rows: - full_data = concat_dataset_list(data_buffer) - - data_buffer = [] - buffer_row_count = 0 - - num_rows = full_data["seq_ids"].shape[0] - - start_idx = 0 - while start_idx + target_rows <= num_rows: - end_idx = start_idx + target_rows - - chunk_data = slice_dataset(full_data, start_idx, end_idx) - - fname = f"{dataset_name}-{split_suffix}-{output_batch_idx}.pt" - out_path = os.path.join(output_dir, fname) - torch.save(pack_dataset_tuple(chunk_data), out_path) - - chunk_len = end_idx - start_idx - new_batch_files_metadata.append( - {"path": fname, "samples": chunk_len} - ) - total_samples_processed += chunk_len - output_batch_idx += 1 - - start_idx = end_idx - - if start_idx < num_rows: - remainder_data = clone_dataset( - slice_dataset(full_data, start_idx, num_rows) - ) - data_buffer = [remainder_data] - buffer_row_count = remainder_data["seq_ids"].shape[0] - - del full_data - - except Exception as e: - print(f"Error processing {file_path}: {e}") - sys.exit(1) - - if buffer_row_count > 0: - full_data = concat_dataset_list(data_buffer) - fname = f"{dataset_name}-{split_suffix}-{output_batch_idx}.pt" - out_path = os.path.join(output_dir, fname) - torch.save(pack_dataset_tuple(full_data), out_path) - - chunk_len = full_data["seq_ids"].shape[0] - new_batch_files_metadata.append({"path": fname, "samples": chunk_len}) - total_samples_processed += chunk_len - - new_metadata = { - "total_samples": total_samples_processed, - "batch_files": new_batch_files_metadata, - } - with open(os.path.join(output_dir, "metadata.json"), "w") as f: - json.dump(new_metadata, f, indent=4) - - if ( - expected_total_samples is not None - and total_samples_processed != expected_total_samples - ): - print( - f"WARNING: Sample count mismatch! Input: {expected_total_samples}, Output: {total_samples_processed}" + def write_shard(batch): + filename = f"{dataset_name}-{split_suffix}-{len(entries)}.pt" + save_pt_payload(batch, destination / filename, **options) + first_target = torch.maximum( + batch.left_pad_lengths, + batch.split_start_item_positions - batch.start_item_positions, ) - else: - print( - f"Success: {split_suffix} processed. Total samples: {total_samples_processed}" + entries.append( + { + "path": filename, + "samples": len(batch.sequence_ids), + "left_pad_length_histogram": dict( + Counter(batch.left_pad_lengths.tolist()) + ), + "target_valid_from_histogram": dict(Counter(first_target.tolist())), + } ) + for entry in manifest["batch_files"]: + batch = load_pt_payload(source / entry["path"], **options) + count = len(batch.sequence_ids) + if count != entry["samples"]: + raise ValueError( + f"{entry['path']}: tensor batch size differs from metadata" + ) + if count == 0: + continue + if target_rows is None: + size = sum( + tensor.numel() * tensor.element_size() + for tensor in ( + *batch.sequences.values(), + *batch.depth_valid_masks.values(), + batch.sequence_ids, + batch.subsequence_ids, + batch.start_item_positions, + batch.left_pad_lengths, + batch.split_start_item_positions, + batch.split_end_item_positions, + batch.sample_positions, + ) + if tensor is not None + ) + target_rows = max(1, int(target_size_mb * 1024 * 1024 * count / size)) + batches.append(batch) + buffered_rows += count + if buffered_rows >= target_rows: + combined = concatenate_pt_batches(batches) + complete_rows = buffered_rows // target_rows * target_rows + for start in range(0, complete_rows, target_rows): + write_shard(combined.slice_rows(start, start + target_rows)) + batches = ( + [combined.slice_rows(complete_rows, buffered_rows)] + if complete_rows < buffered_rows + else [] + ) + buffered_rows -= complete_rows + if batches: + write_shard(concatenate_pt_batches(batches)) + count = sum(entry["samples"] for entry in entries) + if count != manifest["total_samples"]: + raise ValueError(f"{source}: total sample count differs from metadata") + write_json_atomic( + destination / "metadata.json", + {**manifest, "total_samples": count, "batch_files": entries}, + ) + -# ... [main function remains unchanged] ... def main(): parser = argparse.ArgumentParser( description="Fast Rechunker for Sequifier Datasets" @@ -229,6 +107,8 @@ def main(): parser.add_argument("target_size_mb", type=float, help="Target file size in MB") args = parser.parse_args() + if args.target_size_mb <= 0: + parser.error("target_size_mb must be positive") if not os.path.exists(args.data_folder): print("Data folder not found.") From 1686c860b7a56feb2a9be7f64b48e74979d5ed4b Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 18:00:51 +0100 Subject: [PATCH 57/71] unify phase transitions --- .../artifacts/backbone_repository.py | 20 +- .../artifacts/checkpoint_envelope.py | 34 +- src/sequifier/artifacts/dependencies.py | 198 ++++++ src/sequifier/artifacts/phase.py | 131 ++-- src/sequifier/artifacts/run_checkpoint.py | 35 +- src/sequifier/config/train_config.py | 17 +- src/sequifier/config/training_resolution.py | 13 +- src/sequifier/contract_versions.py | 11 +- src/sequifier/io/images.py | 6 +- src/sequifier/io/output_paths.py | 31 +- src/sequifier/rl/learner.py | 123 ++-- src/sequifier/rl/preparation.py | 11 +- src/sequifier/rl/prepared_inputs.py | 7 + src/sequifier/rl/run_checkpoint.py | 138 ++-- src/sequifier/rl/run_config.py | 1 + src/sequifier/rl/run_state.py | 8 +- src/sequifier/runtime/builder.py | 58 +- src/sequifier/runtime/plan.py | 78 ++- src/sequifier/train.py | 15 +- src/sequifier/training/checkpoint_service.py | 29 +- src/sequifier/training/coordinator.py | 602 ++++++++++-------- src/sequifier/training/engine.py | 167 +++-- src/sequifier/training/phase_runners.py | 73 +-- src/sequifier/training/plan.py | 4 + src/sequifier/training/state.py | 2 +- 25 files changed, 1165 insertions(+), 647 deletions(-) create mode 100644 src/sequifier/artifacts/dependencies.py diff --git a/src/sequifier/artifacts/backbone_repository.py b/src/sequifier/artifacts/backbone_repository.py index 1a3201ba..7781804f 100644 --- a/src/sequifier/artifacts/backbone_repository.py +++ b/src/sequifier/artifacts/backbone_repository.py @@ -95,21 +95,26 @@ def latest_pointer(repository_path: Path) -> dict[str, Any] | None: @beartype -def select_revision(backbone_config: Any, project_root: str) -> dict[str, Any] | None: +def select_revision( + backbone_config: Any, project_root: str, *, revision_id: str | None = None +) -> dict[str, Any] | None: repository = backbone_config.repository if repository is None: return None repository_path = resolve_repository_path(repository.path, project_root) - pointer = latest_pointer(repository_path) - if pointer is None: + pointer = latest_pointer(repository_path) if revision_id is None else None + if revision_id is None and pointer is None: if repository.load_policy == "required": raise FileNotFoundError( f"Required backbone revision does not exist in {repository_path}." ) return None - revision_id = pointer["revision_id"] + if pointer is not None: + revision_id = pointer["revision_id"] + if not revision_id or Path(revision_id).name != revision_id: + raise ValueError("Invalid backbone revision identity") revision_path = repository_path / "revisions" / revision_id manifest_path = revision_path / "manifest.json" weights_path = revision_path / "weights.pt" @@ -143,11 +148,12 @@ def select_revision(backbone_config: Any, project_root: str) -> dict[str, Any] | manifest_fingerprint, expected_fingerprint, ), - "parent_revision_id": ( + } + if pointer is not None: + checks["parent_revision_id"] = ( manifest.get("parent_revision_id"), pointer.get("parent_revision_id"), - ), - } + ) mismatches = [ f"{name}: found {actual!r}, expected {expected!r}" for name, (actual, expected) in checks.items() diff --git a/src/sequifier/artifacts/checkpoint_envelope.py b/src/sequifier/artifacts/checkpoint_envelope.py index 2f5d58a2..2cd37fe4 100644 --- a/src/sequifier/artifacts/checkpoint_envelope.py +++ b/src/sequifier/artifacts/checkpoint_envelope.py @@ -6,13 +6,31 @@ from sequifier.contract_versions import ContractVersions, require_contract_version -def checkpoint_envelope(recovery_type, recovery, model): +def phase_boundary(run_id, *, selected=None, active=None): + """References made authoritative by the same write as the runner state.""" + return { + "format_version": ContractVersions.PHASE_BOUNDARY, + "run_id": run_id, + "selected": selected, + "active": active, + } + + +def selected_model_identity(completed): + if not completed: + return None + latest = completed[-1] + return {"path": latest["model_path"], "sha256": latest["model_sha256"]} + + +def checkpoint_envelope(recovery_type, recovery, model, *, boundary): return { "artifact_type": "sequifier_run_checkpoint", "format_version": ContractVersions.RUN_CHECKPOINT, "model": model.state_dict() if isinstance(model, ModelArtifact) else model, "recovery_type": recovery_type, "recovery": recovery, + "phase_boundary": boundary, } @@ -30,6 +48,15 @@ def validate_checkpoint_envelope(payload): raise ValueError( f"Unsupported checkpoint recovery kind {payload.get('recovery_type')!r}" ) + boundary = payload.get("phase_boundary") + if boundary is not None: + if not isinstance(boundary, dict): + raise ValueError("Run checkpoint requires a phase boundary mapping") + require_contract_version( + "phase boundary", + boundary.get("format_version"), + ContractVersions.PHASE_BOUNDARY, + ) if not isinstance(payload.get("recovery"), dict): raise ValueError("Run checkpoint requires a recovery mapping") @@ -42,3 +69,8 @@ def recovery_payload(payload, recovery_type): f"Checkpoint recovery kind {payload['recovery_type']!r}; expected {recovery_type!r}" ) return payload["recovery"] + + +def require_phase_boundary(payload, expected): + if payload.get("phase_boundary") != expected: + raise ValueError("Phase boundary disagrees with checkpoint recovery") diff --git a/src/sequifier/artifacts/dependencies.py b/src/sequifier/artifacts/dependencies.py new file mode 100644 index 00000000..baab8181 --- /dev/null +++ b/src/sequifier/artifacts/dependencies.py @@ -0,0 +1,198 @@ +"""Content identities for inputs consumed by training.""" + +import json +from pathlib import Path + +from sequifier.trajectory.sources import file_checksum + + +class DependencyManifest: + def __init__(self): + self.identity: dict[str, str] = {} + self.roots: set[Path] = set() + self._media: dict[Path, tuple[str, ...]] = {} + self._paths: dict[Path, Path] = {} + self._embedded_references: set[str] = set() + + @property + def protected_paths(self): + return self.roots | {Path(path) for path in self.identity} + + def include(self, checksums): + for path, checksum in checksums.items(): + path = Path(path).absolute() + resolved = path.resolve() + self._paths[path] = resolved + previous = self.identity.setdefault(str(resolved), checksum) + if previous != checksum: + raise ValueError( + f"Training dependency changed during preparation: {path}" + ) + + def add(self, path): + resolved = Path(path).resolve() + if str(resolved) not in self.identity: + self.include({path: file_checksum(resolved)}) + else: + self._paths[Path(path).absolute()] = resolved + + def add_reference(self, path, checkpoint_checksum): + resolved = str(path.resolve()) + if checkpoint_checksum is None and resolved in self._embedded_references: + # A later phase can still need this same file to create its reference. + self.include({path: file_checksum(path)}) + self._embedded_references.remove(resolved) + return + try: + self.add(path) + except FileNotFoundError: + if checkpoint_checksum is None: + raise + self.include({path: checkpoint_checksum}) + self._embedded_references.add(resolved) + + def add_media_directory(self, directory, files): + self.roots.add(directory.resolve()) + self._media[directory] = tuple(file.name for file in files) + for path in files: + self.add(path) + + def merge(self, other): + self.include(other.identity) + self.roots.update(other.roots) + self._media.update(other._media) + self._paths.update(other._paths) + self._embedded_references.update(other._embedded_references) + + def validate(self, expected): + if self.identity != expected: + changed = sorted( + path + for path in self.identity.keys() | expected.keys() + if self.identity.get(path) != expected.get(path) + ) + raise ValueError( + f"Training dependencies differ from checkpoint: {changed[:10]}" + ) + + def revalidate(self): + from sequifier.io.images import media_files + + for directory, files in self._media.items(): + current = media_files(str(directory), str(directory.parent)) + if tuple(file.name for file in current) != files: + raise ValueError(f"Training media directory changed: {directory}") + for path, resolved in self._paths.items(): + if path.resolve() != resolved: + raise ValueError(f"Training dependency moved: {path}") + for path, checksum in self.identity.items(): + try: + current = file_checksum(path) + except FileNotFoundError: + if path in self._embedded_references: + continue + raise + if current != checksum: + raise ValueError(f"Training dependency changed: {path}") + + +def add_initialization(manifest, config, incoming=None): + if path := incoming or config.initialize_from: + manifest.add(Path(config.project_root) / path) + + +def model_metadata_paths(config, interface_name=None): + interfaces = (config.ownership.get("model") or {}).get("interfaces", {}) + return [ + str(Path(config.project_root) / path) + for name, interface in interfaces.items() + if (interface_name is None or name == interface_name) + and (path := interface.get("input_contract", {}).get("metadata_config_path")) + ] + + +def _add_loader_media(manifest, view): + from sequifier.io.images import media_files + + root = Path(view.project_root) + if view.static_columns and view.static_features_path: + manifest.add(root / view.static_features_path) + media = [] + for source in view.image_sources.values(): + if source.path: + media.append(source.path) + if source.time_index_path: + manifest.add(root / source.time_index_path) + for path in source.video_index_paths.values(): + manifest.add(root / path) + if view.preconditioning_image: + media.extend(view.preconditioning_image.views.values()) + for path in media: + manifest.add_media_directory(root / path, media_files(path, view.project_root)) + + +def supervised_dependencies(config, incoming=None): + from sequifier.config.train_config import dataset_part_view + + manifest = DependencyManifest() + root = Path(config.project_root) + add_initialization(manifest, config, incoming) + for path in model_metadata_paths(config): + manifest.add(path) + for name, dataset in config.dataset_training.items(): + for part_name, part in dataset.parts.items(): + manifest.add(root / part.metadata_config_path) + for split in (part.training_data_path, part.validation_data_path): + if split is None: + continue + path = root / split + if part.storage_form == "file": + manifest.add(path) + else: + manifest.roots.add(path.resolve()) + metadata = path / "metadata.json" + manifest.add(metadata) + for shard in json.loads(metadata.read_text())["batch_files"]: + manifest.add(path / shard["path"]) + _add_loader_media( + manifest, + dataset_part_view( + config, name, part_name, batch_size=1, window_stride=None + ), + ) + return manifest + + +def policy_dependencies( + config, prepared, inputs, *, saved=None, committed_reference_count=0 +): + manifest = DependencyManifest() + add_initialization(manifest, config) + for path in config.metadata_paths: + manifest.add(path) + for index, phase in enumerate(config.phases): + manifest.include(prepared.phase_checksums(config, phase, inputs)) + reference = getattr(phase, "reference", None) + if reference and reference.path: + path = Path(config.project_root) / reference.path + checksum = ( + saved.get(str(path.resolve())) + if saved is not None and index < committed_reference_count + else None + ) + manifest.add_reference(path, checksum) + return manifest + + +def add_backbone_revision(manifest, config, revision_id): + from sequifier.artifacts.backbone_repository import resolve_repository_path + + repository = config.model.backbone.repository + if repository is not None and revision_id is not None: + directory = ( + resolve_repository_path(repository.path, config.project_root) + / "revisions" + / revision_id + ) + manifest.add(directory / "manifest.json") + manifest.add(directory / "weights.pt") diff --git a/src/sequifier/artifacts/phase.py b/src/sequifier/artifacts/phase.py index ce084777..6bd4853c 100644 --- a/src/sequifier/artifacts/phase.py +++ b/src/sequifier/artifacts/phase.py @@ -1,12 +1,13 @@ -"""Paths, identity and publication records at committed phase boundaries.""" +"""Selected models and sidecar identities at committed phase boundaries.""" -import json -from dataclasses import dataclass +from dataclasses import asdict, dataclass, replace from pathlib import Path +from shutil import copyfile +from uuid import uuid4 from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.fingerprints import fingerprint -from sequifier.io.atomic import write_json_atomic +from sequifier.io.atomic import atomic_path, write_json_atomic from sequifier.trajectory.sources import file_checksum @@ -22,10 +23,6 @@ def for_phase(cls, directory, index, name, filename="model.pt"): def report(self): return self.model.with_name("report.json") - @property - def receipt(self): - return self.model.with_name("completed.json") - @property def telemetry(self): return self.model.parent / "telemetry" / "records.sqlite3" @@ -33,47 +30,93 @@ def telemetry(self): def describe(self): return {"actor_path": str(self.model), "report_path": str(self.report)} - def publish_model(self, artifact): + +@dataclass(frozen=True) +class PhaseCommit: + """An immutable model and report payload published by a run checkpoint.""" + + model_path: str + model_sha256: str + result: dict + result_fingerprint: str + sidecars: dict + format_version: int = ContractVersions.PHASE_COMMIT + + @classmethod + def stage(cls, artifacts, result, model, *, telemetry=None): from sequifier.artifacts.model_artifact import save_model_artifact - save_model_artifact(artifact, self.model) - - def publish_report(self, report): - write_json_atomic(self.report, report) - - def record(self, report): - return { - "sha256": file_checksum(self.model), - "result_fingerprint": fingerprint(report), - } - - def read_receipt(self, identity, *, run_id=None): - if not self.receipt.is_file(): - return None - receipt = json.loads(self.receipt.read_text()) - if run_id is not None and receipt.get("run_id") != run_id: - return None - if receipt["identity"] != identity or receipt["sha256"] != file_checksum( - self.model - ): - raise ValueError("completed phase artifact or receipt changed") + path = artifacts.model.parent / "commits" / f"{uuid4().hex}.pt" + save_model_artifact(model, path) + sidecars = {"telemetry": telemetry} if telemetry is not None else {} + return cls( + str(path), + file_checksum(path), + result, + fingerprint({"result": result, "sidecars": sidecars}), + sidecars, + ) + + def with_result(self, result): + return replace( + self, + result=result, + result_fingerprint=fingerprint( + {"result": result, "sidecars": self.sidecars} + ), + ) + + def as_dict(self): + return asdict(self) + + @classmethod + def load(cls, values): + commit = cls(**values) require_contract_version( - "phase report", - receipt["report"].get("format_version"), - ContractVersions.PHASE_REPORT, + "phase commit", commit.format_version, ContractVersions.PHASE_COMMIT ) - return receipt - - def write_receipt(self, identity, report, *, run_id=None): - write_json_atomic( - self.receipt, - { - "identity": identity, - "run_id": run_id, - "sha256": file_checksum(self.model), - "report": report, - }, + if fingerprint({"result": commit.result, "sidecars": commit.sidecars}) != ( + commit.result_fingerprint + ): + raise ValueError("Committed phase result changed") + verify_artifact( + commit.model_path, commit.model_sha256, "Committed phase model changed" + ) + return commit + + @property + def model_identity(self): + return {"path": self.model_path, "sha256": self.model_sha256} + + def report_result(self): + from sequifier.rl.telemetry import TelemetryJournal + + result = dict(self.result) + if result["algorithm"] == "supervised": + return result + sidecar = self.sidecars.get("telemetry") + history = ( + TelemetryJournal.read(Path(sidecar["path"]), sidecar["position"]) + if sidecar + else {} ) + result["metrics"] = history.get("metrics", []) + if result["algorithm"] in {"ppo", "grpo"}: + result["provenance"] = { + **result["provenance"], + **{ + key: history.get(key, []) + for key in ("evaluations", "continuations") + }, + } + return result + + def publish(self, artifacts, report): + path = artifacts.model + if not path.is_file() or file_checksum(path) != self.model_sha256: + with atomic_path(path, durable=True) as temporary: + copyfile(self.model_path, temporary) + write_json_atomic(artifacts.report, report) def verify_artifact(path, checksum, message): diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 2571ba0a..5d70b977 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -48,6 +48,7 @@ class RunCheckpoint: loader_state: LoaderState integration_state: IntegrationState training_config: Any + dependencies: dict[str, str] def validate(self) -> None: require_contract_version( @@ -57,10 +58,27 @@ def validate(self) -> None: ) self.model.validate() + def phase_boundary(self): + from sequifier.artifacts.checkpoint_envelope import ( + phase_boundary, + selected_model_identity, + ) + + state = self.run_state + completed = state["phase_commits"] + active = ( + {"index": state["phase_index"], "transaction": state["optimizer_step"]} + if len(completed) <= state["phase_index"] + else None + ) + return phase_boundary( + state["run_id"], selected=selected_model_identity(completed), active=active + ) + def state_dict(self) -> dict[str, Any]: - self.validate() from sequifier.artifacts.checkpoint_envelope import checkpoint_envelope + self.validate() recovery = { "artifact_type": "sequifier_run_checkpoint", "recovery_type": "supervised", @@ -73,15 +91,22 @@ def state_dict(self) -> dict[str, Any]: "integration_state": {"values": self.integration_state.values}, "initialization_policy_version": ContractVersions.INITIALIZATION_POLICY, "initialization_seed_derivation_version": ContractVersions.INITIALIZATION_SEED_DERIVATION, + "dependencies": self.dependencies, "training_config": self.training_config.model_dump(mode="python"), } - return checkpoint_envelope("supervised", recovery, self.model) + return checkpoint_envelope( + "supervised", recovery, self.model, boundary=self.phase_boundary() + ) @classmethod def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": - from sequifier.artifacts.checkpoint_envelope import recovery_payload + from sequifier.artifacts.checkpoint_envelope import ( + recovery_payload, + require_phase_boundary, + ) - payload = recovery_payload(payload, "supervised") + envelope = payload + payload = recovery_payload(envelope, "supervised") require_contract_version( "supervised recovery", payload.get("format_version"), @@ -129,8 +154,10 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": values=dict(payload["integration_state"]["values"]) ), training_config=training_config, + dependencies=dict(payload["dependencies"]), ) checkpoint.validate() + require_phase_boundary(envelope, checkpoint.phase_boundary()) return checkpoint diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index c051f8a3..465076fb 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -544,21 +544,14 @@ def validate_unique_names(self): names = [phase.name for phase in self.phases] if len(names) != len(set(names)): raise ValueError("Training phase names must be unique") - mixed = len({phase.algorithm == "supervised" for phase in self.phases}) > 1 + from sequifier.runtime.plan import validate_continuation + for index, phase in enumerate(self.phases): if phase.optimizer_state != "continue": continue - if index == 0: - raise ValueError("the first phase cannot continue optimizer state") - previous = self.phases[index - 1] - if mixed: - raise ValueError( - "mixed-plan coordination cannot preserve optimizer state" - ) - if previous.algorithm != phase.algorithm or previous.handoff != "last": - raise ValueError( - "optimizer continuation requires the same algorithm and a last handoff" - ) + previous = self.phases[index - 1] if index else None + validate_continuation(previous, phase) + assert previous is not None if phase.algorithm == "supervised" and ( {ref.source.partition(".")[0] for ref in previous.sources} != {ref.source.partition(".")[0] for ref in phase.sources} diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index b87c42c5..edc4ad36 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -550,15 +550,12 @@ def resolve_training_run(authored: SequifierConfig): archive = getattr(phase, "archive_directory", None) if archive: phase.archive_directory = str((root / archive).resolve()) + from sequifier.runtime.plan import validate_continuation + for previous, current in zip( config.training_plan.phases, config.training_plan.phases[1:] ): - if ( - current.optimizer_state == "continue" - and previous.algorithm != "supervised" - and previous.model_interface != current.model_interface - ): - raise ValueError("optimizer continuation cannot change model_interface") + validate_continuation(previous, current, resolved=True) if any(phase.algorithm == "supervised" for phase in config.training_plan.phases): subset = _supervised_subset(config).model_copy( update={"initialize_from": None, "actor_initialization": "attach"} @@ -635,6 +632,7 @@ def policy_run_config( resuming=None, ): """Select resolved inputs for one native policy lifecycle.""" + from sequifier.artifacts.dependencies import model_metadata_paths from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.artifacts.model_config import model_execution_from_model_config from sequifier.artifacts.model_export import model_execution_config @@ -694,6 +692,7 @@ def policy_run_config( execution_model=model_execution_config(execution), policy_interface=name, initialize_from=initialization, + metadata_paths=model_metadata_paths(config, name), actor_initialization=actor_initialization, normalization=config.normalization_by_interface.get(name), units=binding.units, @@ -707,7 +706,7 @@ def policy_run_config( log_interval_iterations=training.log_interval_iterations, resume=ResumeConfig( checkpoint_path=str(checkpoint), - policy=("if_exists" if resuming else "never") + policy=("required" if resuming else "never") if resuming is not None else (resume.policy if resume else "never"), mode=resume.mode if resume else "reset_environments", diff --git a/src/sequifier/contract_versions.py b/src/sequifier/contract_versions.py index 3e07c30f..4739e190 100644 --- a/src/sequifier/contract_versions.py +++ b/src/sequifier/contract_versions.py @@ -4,14 +4,14 @@ class ContractVersions: HISTORICAL_MIN = 0 HISTORICAL_MAX = 99 - RETIRED = () + RETIRED = (101, 102, 104) # Checkpoint envelope and runner-owned payloads. RUN_CHECKPOINT = 100 - SUPERVISED_RECOVERY = 101 - POLICY_RECOVERY = 102 + SUPERVISED_RECOVERY = 140 + POLICY_RECOVERY = 141 POLICY_PHASE_STATE = 103 - LEARNING_RUN_STATE = 104 + LEARNING_RUN_STATE = 142 # Portable artifacts and training telemetry. MODEL_ARTIFACT = 105 @@ -58,6 +58,9 @@ class ContractVersions: CATEGORICAL_CONTROL = 136 TOY_CONTROL = 137 + PHASE_COMMIT = 143 + PHASE_BOUNDARY = 144 + # External ONNX operator set, outside the Sequifier allocation. ONNX_OPSET = 18 diff --git a/src/sequifier/io/images.py b/src/sequifier/io/images.py index 4364e618..004799a3 100644 --- a/src/sequifier/io/images.py +++ b/src/sequifier/io/images.py @@ -91,7 +91,7 @@ def _time_ms(value: Any, *, absolute: bool) -> int: return ((hours * 60 + minutes) * 60 + seconds) * 1000 + millis -def _files(path: str, project_root: str) -> list[Path]: +def media_files(path: str, project_root: str) -> list[Path]: directory = Path(normalize_path(path, project_root)) if not directory.is_dir(): raise ValueError(f"Image source is not a directory: {directory}") @@ -104,7 +104,7 @@ def _files(path: str, project_root: str) -> list[Path]: def _signature(path: str, project_root: str) -> str: digest = hashlib.sha256() - for file in _files(path, project_root): + for file in media_files(path, project_root): stat = file.stat() digest.update(f"{file.name}\0{stat.st_size}\0{stat.st_mtime_ns}\n".encode()) return digest.hexdigest() @@ -139,7 +139,7 @@ def _index( extensions = IMAGE_EXTENSIONS if kind == "images" else VIDEO_EXTENSIONS result: dict[int, dict[int, Path]] = {} timing: dict[int, dict[int, tuple[str | None, int | None]]] = {} - for file in _files(path, project_root): + for file in media_files(path, project_root): if file.suffix.lower() not in extensions: raise ValueError(f"Unsupported file in image source: {file}") if preconditioning: diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py index f689c926..6a70d5c8 100644 --- a/src/sequifier/io/output_paths.py +++ b/src/sequifier/io/output_paths.py @@ -65,7 +65,7 @@ def validate_training_destinations(config, checkpoint, protected): for path in ( artifacts.model, artifacts.report, - artifacts.receipt, + artifacts.model.parent / "commits", artifacts.model.with_name("run.pt"), artifacts.model.parent / "native", artifacts.model.parent / "checkpoints", @@ -144,35 +144,6 @@ def validate_training_destinations(config, checkpoint, protected): ) -def supervised_input_paths(config): - """Include split directories, metadata and media that native loaders consume.""" - root = Path(config.project_root) - paths = set() - if config.initialize_from: - paths.add((root / config.initialize_from).resolve()) - for name, dataset in config.dataset_training.items(): - for part in dataset.parts.values(): - for field in ( - "metadata_config_path", - "training_data_path", - "validation_data_path", - ): - if value := getattr(part, field, None): - paths.add((root / value).resolve()) - interface = config.dataset_interface(name) - if interface.static_features_path: - paths.add((root / interface.static_features_path).resolve()) - for source in interface.image_sources.values(): - if source.path: - paths.add((root / source.path).resolve()) - if interface.preconditioning_image: - paths.update( - (root / path).resolve() - for path in (interface.preconditioning_image.paths or {}).values() - ) - return paths - - def resolve_training_output_paths(config): root = Path(config.project_root) defaults = { diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 25aa74cd..3b980212 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -8,6 +8,7 @@ import torch +from sequifier.artifacts.dependencies import policy_dependencies from sequifier.artifacts.phase import PhaseArtifacts from sequifier.artifacts.reports import phase_report, publish_run_report, run_report from sequifier.fingerprints import fingerprint @@ -37,7 +38,12 @@ weight_fingerprint, ) from sequifier.rl.telemetry import TelemetryJournal -from sequifier.runtime.plan import PhaseOutcome, RunIdentity, execute_phase, run_phases +from sequifier.runtime.plan import ( + PhaseOutcome, + RunIdentity, + run_phases, + validate_continuation, +) from sequifier.runtime.random_state import capture_rng_state, restore_rng_state if TYPE_CHECKING: @@ -70,7 +76,7 @@ def _phase_report(result, checkpoint): def _publish_phase_report(record, checkpoint): result = record.report_result() artifacts = PhaseArtifacts(Path(result.provenance["actor_path"])) - artifacts.publish_report(_phase_report(result, checkpoint)) + record.commit.publish(artifacts, _phase_report(result, checkpoint)) def _new_actor(config): @@ -171,42 +177,54 @@ def run_policy_training( return run_policy_plan(config) -def run_policy_phase( - config: ResolvedPolicyRun, identity: RunIdentity -) -> PolicyRunResult: - """Run one prepared policy phase for a portable handoff.""" - if len(config.phases) != 1: - raise ValueError("a portable policy phase requires one resolved phase") - return _run_policy(config, phase_only=True, identity=identity) +def run_policy_group(config: ResolvedPolicyRun, identity: RunIdentity, on_checkpoint): + """Run a group of policy phases with a shared actor and optimizer lifecycle.""" + _run_policy(config, group=True, identity=identity, on_checkpoint=on_checkpoint) -def _run_policy(config, *, phase_only=False, identity=None): +def _run_policy(config, *, group=False, identity=None, on_checkpoint=None): prepared = PreparedTrainingInputs.for_run(config) with prepared.run_scope(): try: - return _execute_policy(config, prepared, phase_only, identity) + return _execute_policy(config, prepared, group, identity, on_checkpoint) finally: config._metrics_service = None -def _execute_policy(config, prepared, phase_only, identity): +def _execute_policy(config, prepared, group, identity, on_checkpoint): torch.set_float32_matmul_precision("highest") configure_determinism(config.seed, config.enforce_determinism) + for index, phase in enumerate(config.phases): + validate_continuation( + config.phases[index - 1] if index else None, phase, resolved=True + ) plan = _plan_contract(config) inputs = prepared.resolve(config) inputs.revalidate(inputs.identity, "run.source_identity") - data_identity = inputs.identity - validate_output_paths(config, inputs) - configure_logger(config.project_root, config.model_name, 0) checkpoint = Path(config.checkpoint_path) recovery = RunCheckpoint.load( checkpoint, config, plan, - data_identity, run_id=identity.run_id if identity is not None else None, ) resuming = recovery is not None + dependencies = policy_dependencies( + config, + prepared, + inputs, + saved=recovery.data_identity if recovery is not None else None, + committed_reference_count=( + recovery.phase_index + int(recovery.phase_state is not None) + if recovery is not None + else 0 + ), + ) + data_identity = dependencies.identity + if recovery is not None: + recovery.validate_dependencies(config, data_identity) + validate_output_paths(config, dependencies) + configure_logger(config.project_root, config.model_name, 0) adapter_contract = preflight_environments(config) if recovery is not None: require_equal( @@ -266,19 +284,25 @@ def _execute_policy(config, prepared, phase_only, identity): step_budgets = preflight_optimization(config, actor, data) restore_rng_state(actor.device, recovery.random if resuming else saved_random) + def save_checkpoint(state=None): + recovery.save(checkpoint, actor.device, state) + if on_checkpoint is not None: + on_checkpoint( + checkpoint, [record.as_dict() for record in recovery.completed] + ) + if not resuming: # Persist the complete fitted representation before the first update. - recovery.save(checkpoint, actor.device) - for record in recovery.completed: - _publish_phase_report(record, checkpoint) + save_checkpoint() + + def replay(): + for record in recovery.completed: + _publish_phase_report(record, checkpoint) - def run_phase(phase_index, phase): + def run_phase(transition): + phase_index, phase = transition.index, transition.phase current_state = recovery.phase_state - inputs.revalidate( - data_identity, - f"phases.{phase.name}.source_identity", - ) - validate_output_paths(config, inputs) + dependencies.revalidate() incoming_weight_hash = weight_fingerprint(actor.network.state_dict()) initialization = {} if getattr(phase, "history_initialization", None) == "phase_start_actor": @@ -305,9 +329,9 @@ def commit(state): representation.validate_actor( actor, config.units, f"phases.{phase.name}.actor" ) - recovery.save(checkpoint, actor.device, state) + save_checkpoint({**state, "telemetry_path": str(artifacts.telemetry)}) - if recovery.continuation is not None: + if transition.continues_optimizer: actor._phase_continuation = recovery.continuation artifacts = PhaseArtifacts.for_phase( @@ -327,6 +351,7 @@ def commit(state): commit, step_budgets[phase_index], telemetry, + preserve_optimizer=transition.preserves_optimizer, ) selected, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) position = telemetry.commit() @@ -344,18 +369,22 @@ def commit(state): **initialization, **artifacts.describe(), "resource_accounting": prepared.accounting.snapshot(), + "source_provenance": data_identity, }, ) return PhaseOutcome((selected, result, position), result.stop_scope) - def commit_phase(index, phase, outcome): + def commit_phase(transition, outcome): nonlocal actor actor, result, position = outcome.result recovery.continuation = getattr(actor, "_phase_continuation", None) recovery.finish_phase(actor, result, position) - recovery.save(checkpoint, actor.device) - _publish_phase_report(recovery.completed[-1], checkpoint) - if not phase_only and index == last_data_phase: + save_checkpoint() + return recovery.completed[-1] + + def publish(record): + _publish_phase_report(record, checkpoint) + if not group and record.result.index == last_data_phase: release_phase_data() last_data_phase = max( @@ -366,22 +395,21 @@ def release_phase_data(): data.clear() prepared.release_data() - completion_reason = None - if phase_only: - if not recovery.completed: - execute_phase(0, config.phases[0], run_phase, commit_phase) - else: - if last_data_phase < recovery.phase_index: - release_phase_data() - completion_reason = run_phases( - config.phases, - run_phase, - commit_phase, - start=recovery.phase_index, - stopped=recovery.stopped, - ) - results = tuple(record.report_result() for record in recovery.completed) - if completion_reason is not None: + if not group and last_data_phase < recovery.phase_index: + release_phase_data() + completion_reason = run_phases( + config.phases, + run_phase, + commit_phase, + start=recovery.phase_index, + stopped=recovery.stopped, + replay=replay, + publish=publish, + optimizer_continues=lambda phase: phase.optimizer_state == "continue", + ) + results = () + if not group: + results = tuple(record.report_result() for record in recovery.completed) data.clear() prepared.close() report = run_report( @@ -404,7 +432,6 @@ def release_phase_data(): ) save_policy(actor.eval(), config.model_path, metadata=report) publish_run_report(report, config.report_path) - recovery.save(checkpoint, actor.device) return PolicyRunResult( actor, results, diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index cf4819d2..f7216e07 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -26,16 +26,9 @@ from sequifier.runtime.random_state import capture_rng_state, restore_rng_state -def validate_output_paths(config, inputs): +def validate_output_paths(config, dependencies): """Keep every output and archive separate from inputs and other outputs.""" - protected = {path.resolve() for path in inputs.checksums} - if config.initialize_from: - protected.add(Path(config.initialize_from).resolve()) - protected.update( - Path(phase.reference.path).resolve() - for phase in config.phases - if getattr(phase, "reference", None) is not None and phase.reference.path - ) + protected = dependencies.protected_paths outputs = [ Path(config.model_path), Path(config.report_path), diff --git a/src/sequifier/rl/prepared_inputs.py b/src/sequifier/rl/prepared_inputs.py index 83e46ccc..55cfca35 100644 --- a/src/sequifier/rl/prepared_inputs.py +++ b/src/sequifier/rl/prepared_inputs.py @@ -193,6 +193,13 @@ def _phase_dependencies(self, config, phase, inputs): ) return sources, preferences, checksums + def phase_checksums(self, config, phase, inputs): + from sequifier.rl.registry import BUILTIN_ALGORITHMS + + if BUILTIN_ALGORITHMS[phase.algorithm].prepare_data is None: + return {} + return self._phase_dependencies(config, phase, inputs)[2] + def resolve(self, config): declarations = { name: source.model_dump(mode="json") diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index eb47e22a..ead55d17 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -1,23 +1,24 @@ """Policy recovery with artifact-backed, authoritative committed phase boundaries.""" import math -from dataclasses import asdict, dataclass, field, fields, replace +from dataclasses import dataclass, field, fields from pathlib import Path import torch from sequifier.artifacts.checkpoint_envelope import ( checkpoint_envelope, + phase_boundary, recovery_payload, + require_phase_boundary, ) from sequifier.artifacts.model_artifact import ( ModelArtifact, ModelArtifactMetadata, ModelExecutionConfig, ) -from sequifier.artifacts.phase import PhaseArtifacts +from sequifier.artifacts.phase import PhaseArtifacts, PhaseCommit from sequifier.contract_versions import ContractVersions, require_contract_version -from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_save from sequifier.rl.checkpoint import phase_actor_path, policy_model_artifact from sequifier.rl.references import validate_reference_state @@ -28,49 +29,29 @@ require_equal, weight_fingerprint, ) -from sequifier.rl.telemetry import TelemetryJournal from sequifier.runtime.plan import new_run_id from sequifier.runtime.random_state import capture_rng_state -from sequifier.trajectory.sources import file_checksum - -ONLINE_TELEMETRY = ("evaluations", "continuations") @dataclass(frozen=True) class CompletedPhase: - result: PolicyPhaseResult - sha256: str - result_fingerprint: str - telemetry_position: str | None = None + commit: PhaseCommit + + @property + def result(self): + values = self.commit.result + return PolicyPhaseResult(**{**values, "metrics": tuple(values["metrics"])}) @classmethod def from_dict(cls, values): - result = values["result"] - return cls( - PolicyPhaseResult(**{**result, "metrics": tuple(result["metrics"])}), - values["sha256"], - values["result_fingerprint"], - values["telemetry_position"], - ) + return cls(PhaseCommit.load(values)) def as_dict(self): - return asdict(self) + return self.commit.as_dict() def report_result(self): - artifacts = PhaseArtifacts(Path(self.result.provenance["actor_path"])) - history = TelemetryJournal.read(artifacts.telemetry, self.telemetry_position) - return replace( - self.result, - metrics=tuple(history.get("metrics", ())), - provenance={ - **self.result.provenance, - **{ - key: history.get(key, []) - for key in ONLINE_TELEMETRY - if self.result.algorithm in {"ppo", "grpo"} - }, - }, - ) + values = self.commit.report_result() + return PolicyPhaseResult(**{**values, "metrics": tuple(values["metrics"])}) @dataclass @@ -95,7 +76,7 @@ def phase_index(self): return len(self.completed) @classmethod - def load(cls, path, config, plan, data_identity, *, run_id=None): + def load(cls, path, config, plan, data_identity=None, *, run_id=None): path = Path(path) if config.resume.policy == "never": return None @@ -103,9 +84,8 @@ def load(cls, path, config, plan, data_identity, *, run_id=None): if config.resume.policy == "required": raise FileNotFoundError(path) return None - payload = recovery_payload( - torch.load(path, map_location="cpu", weights_only=False), "policy" - ) + envelope = torch.load(path, map_location="cpu", weights_only=False) + payload = recovery_payload(envelope, "policy") require_contract_version( "policy recovery", payload.get("format_version"), @@ -114,11 +94,12 @@ def load(cls, path, config, plan, data_identity, *, run_id=None): if payload.get("artifact_type") != "sequifier_policy_run": raise ValueError("Expected a policy checkpoint recovery payload") if run_id is not None and payload.get("run_id") != run_id: + if config.resume.policy == "required": + raise ValueError("Policy recovery belongs to another run") return None try: saved_plan = payload["plan"] require_equal(plan, saved_plan, "resume.plan") - require_equal(data_identity, payload["data_identity"], "resume.data") completed = [ CompletedPhase.from_dict(record) for record in payload["completed"] ] @@ -136,11 +117,24 @@ def load(cls, path, config, plan, data_identity, *, run_id=None): payload["representation_fingerprint"], "resume.representation_fingerprint", ) + if data_identity is not None: + checkpoint.validate_dependencies(config, data_identity) checkpoint.validate(config) + require_phase_boundary(envelope, checkpoint.phase_boundary()) return checkpoint except (KeyError, TypeError, AttributeError, IndexError) as exc: raise ValueError(f"malformed policy run checkpoint: {exc}") from exc + def validate_dependencies(self, config, data_identity): + if config.actor_initialization == "history_only": + path = str((Path(config.project_root) / config.initialize_from).resolve()) + require_equal( + self.initialization["transfer"]["source_sha256"], + data_identity[path], + "resume.initialize_from.sha256", + ) + require_equal(data_identity, self.data_identity, "resume.data") + def validate(self, config): if self.phase_index > len(config.phases): raise ValueError("checkpoint completed handoffs disagree with the plan") @@ -164,17 +158,13 @@ def validate(self, config): result.provenance["report_path"], "resume.report_path", ) - require_equal( - record.sha256, file_checksum(path), "resume.phase_actor.sha256" - ) - result_data = result.as_dict() - result_data["telemetry_position"] = record.telemetry_position - require_equal( - record.result_fingerprint, - fingerprint(result_data), - "resume.phase_result", - ) - validate_finite_state(result_data, "completed_phase") + if sidecar := record.commit.sidecars.get("telemetry"): + require_equal( + str(PhaseArtifacts(path).telemetry), + sidecar["path"], + "resume.telemetry.path", + ) + validate_finite_state(record.commit.result, "completed_phase") incoming_identity = weight_fingerprint(self.incoming_actor) require_equal( self.completed[-1].result.selected_weight_hash @@ -193,6 +183,14 @@ def validate(self, config): state.get("version"), ContractVersions.POLICY_PHASE_STATE, ) + artifacts = PhaseArtifacts.for_phase( + config.phase_directory, + self.phase_index, + config.phases[self.phase_index].name, + ) + require_equal( + str(artifacts.telemetry), state["telemetry_path"], "resume.telemetry.path" + ) if not isinstance(state["telemetry_position"], str): raise ValueError("phase checkpoint requires a committed telemetry position") transaction = state["transaction"] @@ -243,6 +241,28 @@ def validate_actor(self, actor, units): ) actor.network.load_state_dict(self.incoming_actor, strict=True) + def phase_boundary(self): + state = self.phase_state + active = None + if state is not None: + active = { + "index": self.phase_index, + "transaction": state["transaction"], + "sidecars": { + "telemetry": { + "path": state["telemetry_path"], + "position": state["telemetry_position"], + } + }, + } + return phase_boundary( + self.run_id, + selected=self.completed[-1].commit.model_identity + if self.completed + else None, + active=active, + ) + def save(self, path, device, state=None): """Atomically commit the run and its algorithm-owned recovery state.""" self.random = capture_rng_state(device) @@ -270,6 +290,7 @@ def save(self, path, device, state=None): "completed": [record.as_dict() for record in self.completed], }, model, + boundary=self.phase_boundary(), ), ) @@ -282,17 +303,16 @@ def finish_phase(self, actor, result, telemetry_position): ) artifacts = PhaseArtifacts(Path(result.provenance["actor_path"])) result_data = result.as_dict() - artifacts.publish_model(policy_model_artifact(actor, metadata=result_data)) - weights = clone_weights(actor.network) - self.completed.append( - CompletedPhase( - result, - **artifacts.record( - {**result_data, "telemetry_position": telemetry_position} - ), - telemetry_position=telemetry_position, - ) + commit = PhaseCommit.stage( + artifacts, + result_data, + model=policy_model_artifact(actor, metadata=result_data), + telemetry={"path": str(artifacts.telemetry), "position": telemetry_position} + if telemetry_position is not None + else None, ) + weights = clone_weights(actor.network) + self.completed.append(CompletedPhase(commit)) self.stopped = result.stop_scope == "end_plan" self.incoming_actor = weights self.phase_state = None diff --git a/src/sequifier/rl/run_config.py b/src/sequifier/rl/run_config.py index 9e77618e..cef4b167 100644 --- a/src/sequifier/rl/run_config.py +++ b/src/sequifier/rl/run_config.py @@ -27,6 +27,7 @@ class ResolvedPolicyRun(BaseModel): execution_model: dict policy_interface: str initialize_from: str | None = None + metadata_paths: list[str] = Field(default_factory=list) actor_initialization: Literal["attach", "history_only"] = "attach" normalization: FrozenNormalizationConfig | None = None sources: dict[str, SequenceSourceConfig] diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index e36e904a..3c1776b2 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -146,6 +146,7 @@ class PhaseContext(Generic[PhaseType, DataType]): commit: Callable[[dict], None] step_budget: int | None telemetry: TelemetryJournal + preserve_optimizer: bool = False class Selection: @@ -569,12 +570,7 @@ def checkpoint(self, final, **extra): def finish(self, clocks, provenance): if self.context.run._metrics_service is not None: self.context.run._metrics_service.flush(self) - next_index = self.context.index + 1 - preserve = ( - next_index < len(self.context.run.phases) - and self.context.run.phases[next_index].optimizer_state == "continue" - ) - if preserve: + if self.context.preserve_optimizer: self.context.actor._phase_continuation = { "algorithm": self.context.phase.algorithm, "networks": { diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 274d18c6..c53c9219 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -11,6 +11,11 @@ from loguru import logger from sequifier.artifacts.backbone_repository import load_revision, select_revision +from sequifier.artifacts.dependencies import ( + DependencyManifest, + add_backbone_revision, + supervised_dependencies, +) from sequifier.artifacts.run_checkpoint import RunCheckpointStore, select_run_checkpoint from sequifier.config.optimizer_config import OptimizerPlan from sequifier.evaluation.service import EvaluationService @@ -72,15 +77,20 @@ class TrainingRun: loss: LossService loader_state: LoaderStateService context: RunContext + dependencies: DependencyManifest precision: PrecisionPolicy def start_phase( - self, view: SupervisedPhaseView, datasets: DatasetRuntimeRegistry + self, + view: SupervisedPhaseView, + datasets: DatasetRuntimeRegistry, + *, + continue_optimizer: bool, ) -> None: """Enter the phase with its resolved reset or continuation policy.""" phase = view.spec() optimizer_step = self.optimization.optimizer_step - if phase.optimizer_state == "reset": + if not continue_optimizer: self.optimization = self.optimization_factory(view) self.optimization.gradient_policy = GradientPolicy( phase.accumulation_batches, @@ -170,24 +180,20 @@ def build( *, initial_artifact=None, identity: RunIdentity | None = None, + on_checkpoint=None, + backbone_revision=None, + dependencies: DependencyManifest | None = None, ) -> TrainingRun: from sequifier.artifacts.run_checkpoint import ( checkpoint_path as output_checkpoint_path, ) from sequifier.io.output_paths import ( resolve_training_output_paths, - supervised_input_paths, validate_training_destinations, ) config = config.model_copy() resolve_training_output_paths(config) - protected = supervised_input_paths(config) - if initial_artifact is not None: - protected.add(Path(initial_artifact).resolve()) - validate_training_destinations( - config, output_checkpoint_path(config), protected - ) precision = PrecisionPolicy.from_settings( config.runtime_settings, execution.device ) @@ -203,9 +209,31 @@ def build( and identity is not None and loaded.checkpoint.run_state["run_id"] != identity.run_id ): + if config.runtime_settings.resume.policy == "required": + raise ValueError("Supervised recovery belongs to another run") loaded = None + revision = None + dependencies = dependencies or supervised_dependencies(config, initial_artifact) + if initial_artifact is None and config.initialize_from is None: + if loaded is None: + revision = ( + select_revision(config.model.backbone, config.project_root) + if backbone_revision is None + else backbone_revision or None + ) + revision_id = ( + loaded.checkpoint.run_state["backbone_parent_revision_id"] + if loaded is not None + else revision["revision_id"] + if revision is not None + else None + ) + add_backbone_revision(dependencies, config, revision_id) + validate_training_destinations( + config, output_checkpoint_path(config), dependencies.protected_paths + ) if loaded is not None: - restorer.validate_for_restore(loaded, config) + restorer.validate_for_restore(loaded, config, dependencies) # Optimizer state uses the parameter order of the saved network. interfaces = { name: config.interfaces[name] @@ -249,11 +277,9 @@ def build( from sequifier.artifacts.model_config import load_attached_model_weights load_attached_model_weights(network, artifact) - if loaded is None and initial_artifact is None: - revision = select_revision(config.model.backbone, config.project_root) - if revision is not None: - load_revision(network.backbone, revision) - state.backbone_parent_revision_id = revision["revision_id"] + if revision is not None: + load_revision(network.backbone, revision) + state.backbone_parent_revision_id = revision["revision_id"] freezing_plans = [] planned_phases = [] @@ -394,11 +420,13 @@ def optimization_factory(view: SupervisedPhaseView) -> OptimizationRuntime: strategy, random_manager, loader_state, + on_checkpoint=on_checkpoint, ), metrics=MetricsService(context, config), loss=loss_service, loader_state=loader_state, context=context, + dependencies=dependencies, precision=precision, ) self._construct_loaders(run) diff --git a/src/sequifier/runtime/plan.py b/src/sequifier/runtime/plan.py index bc860324..e84904b1 100644 --- a/src/sequifier/runtime/plan.py +++ b/src/sequifier/runtime/plan.py @@ -7,6 +7,7 @@ Phase = TypeVar("Phase") Result = TypeVar("Result") +Commit = TypeVar("Commit") def new_run_id() -> str: @@ -44,31 +45,82 @@ def select_handoff(handoff: str, last: Result, best: Result | None) -> Result: return best if handoff == "best" and best is not None else last -def execute_phase( - index: int, - phase: Phase, - run_phase: Callable[[int, Phase], PhaseOutcome[Result]], - commit_phase: Callable[[int, Phase, PhaseOutcome[Result]], None], -) -> PhaseOutcome[Result]: - outcome = run_phase(index, phase) - commit_phase(index, phase, outcome) - return outcome +@dataclass(frozen=True) +class PhaseTransition(Generic[Phase]): + index: int + phase: Phase + previous: Phase | None + continues_optimizer: bool = False + preserves_optimizer: bool = False + + +def validate_continuation(previous, phase, *, resolved=False): + if phase.optimizer_state != "continue": + return + if previous is None: + raise ValueError("the first phase cannot continue optimizer state") + if previous.algorithm != phase.algorithm or previous.handoff != "last": + raise ValueError( + "optimizer continuation requires the same algorithm and a last handoff" + ) + if ( + resolved + and phase.algorithm != "supervised" + and previous.model_interface != phase.model_interface + ): + raise ValueError("optimizer continuation cannot change model_interface") + + +def continuation_groups(phases): + """A reset starts a new portable execution boundary.""" + start = 0 + for index, phase in enumerate(phases[1:], 1): + if phase.optimizer_state != "continue": + yield start, phases[start:index] + start = index + yield start, phases[start:] def run_phases( phases: Sequence[Phase], - run_phase: Callable[[int, Phase], PhaseOutcome[Result]], - commit_phase: Callable[[int, Phase, PhaseOutcome[Result]], None], + run_phase: Callable[[PhaseTransition[Phase]], PhaseOutcome[Result]], + commit_phase: Callable[[PhaseTransition[Phase], PhaseOutcome[Result]], Commit], *, start: int = 0, stopped: bool = False, completion_reason: str = "completed", + replay: Callable[[], None] | None = None, + publish: Callable[[Commit], None] | None = None, + terminal_reason: str | None = None, + optimizer_continues: Callable[[Phase], bool] | None = None, ) -> str: - """Commit each native result before acting on its stop or advancing.""" + """Replay committed publications, then advance only after a durable commit.""" + if replay is not None: + replay() if stopped: return "early_stopping" + if terminal_reason is not None: + return terminal_reason for index in range(start, len(phases)): - outcome = execute_phase(index, phases[index], run_phase, commit_phase) + transition = PhaseTransition( + index, + phases[index], + phases[index - 1] if index else None, + continues_optimizer=( + index > 0 + and optimizer_continues is not None + and optimizer_continues(phases[index]) + ), + preserves_optimizer=( + index + 1 < len(phases) + and optimizer_continues is not None + and optimizer_continues(phases[index + 1]) + ), + ) + outcome = run_phase(transition) + commit = commit_phase(transition, outcome) + if publish is not None: + publish(commit) if outcome.stop_scope == "end_plan": return "early_stopping" if outcome.completion_reason is not None: diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 27369f9e..b86e8d34 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -84,14 +84,13 @@ def train_worker( ) -> None: """Initialize one execution environment and run the composed runtime.""" + from sequifier.artifacts.dependencies import supervised_dependencies from sequifier.artifacts.run_checkpoint import checkpoint_path - from sequifier.io.output_paths import ( - supervised_input_paths, - validate_training_destinations, - ) + from sequifier.io.output_paths import validate_training_destinations + dependencies = supervised_dependencies(config) validate_training_destinations( - config, checkpoint_path(config), supervised_input_paths(config) + config, checkpoint_path(config), dependencies.protected_paths ) configure_logger( config.project_root, @@ -133,7 +132,11 @@ def train_worker( run = None try: run = RunBuilder(semantic_optimizer_grouping=semantic_optimizer_grouping).build( - config, execution, integrations, initial_artifact=config.initialize_from + config, + execution, + integrations, + initial_artifact=config.initialize_from, + dependencies=dependencies, ) loguru_logger.info( f"--- Starting Training for model: {run.context.model_name} | " diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index 66c0aee0..b5bc91a6 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Mapping +from sequifier.artifacts.dependencies import DependencyManifest from sequifier.artifacts.model_artifact import build_model_artifact from sequifier.artifacts.model_export import model_execution_config from sequifier.artifacts.run_checkpoint import ( @@ -38,9 +39,13 @@ def validate(self, checkpoint: RunCheckpoint) -> None: checkpoint.validate() def validate_for_restore( - self, checkpoint: RunCheckpoint, current_config: Any + self, + checkpoint: RunCheckpoint, + current_config: Any, + dependencies: DependencyManifest, ) -> None: self.validate(checkpoint) + dependencies.validate(checkpoint.dependencies) saved = self._resume_settings( checkpoint.training_config, model_config=checkpoint.model.model_config.values, @@ -82,16 +87,16 @@ def strip_provenance(value: Any) -> Any: execution = strip_provenance(execution) for interface in execution.get("interfaces", {}).values(): - # Paths locate data for inference; an exact resume may use a new - # project directory containing the same splits. + # The loader paths are compared with their owning dataset parts. interface.pop("split_paths", None) training = config.runtime_settings datasets = { name: { "model_interface": dataset.model_interface, "parts": list(dataset.parts), - "part_formats": { - name: part.read_format for name, part in dataset.parts.items() + "part_inputs": { + name: part.model_dump(mode="python", exclude={"metadata"}) + for name, part in dataset.parts.items() }, } for name, dataset in config.dataset_training.items() @@ -181,7 +186,9 @@ def __init__( random_state: Any, loader_state: LoaderStateService, compatibility: CheckpointCompatibility | None = None, + on_checkpoint=None, ) -> None: + self.on_checkpoint = on_checkpoint self.store = store self.distributed = distributed self.random_state = random_state @@ -220,12 +227,15 @@ def save(self, request: CheckpointRequest, run: Any) -> Path | None: random_state=random_state, loader_state=loader_state, integration_state=IntegrationState(values=integration_state), + dependencies=run.dependencies.identity, training_config=run.config.model_copy( update={"training_plan": [phase.spec() for phase in run.plan.phases]} ), ) self.compatibility.validate(checkpoint) written = self.store.save(checkpoint, request.suffix) + if self.on_checkpoint is not None: + self.on_checkpoint(self.store.latest_path, run.state.phase_commits) run.integrations.emit( CheckpointSaved(access=access, identity=request.identity, path=written) ) @@ -242,9 +252,14 @@ def load(self, path: Path) -> LoadedRunCheckpoint: return LoadedRunCheckpoint(checkpoint, path) def validate_for_restore( - self, loaded: LoadedRunCheckpoint, current_config: Any + self, + loaded: LoadedRunCheckpoint, + current_config: Any, + dependencies: DependencyManifest, ) -> None: - self.compatibility.validate_for_restore(loaded.checkpoint, current_config) + self.compatibility.validate_for_restore( + loaded.checkpoint, current_config, dependencies + ) def restore_model( self, loaded: LoadedRunCheckpoint, network: Any, distributed: Any diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index ad2bdaa8..c1d1253a 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -1,37 +1,53 @@ -"""Atomic phase coordination around the native supervised and policy runners. - -The envelope commits an immutable incoming artifact before starting a phase. -Native checkpoints own in-phase optimizer, sampler, and environment recovery. -Completing a phase publishes its selected artifact before advancing the envelope. -""" +"""Coordinate native training and commit portable handoffs.""" from dataclasses import dataclass +from os import link from pathlib import Path -from typing import Any +from shutil import copyfile +from uuid import uuid4 import torch +from sequifier.artifacts.backbone_repository import select_revision from sequifier.artifacts.checkpoint_envelope import ( checkpoint_envelope, + phase_boundary, recovery_payload, + require_phase_boundary, +) +from sequifier.artifacts.dependencies import ( + DependencyManifest, + add_backbone_revision, + add_initialization, + policy_dependencies, + supervised_dependencies, ) from sequifier.artifacts.model_artifact import ( - ModelArtifact, read_execution_artifact, save_model_artifact, ) -from sequifier.artifacts.phase import PhaseArtifacts, verify_artifact +from sequifier.artifacts.phase import PhaseArtifacts, PhaseCommit, verify_artifact from sequifier.artifacts.reports import phase_report, publish_run_report, run_report +from sequifier.artifacts.run_checkpoint import checkpoint_path +from sequifier.config.components import ResumeConfig from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.fingerprints import fingerprint -from sequifier.io.atomic import atomic_save -from sequifier.runtime.plan import PhaseOutcome, RunIdentity, new_run_id, run_phases +from sequifier.io.atomic import atomic_path, atomic_save +from sequifier.io.output_paths import validate_training_destinations +from sequifier.runtime.plan import ( + PhaseOutcome, + RunIdentity, + continuation_groups, + new_run_id, + run_phases, + validate_continuation, +) from sequifier.training.phase_runners import ( - PortablePhaseRequest, - policy_phase_config, - run_portable_policy_phase, - run_supervised_phase, - supervised_phase_config, + PortableGroup, + policy_group_config, + run_portable_policy_group, + run_supervised_group, + supervised_group_config, ) from sequifier.trajectory.sources import file_checksum @@ -44,14 +60,6 @@ class LearningRunResult: phases: tuple[dict, ...] -@dataclass(frozen=True) -class PortablePhaseCommit: - artifacts: PhaseArtifacts - fingerprint: str - model: ModelArtifact | None - report: dict - - def coordinate_training( config, *, @@ -94,20 +102,43 @@ def run_policy_plan(config): return run_learning_plan(config) -def _completed_report(item): - if item["algorithm"] == "supervised": - report = item["report"] - require_contract_version( - "phase report", report.get("format_version"), ContractVersions.PHASE_REPORT - ) - return report - artifacts = PhaseArtifacts(Path(item["model_path"])) - receipt = artifacts.read_receipt(**item["report_receipt"]) - if receipt is None: - raise ValueError( - f"Missing committed policy report receipt: {artifacts.receipt}" +def _snapshot_recovery(source, destination): + """Pin an atomic native checkpoint without depending on its rolling path.""" + with atomic_path(destination, durable=True) as temporary: + try: + link(source, temporary) + except OSError: + copyfile(source, temporary) + + +def _portable_commit(config, index, phase, checkpoint, values): + commit = PhaseCommit(**values) + artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) + details = dict(commit.result) + for key in ( + "index", + "name", + "algorithm", + "model_path", + "checkpoint_path", + "artifact_type", + "format_version", + ): + details.pop(key, None) + details["provenance"] = {**details["provenance"], **artifacts.describe()} + if phase.algorithm == "supervised": + details["resolved_seed"] = config.seed + index + details["optimizer_steps"] = details["clocks"]["optimizer_steps"] + return commit.with_result( + phase_report( + index=index, + name=phase.name, + algorithm=phase.algorithm, + model_path=artifacts.model, + checkpoint_path=checkpoint, + **details, ) - return receipt["report"] + ) def run_learning_plan(config) -> LearningRunResult: @@ -118,37 +149,92 @@ def run_learning_plan(config) -> LearningRunResult: return _run_learning_plan(config, prepared) -def _run_learning_plan(config, prepared) -> LearningRunResult: - phases = config.training_plan - if any(phase.optimizer_state == "continue" for phase in phases): - raise ValueError("portable phase handoffs cannot continue optimizer state") - resume = config.runtime_settings.resume - from sequifier.config.components import ResumeConfig +def _prepare_plan(config, prepared, groups, revision, state): + from sequifier.rl.preparation import ( + preflight_environments, + prepare_data, + validate_output_paths, + ) + + manifest = DependencyManifest() + add_initialization(manifest, config) + add_backbone_revision( + manifest, config, revision["revision_id"] if revision else None + ) + last_data_phase = -1 + supervised_runs = [] + committed = len(state["completed"]) + active = state["active"] + for index, phases in groups: + phase = phases[0] + if phase.algorithm == "supervised": + native = supervised_group_config(config, phases, index, None, False) + manifest.merge(supervised_dependencies(native)) + supervised_runs.append(native) + continue + artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) + native = policy_group_config( + config, phases, index, None, artifacts.model, False + ) + inputs = prepared.resolve(native) + inputs.revalidate(inputs.identity, f"phases.{phase.name}.source_identity") + # Preflight can defer the active reference to the native checkpoint. + # The policy runner verifies that its weights were actually embedded. + can_recover_active = ( + active is not None + and active["index"] == index + and active["recovery"] is not None + ) + dependencies = policy_dependencies( + native, + prepared, + inputs, + saved=state.get("dependencies"), + committed_reference_count=max( + 0, committed - index + int(can_recover_active) + ), + ) + validate_output_paths(native, dependencies) + manifest.merge(dependencies) + preflight_environments(native) + for offset, data in enumerate(prepare_data(native, native.policy, inputs)): + if data is not None: + last_data_phase = index + offset + for native in supervised_runs: + validate_training_destinations( + native, checkpoint_path(native), manifest.protected_paths + ) + return manifest, last_data_phase - resume = resume or ResumeConfig() - from sequifier.artifacts.run_checkpoint import checkpoint_path +def _run_learning_plan(config, prepared): + phases = config.training_plan + if not phases: + raise ValueError("A learning plan requires at least one phase") + for index, phase in enumerate(phases): + validate_continuation( + phases[index - 1] if index else None, phase, resolved=True + ) + groups = [(index, tuple(group)) for index, group in continuation_groups(phases)] + resume = config.runtime_settings.resume or ResumeConfig() checkpoint = checkpoint_path(config) plan = config.model_dump(mode="json") plan["runtime_settings"].pop("resume", None) plan.get("ownership", {}).get("run", {}).get("recovery", {}).pop("resume", None) - identity = fingerprint(plan) - state: dict[str, Any] = { + plan_identity = fingerprint(plan) + state = { "artifact_type": "sequifier_learning_run", "format_version": ContractVersions.LEARNING_RUN_STATE, - "plan": identity, + "plan": plan_identity, "completed": [], - "incoming": config.initialize_from, - "incoming_sha256": file_checksum(config.initialize_from) - if config.initialize_from - else None, - "active": False, + "active": None, "run_id": new_run_id(), } - if resume.policy != "never" and checkpoint.is_file(): - state = recovery_payload( - torch.load(checkpoint, map_location="cpu", weights_only=False), "learning" - ) + envelope = None + resuming = resume.policy != "never" and checkpoint.is_file() + if resuming: + envelope = torch.load(checkpoint, map_location="cpu", weights_only=False) + state = recovery_payload(envelope, "learning") require_contract_version( "learning run coordinator state", state.get("format_version"), @@ -156,242 +242,220 @@ def _run_learning_plan(config, prepared) -> LearningRunResult: ) if ( state.get("artifact_type") != "sequifier_learning_run" - or state.get("plan") != identity + or state.get("plan") != plan_identity ): raise ValueError("learning run checkpoint differs from the requested plan") - if state["active"] and state.get("active_run_id") != state["run_id"]: - raise ValueError( - "learning run checkpoint requires the active phase run identity" - ) elif resume.policy == "required": raise FileNotFoundError(checkpoint) - identity_scope = RunIdentity.start(state["run_id"]) - state["run_id"] = identity_scope.run_id - if not 0 <= len(state["completed"]) <= len(phases): - raise ValueError("learning run checkpoint exceeds the phase plan") - verify_artifact( - state["incoming"], - state["incoming_sha256"], - "committed incoming model artifact changed", - ) - initialization = {"path": config.initialize_from} - if config.actor_initialization == "history_only" and state["completed"]: - initialization = _completed_report(state["completed"][0])["provenance"][ - "actor_initialization" - ] - verify_artifact( - config.initialize_from, - initialization["transfer"]["source_sha256"], - "history initialization artifact changed", - ) - # Validate model/source/environment contracts for the complete plan first. - from sequifier.rl.preparation import ( - preflight_environments, - prepare_data, - validate_output_paths, - ) - dependencies = {} - last_data_phase = -1 - protected_inputs = set() - for index, phase in enumerate(phases): - if index == 0: - phase_incoming = config.initialize_from - elif index <= len(state["completed"]): - phase_incoming = state["completed"][index - 1]["model_path"] - else: - phase_incoming = None - if phase.algorithm == "supervised": - resolved = supervised_phase_config( - config, phase, index, phase_incoming, False + revision = None + if ( + phases[0].algorithm == "supervised" + and not config.initialize_from + and (not resuming or state["backbone_revision"] is not None) + ): + revision = select_revision( + config.model.backbone, + config.project_root, + revision_id=state.get("backbone_revision"), + ) + state["backbone_revision"] = revision["revision_id"] if revision else None + identity = RunIdentity.start(state["run_id"]) + completed = [PhaseCommit.load(values) for values in state["completed"]] + if len(completed) > len(phases) or any( + (commit.result["index"], commit.result["name"], commit.result["algorithm"]) + != (index, phase.name, phase.algorithm) + for index, (commit, phase) in enumerate(zip(completed, phases)) + ): + raise ValueError("learning run checkpoint differs from the phase plan") + active = state["active"] + if active is not None: + ends = {start: start + len(group) for start, group in groups} + start = active["index"] + if ( + active["run_id"] != identity.run_id + or start not in ends + or not start <= len(completed) < ends[start] + or active["phase_index"] != len(completed) + or (active["recovery"] is None and start != len(completed)) + ): + raise ValueError( + "learning run checkpoint has an invalid active phase group" ) - from sequifier.io.output_paths import supervised_input_paths + manifest, last_data_phase = _prepare_plan(config, prepared, groups, revision, state) + if resuming: + manifest.validate(state["dependencies"]) + state["dependencies"] = manifest.identity + validate_training_destinations(config, checkpoint, manifest.protected_paths) - protected_inputs.update(supervised_input_paths(resolved)) - for dataset in resolved.dataset_training.values(): - for part in dataset.parts.values(): - path = str(Path(config.project_root) / part.metadata_config_path) - dependencies[path] = file_checksum(path) - else: - destination = PhaseArtifacts.for_phase( - config.phase_directory, index, phase.name, "model.pt" - ).model - run = policy_phase_config( - config, - phase, - index, - phase_incoming, - destination, - False, - ) - inputs = prepared.resolve(run) - inputs.revalidate(inputs.identity, f"phases.{phase.name}.source_identity") - validate_output_paths(run, inputs) - dependencies.update(inputs.identity) - preflight_environments(run) - if prepare_data(run, run.policy, inputs)[0] is not None: - last_data_phase = index - if run.execution_model is None or run.policy_interface is None: - raise ValueError("a policy phase has no resolved model interface") - if last_data_phase < len(state["completed"]): + if last_data_phase < len(completed): prepared.release_data() - if "dependencies" in state and state["dependencies"] != dependencies: - raise ValueError("learning plan source identities changed since its checkpoint") - state["dependencies"] = dependencies - protected = {Path(path).resolve() for path in dependencies} | protected_inputs - for phase in phases: - reference = getattr(phase, "reference", None) - if reference is not None and reference.path: - protected.add(Path(reference.path).resolve()) - if config.initialize_from: - protected.add(Path(config.initialize_from).resolve()) - from sequifier.io.output_paths import validate_training_destinations - - validate_training_destinations(config, checkpoint, protected) - def save_boundary(): - model = ( - read_execution_artifact(state["incoming"]) if state["incoming"] else None - ) - atomic_save(checkpoint, checkpoint_envelope("learning", state, model)) - - def run_phase(index, phase): - incoming = state["incoming"] - verify_artifact( - incoming, - state["incoming_sha256"], - "incoming model artifact changed since the committed boundary", + def artifact_before(index): + return ( + completed[index - 1].model_path if index else config.initialize_from or None ) - artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) - resuming = state["active"] and resume.policy != "never" - if not state["active"]: - state.update(active=True, active_run_id=identity_scope.run_id) - save_boundary() - phase_identity = fingerprint( - { - "phase": phase.model_dump(mode="json"), - "incoming": state["incoming_sha256"], - } - ) - receipt = ( - artifacts.read_receipt(phase_identity, run_id=state.get("active_run_id")) - if resuming - else None + + initial = config.initialize_from + initial_identity = ( + {"path": initial, "sha256": manifest.identity[str(Path(initial).resolve())]} + if initial + else None + ) + + def boundary(): + return phase_boundary( + identity.run_id, + selected=completed[-1].model_identity if completed else initial_identity, + active=state["active"], ) - result = None - if receipt is not None: - report = receipt["report"] - else: - request = PortablePhaseRequest( - index, incoming, artifacts, resuming, identity_scope - ) - runner = ( - run_supervised_phase - if phase.algorithm == "supervised" - else run_portable_policy_phase - ) - result = runner(config, phase, request) - details = { - key: value - for key, value in result.report.items() - if key - not in { - "index", - "name", - "algorithm", - "model_path", - "checkpoint_path", - "artifact_type", - "format_version", - } - } - if phase.algorithm != "supervised": - details["provenance"] = { - **details["provenance"], - **artifacts.describe(), - } - report = phase_report( - index=index, - name=phase.name, - algorithm=phase.algorithm, - model_path=artifacts.model, - checkpoint_path=artifacts.model.with_name("run.pt"), - **details, + + if envelope is not None: + require_phase_boundary(envelope, boundary()) + + selected_path, selected_model = None, None + + def save_boundary(): + nonlocal selected_path, selected_model + selected = artifact_before(len(completed)) + if selected != selected_path: + selected_model = ( + read_execution_artifact(selected) if selected is not None else None ) - return PhaseOutcome( - PortablePhaseCommit( - artifacts, phase_identity, result.artifact if result else None, report + selected_path = selected + atomic_save( + checkpoint, + checkpoint_envelope( + "learning", + state, + selected_model, + boundary=boundary(), ), - report.get("stop_scope"), ) - def commit_phase(index, phase, outcome): - nonlocal initialization - commit = outcome.result - artifacts, report = commit.artifacts, commit.report - if commit.model is not None: - artifacts.publish_model(commit.model) - artifacts.write_receipt( - commit.fingerprint, report, run_id=state.get("active_run_id") + def publish(commit): + report = commit.report_result() + commit.publish(PhaseArtifacts(Path(report["model_path"])), report) + + def replay(): + for commit in completed: + publish(commit) + + def run_group(transition): + index, group = transition.phase + first = group[0] + base = PhaseArtifacts.for_phase(config.phase_directory, index, first.name) + native_checkpoint = base.model.with_name("run.pt") + manifest.revalidate() + if index: + incoming = completed[index - 1] + verify_artifact( + incoming.model_path, + incoming.model_sha256, + "Selected phase model changed", ) - artifacts.publish_report(report) - if index == 0 and config.actor_initialization == "history_only": - initialization = report["provenance"]["actor_initialization"] - state["stopped"] = outcome.stop_scope == "end_plan" - checksum = file_checksum(artifacts.model) - item = { - "name": phase.name, - "algorithm": phase.algorithm, - "model_path": str(artifacts.model), - } - if phase.algorithm == "supervised": - item["report"] = report - else: - item["report_receipt"] = { - "identity": commit.fingerprint, - "run_id": state.get("active_run_id"), + active = state["active"] + if active is None: + active = { + "index": index, + "phase_index": index, + "run_id": identity.run_id, + "recovery": None, } - state["completed"].append(item) - state.update( - incoming=str(artifacts.model), - incoming_sha256=checksum, - active=False, + state["active"] = active + save_boundary() + prior = active["recovery"] + if prior is not None: + verify_artifact( + prior["path"], prior["sha256"], "Active phase recovery changed" + ) + _snapshot_recovery(prior["path"], native_checkpoint) + + def save_progress(path, records): + previous = active["recovery"] + first_new = len(completed) - index + new = [ + _portable_commit( + config, index + offset, group[offset], native_checkpoint, values + ) + for offset, values in enumerate(records[first_new:], first_new) + ] + finished = len(records) == len(group) or ( + new and new[-1].result["stop_scope"] == "end_plan" + ) + if finished: + state["active"] = None + else: + recovery_path = ( + base.model.parent / "commits" / f"{uuid4().hex}.recovery.pt" + ) + _snapshot_recovery(path, recovery_path) + active["phase_index"] = len(completed) + len(new) + active["recovery"] = { + "path": str(recovery_path), + "sha256": file_checksum(recovery_path), + } + completed.extend(new) + state["completed"].extend(commit.as_dict() for commit in new) + save_boundary() + for commit in new: + publish(commit) + if previous is not None: + Path(previous["path"]).unlink(missing_ok=True) + + request = PortableGroup( + index, + group, + artifact_before(index), + prior is not None, + identity, + save_progress, ) - state.pop("active_run_id", None) - save_boundary() - if index == last_data_phase: + if first.algorithm == "supervised": + run_supervised_group(config, request, revision if revision else {}) + else: + run_portable_policy_group(config, request) + stop = completed[-1].result["stop_scope"] + if stop != "end_plan" and len(completed) != index + len(group): + raise ValueError("native phase group returned before committing its phases") + return PhaseOutcome(None, stop) + + def finish_group(transition, outcome): + if transition.phase[0] <= last_data_phase < len(completed): prepared.release_data() - for index, item in enumerate(state["completed"]): - artifacts = PhaseArtifacts.for_phase( - config.phase_directory, index, phases[index].name - ) - artifacts.publish_report(_completed_report(item)) + next_index = state["active"]["index"] if state["active"] else len(completed) + start = next( + (offset for offset, (index, _) in enumerate(groups) if index == next_index), + len(groups), + ) + stopped = bool(completed and completed[-1].result["stop_scope"] == "end_plan") + if start == len(groups) and next_index != len(phases) and not stopped: + raise ValueError("learning run checkpoint is between inconsistent phase groups") completion_reason = run_phases( - phases, - run_phase, - commit_phase, - start=len(state["completed"]), - stopped=state.get("stopped", False), + groups, + run_group, + finish_group, + start=start, + stopped=stopped and state["active"] is None, + replay=replay, ) - final = read_execution_artifact(state["incoming"]) + final = read_execution_artifact(completed[-1].model_path) save_model_artifact(final, config.model_path) - completed = [] - for item in state["completed"]: - result = dict(item) - result.pop("report_receipt", None) - result["report"] = _completed_report(item) - completed.append(result) + reports = [commit.report_result() for commit in completed] + initialization = reports[0]["provenance"].get( + "actor_initialization", {"path": config.initialize_from} + ) prepared.close() publish_run_report( run_report( model_path=config.model_path, checkpoint_path=checkpoint, - phases=[item["report"] for item in completed], - plan_fingerprint=identity, - run_id=state["run_id"], - session_id=identity_scope.session_id, - source_provenance=dependencies, + phases=reports, + plan_fingerprint=plan_identity, + run_id=identity.run_id, + session_id=identity.session_id, + source_provenance=manifest.identity, initialization=initialization, resume=resume.model_dump(mode="json"), exports=[config.model_path], @@ -405,5 +469,11 @@ def commit_phase(index, phase, outcome): config.model_path, str(checkpoint), config.report_path, - tuple(completed), + tuple( + { + **{key: report[key] for key in ("name", "algorithm", "model_path")}, + "report": report, + } + for report in reports + ), ) diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index b6717bc8..269a5f00 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -14,8 +14,12 @@ import torch from sequifier.artifacts.backbone_repository import publish_revision -from sequifier.artifacts.model_artifact import build_model_artifact, save_model_artifact -from sequifier.artifacts.phase import PhaseArtifacts +from sequifier.artifacts.model_artifact import ( + build_model_artifact, + read_execution_artifact, + save_model_artifact, +) +from sequifier.artifacts.phase import PhaseArtifacts, PhaseCommit from sequifier.artifacts.reports import phase_report, publish_run_report, run_report from sequifier.artifacts.run_checkpoint import LoaderState from sequifier.artifacts.state_dict import canonicalize_state_dict @@ -34,7 +38,6 @@ from sequifier.logging_paths import model_log_directory from sequifier.runtime.plan import ( PhaseOutcome, - execute_phase, metric_improved, run_phases, select_handoff, @@ -244,7 +247,8 @@ def _manifest( run_report( model_path=run.config.model_path, checkpoint_path=run.checkpoints.store.latest_path, - phases=run.state.phase_reports, + phases=[record["result"] for record in run.state.phase_commits], + source_provenance=run.dependencies.identity, run_id=run.state.run_id, session_id=run.state.session_id, status=status, @@ -258,8 +262,8 @@ def _manifest( run.config.report_path, ) - def _publish_phase(self, run, index, phase): - """Publish the selected portable artifact on the supervised phase boundary.""" + def _stage_phase(self, run, index, phase, outcome): + """Stage the selected model for the next checkpoint publication.""" from sequifier.rl.run_state import weight_fingerprint last = run.distributed.capture_model_state(run.network) @@ -269,9 +273,6 @@ def _publish_phase(self, run, index, phase): artifacts = PhaseArtifacts.for_phase( run.config.phase_directory, index, phase.name ) - artifacts.publish_model( - build_model_artifact(run.network, run.config, state_dict=selected) - ) metric = run.state.best_validation_loss report = phase_report( index=index, @@ -300,26 +301,30 @@ def _publish_phase(self, run, index, phase): stop_scope=stop_scope( phase.stopping, run.state.stopping_state.get("triggered") ), + provenance={"source_provenance": run.dependencies.identity}, + completion_reason=( + outcome.completion_reason + or ( + "early_stopping" + if run.state.stopping_state.get("triggered") + else "completed" + ) + ), + ) + return PhaseCommit.stage( + artifacts, + report, + model=build_model_artifact(run.network, run.config, state_dict=selected), ) - artifacts.publish_report(report) - run.state.phase_reports = [ - r for r in run.state.phase_reports if r["index"] != index - ] + [report] def run(self, run: Any) -> RunResult: - return self._run(run, phase_only=False) - - def run_phase(self, run: Any) -> dict: - """Run one prepared supervised phase for a portable handoff.""" - if len(run.plan.phases) != 1: - raise ValueError("a portable supervised phase requires one resolved phase") - result = self._run(run, phase_only=True) - return { - **run.state.phase_reports[-1], - "completion_reason": result.completion_reason, - } - - def _run(self, run: Any, *, phase_only: bool) -> RunResult: + return self._run(run, group=False) + + def run_group(self, run: Any) -> None: + """Run a native group with its supplied checkpoint publisher.""" + self._run(run, group=True) + + def _run(self, run: Any, *, group: bool) -> RunResult: config = run.config state = run.state phases = run.plan.phases @@ -424,16 +429,30 @@ def flush_metrics( current_batch = sum(state.iterator_positions.values()) current_batches_total = 1 - def run_phase(phase_index, view): + def run_phase(transition): nonlocal access, active_dataset, accumulation_count, last_identity nonlocal boundary, boundary_identity, batch_in_progress, batch_committed nonlocal last_snapshot, last_snapshot_step, last_latest nonlocal current_epoch, current_batch, current_batches_total nonlocal stop_requested, completion_reason + phase_index, view = transition.index, transition.phase phase = view.spec() + if ( + phase_index == state.phase_index + and state.phase_epoch_complete + and ( + state.phase_epoch >= phase.epochs + or state.stopping_state.get("triggered") + ) + ): + return PhaseOutcome( + None, + stop_scope(phase.stopping, state.stopping_state.get("triggered")), + ) + run.dependencies.revalidate() run.metrics.set_phase(phase) if phase_index != state.phase_index: - previous = phases[phase_index - 1].spec() + previous = transition.previous.spec() if previous.handoff == "best": selected = run.distributed.gather_objects( state.best_model_state_dict @@ -504,7 +523,11 @@ def run_phase(phase_index, view): state.phase_epoch = phase_epoch state.phase_epoch_complete = False if run.optimization_phase_index != phase_index: - run.start_phase(view, datasets) + run.start_phase( + view, + datasets, + continue_optimizer=transition.continues_optimizer, + ) state.optimizer_step = run.optimization.optimizer_step state.epoch = current_epoch state.batch = 0 @@ -805,38 +828,58 @@ def run_phase(phase_index, view): completion_reason if stop_requested else None, ) - def commit_phase(index, view, outcome): - self._publish_phase(run, index, view.spec()) + committing = False + + def publish(commit): + if run.distributed.rank == 0: + commit.publish( + PhaseArtifacts(Path(commit.result["model_path"])), commit.result + ) + + def replay(): + if run.distributed.rank == 0: + for values in state.phase_commits: + publish(PhaseCommit.load(values)) + + def commit_phase(transition, outcome): + nonlocal committing + committing = True + commit = self._stage_phase( + run, transition.index, transition.phase.spec(), outcome + ) + commit = run.distributed.gather_objects(commit)[0] + state.phase_commits.append(commit.as_dict()) run.checkpoints.save(CheckpointRequest("latest", last_identity), run) + return commit + + def finish_publication(commit): + nonlocal committing, boundary + publish(commit) + boundary = capture_runtime_boundary(run) + committing = False try: - start = state.phase_index - view = phases[start] - phase = view.spec() - if state.phase_epoch_complete and ( - state.plan_stopped - or state.phase_epoch >= phase.epochs - or state.stopping_state.get("triggered") - ): - # A native checkpoint can precede publication. Rebuild the - # derived artifact on every rank before moving forward. - self._publish_phase(run, start, phase) - start += 1 - if phase_only and start == 0: - execute_phase(0, view, run_phase, commit_phase) - elif not phase_only: - plan_reason = run_phases( - phases, - run_phase, - commit_phase, - start=start, - stopped=state.plan_stopped, - completion_reason="normal_completion", - ) - if plan_reason != "normal_completion": - completion_reason = plan_reason + completion_reason = run_phases( + phases, + run_phase, + commit_phase, + start=len(state.phase_commits), + stopped=state.plan_stopped + and len(state.phase_commits) > state.phase_index, + completion_reason="normal_completion", + replay=replay, + publish=finish_publication, + optimizer_continues=lambda view: view.optimizer_state == "continue", + terminal_reason=( + "integration_requested_stop" + if state.phase_commits + and state.phase_commits[-1]["result"]["completion_reason"] + == "integration_requested_stop" + else None + ), + ) except BaseException: - if update_failed: + if committing or update_failed: run.optimization.optimizer.zero_grad(set_to_none=True) raise if batch_in_progress and not batch_committed: @@ -852,8 +895,7 @@ def commit_phase(index, view, outcome): ) raise - run.checkpoints.save(CheckpointRequest("latest", last_identity), run) - if phase_only: + if group: return RunResult( "early_stopping" if state.stopping_state.get("triggered") @@ -883,12 +925,9 @@ def commit_phase(index, view, outcome): publication, ) if run.distributed.rank == 0: - selected = select_handoff( - phases[state.phase_index].spec().handoff, last_state, best_state - ) + commit = PhaseCommit.load(state.phase_commits[-1]) save_model_artifact( - build_model_artifact(run.network, config, state_dict=selected), - config.model_path, + read_execution_artifact(commit.model_path), config.model_path ) self._manifest(run, result) run.integrations.emit( diff --git a/src/sequifier/training/phase_runners.py b/src/sequifier/training/phase_runners.py index ad73d99c..ec4f4fb6 100644 --- a/src/sequifier/training/phase_runners.py +++ b/src/sequifier/training/phase_runners.py @@ -1,30 +1,27 @@ """Adapt resolved phases to native execution and exact recovery.""" +from collections.abc import Callable from dataclasses import dataclass, replace import torch -from sequifier.artifacts.model_artifact import ModelArtifact, read_execution_artifact +from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.artifacts.phase import PhaseArtifacts from sequifier.runtime.plan import RunIdentity @dataclass(frozen=True) -class PortablePhaseRequest: +class PortableGroup: index: int + phases: tuple incoming: str | None - artifacts: PhaseArtifacts resuming: bool identity: RunIdentity + on_checkpoint: Callable -@dataclass(frozen=True) -class PortablePhaseResult: - artifact: ModelArtifact - report: dict - - -def supervised_phase_config(config, phase, index, incoming, resume): +def supervised_group_config(config, phases, index, incoming, resume): + phase = phases[0] from sequifier.artifacts.model_config import ( extend_prediction_distributions, model_execution_from_model_config, @@ -44,13 +41,17 @@ def supervised_phase_config(config, phase, index, incoming, resume): config.phase_directory, index, phase.name ).model.parent training.resume = ResumeConfig( - policy="if_exists" if resume else "never", + policy="required" if resume else "never", checkpoint_path=str(native / "run.pt"), ) training.log_interval_iterations = 10 - names = {ref.dataset for ref in [*phase.sources, *phase.evaluation_sources]} + names = { + ref.dataset + for item in phases + for ref in [*item.sources, *item.evaluation_sources] + } datasets = { - name: dataset.model_copy(update=phase.learning.datasets[name].model_dump()) + name: dataset for name, dataset in config.dataset_training.items() if name in names } @@ -84,7 +85,7 @@ def supervised_phase_config(config, phase, index, incoming, resume): "model": model, "interfaces": interfaces, "dataset_training": datasets, - "training_plan": [phase.model_copy(deep=True)], + "training_plan": [item.model_copy(deep=True) for item in phases], "runtime_settings": training, "seed": config.seed + index, "model_name": f"{config.model_name}-phase-{index:03d}-{phase.name}", @@ -101,17 +102,15 @@ def supervised_phase_config(config, phase, index, incoming, resume): return resolved -def run_supervised_phase( - config, phase, request: PortablePhaseRequest -) -> PortablePhaseResult: +def run_supervised_group(config, request: PortableGroup, backbone_revision): from sequifier.helpers import configure_determinism, configure_logger from sequifier.integration import IntegrationManager from sequifier.runtime.builder import RunBuilder from sequifier.runtime.context import ExecutionEnvironment from sequifier.training.engine import TrainingEngine - resolved = supervised_phase_config( - config, phase, request.index, request.incoming, request.resuming + resolved = supervised_group_config( + config, request.phases, request.index, request.incoming, request.resuming ) torch.set_float32_matmul_precision( resolved.runtime_settings.float32_matmul_precision @@ -132,27 +131,21 @@ def run_supervised_phase( integrations, initial_artifact=request.incoming, identity=request.identity, + on_checkpoint=request.on_checkpoint, + backbone_revision=backbone_revision, ) try: - report = TrainingEngine().run_phase(run) - return PortablePhaseResult( - read_execution_artifact(report["model_path"]), - { - **report, - "resolved_seed": resolved.seed, - "optimizer_steps": run.state.optimizer_step, - }, - ) + TrainingEngine().run_group(run) finally: run.distributed.finalize() -def policy_phase_config(config, phase, index, incoming, destination, resume): +def policy_group_config(config, phases, index, incoming, destination, resume): from sequifier.config.training_resolution import policy_run_config return policy_run_config( config, - [phase], + phases, incoming=incoming, index=index, destination=destination, @@ -160,20 +153,18 @@ def policy_phase_config(config, phase, index, incoming, destination, resume): ) -def run_portable_policy_phase( - config, phase, request: PortablePhaseRequest -) -> PortablePhaseResult: - from sequifier.rl.learner import run_policy_phase +def run_portable_policy_group(config, request: PortableGroup): + from sequifier.rl.learner import run_policy_group - run = policy_phase_config( + destination = PhaseArtifacts.for_phase( + config.phase_directory, request.index, request.phases[0].name + ).model + run = policy_group_config( config, - phase, + request.phases, request.index, request.incoming, - request.artifacts.model, + destination, request.resuming, ) - result = run_policy_phase(run, request.identity).phases[0] - return PortablePhaseResult( - read_execution_artifact(result.provenance["actor_path"]), result.as_dict() - ) + run_policy_group(run, request.identity, request.on_checkpoint) diff --git a/src/sequifier/training/plan.py b/src/sequifier/training/plan.py index d8114794..b334b97c 100644 --- a/src/sequifier/training/plan.py +++ b/src/sequifier/training/plan.py @@ -51,6 +51,10 @@ def from_spec( _spec=spec.model_copy(deep=True), ) + @property + def optimizer_state(self): + return self._spec.optimizer_state + def spec(self) -> ResolvedTrainingPhase: """Give consumers their own copy of the configuration models.""" return self._spec.model_copy(deep=True) diff --git a/src/sequifier/training/state.py b/src/sequifier/training/state.py index bed0980b..029bd599 100644 --- a/src/sequifier/training/state.py +++ b/src/sequifier/training/state.py @@ -21,7 +21,7 @@ class RunState: run_id: str = field(default_factory=new_run_id) session_id: str = field(default_factory=new_run_id) stopping_state: dict[str, Any] = field(default_factory=dict) - phase_reports: list[dict[str, Any]] = field(default_factory=list) + phase_commits: list[dict[str, Any]] = field(default_factory=list) last_validation_metric: float | None = None best_optimizer_step: int | None = None plan_stopped: bool = False From 8086fc2fa7ccf4bd6bd717f131e37fded3cfbc47 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 18:54:47 +0100 Subject: [PATCH 58/71] typed execution --- src/sequifier/api.py | 2 + src/sequifier/artifacts/dependencies.py | 12 +- src/sequifier/artifacts/loading.py | 6 +- src/sequifier/artifacts/model_artifact.py | 13 +- src/sequifier/artifacts/model_config.py | 93 ++- src/sequifier/artifacts/model_export.py | 15 +- src/sequifier/artifacts/run_checkpoint.py | 127 +++- .../canonical_hyperparameter_search_config.py | 2 +- src/sequifier/config/infer_config.py | 125 ++-- .../config/input_contract_resolution.py | 11 +- src/sequifier/config/policy_infer_config.py | 13 +- src/sequifier/config/policy_train_config.py | 5 +- src/sequifier/config/run_capabilities.py | 69 +++ src/sequifier/config/train_config.py | 549 +++++------------- src/sequifier/config/training_resolution.py | 527 +++++++++-------- src/sequifier/evaluation/compatibility.py | 8 +- src/sequifier/export/preflight.py | 22 +- src/sequifier/export/service.py | 33 +- src/sequifier/model/execution.py | 24 +- src/sequifier/model/execution_options.py | 80 +++ src/sequifier/model/factory.py | 34 +- src/sequifier/model/history_transfer.py | 5 +- src/sequifier/rl/attachment.py | 29 +- src/sequifier/rl/learner.py | 50 +- src/sequifier/rl/policy.py | 25 - src/sequifier/rl/preparation.py | 51 +- src/sequifier/rl/references.py | 2 +- src/sequifier/rl/run_config.py | 5 +- src/sequifier/rl/run_state.py | 4 +- src/sequifier/runtime/builder.py | 11 +- src/sequifier/train.py | 2 +- src/sequifier/training/checkpoint_service.py | 2 + src/sequifier/training/coordinator.py | 28 +- src/sequifier/training/phase_runners.py | 63 +- src/sequifier/training/plan.py | 8 +- src/sequifier/training/runtime.py | 28 +- src/sequifier/training/scheduler_budgets.py | 4 +- 37 files changed, 1033 insertions(+), 1054 deletions(-) create mode 100644 src/sequifier/config/run_capabilities.py create mode 100644 src/sequifier/model/execution_options.py diff --git a/src/sequifier/api.py b/src/sequifier/api.py index 9443c1b8..cb156bb2 100644 --- a/src/sequifier/api.py +++ b/src/sequifier/api.py @@ -13,6 +13,7 @@ canonicalize_state_dict, ) from sequifier.model.execution import ModelExecution +from sequifier.model.execution_options import ModelExecutionOptions from sequifier.model.factory import build_transformer_network from sequifier.model.network import ( ComposableTransformerNetwork, @@ -43,6 +44,7 @@ "ModelArtifactMetadata", "ModelExecution", "ModelExecutionConfig", + "ModelExecutionOptions", "ModelInterfaceModule", "ModelOutput", "ParameterCatalog", diff --git a/src/sequifier/artifacts/dependencies.py b/src/sequifier/artifacts/dependencies.py index baab8181..49dca3d5 100644 --- a/src/sequifier/artifacts/dependencies.py +++ b/src/sequifier/artifacts/dependencies.py @@ -101,16 +101,6 @@ def add_initialization(manifest, config, incoming=None): manifest.add(Path(config.project_root) / path) -def model_metadata_paths(config, interface_name=None): - interfaces = (config.ownership.get("model") or {}).get("interfaces", {}) - return [ - str(Path(config.project_root) / path) - for name, interface in interfaces.items() - if (interface_name is None or name == interface_name) - and (path := interface.get("input_contract", {}).get("metadata_config_path")) - ] - - def _add_loader_media(manifest, view): from sequifier.io.images import media_files @@ -137,7 +127,7 @@ def supervised_dependencies(config, incoming=None): manifest = DependencyManifest() root = Path(config.project_root) add_initialization(manifest, config, incoming) - for path in model_metadata_paths(config): + for path in config.model_metadata_paths.values(): manifest.add(path) for name, dataset in config.dataset_training.items(): for part_name, part in dataset.parts.items(): diff --git a/src/sequifier/artifacts/loading.py b/src/sequifier/artifacts/loading.py index 053b84a7..60ae665e 100644 --- a/src/sequifier/artifacts/loading.py +++ b/src/sequifier/artifacts/loading.py @@ -10,7 +10,7 @@ from torch import nn from sequifier.artifacts.model_artifact import read_artifact, restore_model_artifact -from sequifier.artifacts.model_config import resolved_training_config +from sequifier.model.execution import ModelExecution from sequifier.model.parameter_catalog import ParameterCatalog @@ -26,7 +26,7 @@ class ExecutionOptions: @dataclass class LoadedModel: network: nn.Module - config: Any + config: ModelExecution parameter_catalog: ParameterCatalog artifact_metadata: dict[str, Any] @@ -62,7 +62,7 @@ def load_model_for_analysis( network = torch.compile(network) return LoadedModel( network=network, - config=resolved_training_config(execution), + config=execution, parameter_catalog=catalog, artifact_metadata={ "path": str(artifact_path), diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index a29fcb63..3efeb27b 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -11,7 +11,6 @@ from sequifier.artifacts.model_config import ( model_execution_from_model_config, - resolved_training_config, validate_interface_versions, ) from sequifier.artifacts.model_export import model_execution_config @@ -133,14 +132,13 @@ def load_model_artifact( *, device: str = "cpu", interface_name: str | None = None, -) -> tuple[Any, Any, ModelArtifact]: - """Load a portable artifact and return network, resolved config, and schema.""" +) -> tuple[Any, ModelExecution, ModelArtifact]: + """Load a portable artifact and return its network and execution contract.""" artifact, _ = read_artifact(path, expected_type="sequifier_model") - network, execution, artifact = restore_model_artifact( + return restore_model_artifact( artifact, device=device, interface_name=interface_name ) - return network, resolved_training_config(execution), artifact def restore_model_artifact( @@ -221,10 +219,9 @@ def load_weights_from_run_checkpoint( *, device: str = "cpu", interface_name: str | None = None, -) -> tuple[Any, Any, ModelArtifact]: - network, execution, artifact = restore_model_artifact( +) -> tuple[Any, ModelExecution, ModelArtifact]: + return restore_model_artifact( model_artifact_from_run_checkpoint(path), device=device, interface_name=interface_name, ) - return network, resolved_training_config(execution), artifact diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 3b81f778..667e6be4 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -4,33 +4,42 @@ import json +from sequifier.config.ownership import ExportSettings from sequifier.config.policy_config import ResolvedPolicyBinding +from sequifier.config.run_capabilities import PublicationPlan from sequifier.config.train_config import ( BackboneComponentConfig, ModelInterfaceSpecModel, ModelSpecModel, + ResolvedInterfaceStorage, ResolvedModelInterface, ResolvedSequifierConfig, TrainingRuntimeSettings, ) from sequifier.contract_versions import ContractVersions, require_contract_version -from sequifier.helpers import ModelWindowView, StoredWindowLayout, resolve_window_view +from sequifier.helpers import ModelWindowView, resolve_window_view from sequifier.model.execution import ModelExecution +from sequifier.model.execution_options import ( + DEFAULT_MAX_CONCAT_TENSORS, + ModelExecutionOptions, +) from sequifier.model.execution_schema import ExecutionSchema +from sequifier.objectives import target_offset_for_objective from sequifier.special_tokens import SPECIAL_TOKEN_IDS from sequifier.typechecking import beartype -def validate_execution_attachment(source: dict, current: dict) -> None: +def validate_execution_attachment( + source: dict, current_execution: ModelExecution +) -> None: """Require existing prediction semantics when reusing complete model weights.""" from pydantic_core import to_jsonable_python from sequifier.artifacts.model_export import model_execution_config source_config, _ = model_execution_from_model_config(source, device="cpu") - current_config, _ = model_execution_from_model_config(current, device="cpu") source = model_execution_config(source_config) - current = model_execution_config(current_config) + current = model_execution_config(current_execution) if source["backbone"]["architecture"] != current["backbone"]["architecture"]: raise ValueError("model attachment cannot change backbone architecture") if source.get("layer_type_dtypes") != current.get("layer_type_dtypes"): @@ -38,12 +47,7 @@ def validate_execution_attachment(source: dict, current: dict) -> None: if source["context_length"] != current["context_length"]: raise ValueError("model attachment cannot change context length") - def attention(values): - return values.get("forward_attention") or ( - "bidirectional" if values["training_objective"] == "bert" else "causal" - ) - - if attention(source) != attention(current): + if source["forward_attention"] != current["forward_attention"]: raise ValueError("model attachment cannot change forward attention") for name, interface in source["interfaces"].items(): if name not in current["interfaces"]: @@ -102,8 +106,8 @@ def resolved_config_from_model_config( return resolved_training_config(execution), selected -def validate_interface_storage_versions(interface: dict) -> None: - storage = interface.get("storage_layout") +def validate_interface_storage_versions(values: dict, depth_layouts=None) -> None: + storage = values.get("storage_layout") require_contract_version( "stored-window layout", storage.get("version") if isinstance(storage, dict) else None, @@ -113,7 +117,7 @@ def validate_interface_storage_versions(interface: dict) -> None: # can still refer to a dataset that uses the depth metadata version. expected = ( ContractVersions.DEPTH_TENSOR_METADATA - if interface.get("depth_layouts") + if (values.get("depth_layouts") if depth_layouts is None else depth_layouts) else ( ContractVersions.DATASET_TENSOR_METADATA, ContractVersions.DEPTH_TENSOR_METADATA, @@ -121,13 +125,30 @@ def validate_interface_storage_versions(interface: dict) -> None: ) require_contract_version( "model dataset tensor metadata", - interface.get("tensor_payload_version"), + values.get("tensor_payload_version"), expected, ) +def split_interface_storage( + values: dict, +) -> tuple[dict, ResolvedInterfaceStorage | None]: + """Read the flat storage fields used by existing artifact and checkpoint formats.""" + if not ResolvedInterfaceStorage.model_fields.keys() & values.keys(): + return values, None + validate_interface_storage_versions(values) + model_values = dict(values) + storage = { + name: model_values.pop(name) + for name in ResolvedInterfaceStorage.model_fields + if name in model_values + } + return model_values, ResolvedInterfaceStorage.model_validate(storage) + + def validate_interface_versions(interface: dict) -> None: - validate_interface_storage_versions(interface) + if ResolvedInterfaceStorage.model_fields.keys() & interface.keys(): + validate_interface_storage_versions(interface) schema = interface.get("execution_schema") require_contract_version( "model execution schema", @@ -159,21 +180,21 @@ def model_execution_from_model_config( context_length = int(values["context_length"]) target_offset = int(values.get("target_offset", 1)) objective = str(values["training_objective"]) - global_spec = TrainingRuntimeSettings( + options = ModelExecutionOptions( training_objective=objective, forward_attention=values.get("forward_attention"), context_length=context_length, target_offset=target_offset, - inference_batch_size=1, layer_type_dtypes=values.get("layer_type_dtypes"), - torch_compile="none", + max_concat_tensors=values.get("max_concat_tensors", DEFAULT_MAX_CONCAT_TENSORS), ) authored_interfaces = {} resolved_interfaces = {} + interface_storage = {} window_view = ModelWindowView( context_length=context_length, objective=objective, - target_offset=(0 if objective == "bert" else target_offset), + target_offset=target_offset_for_objective(objective, target_offset), ) interface_order = [interface_name] + [ name for name in interface_values if name != interface_name @@ -181,8 +202,10 @@ def model_execution_from_model_config( for name in interface_order: interface = interface_values[name] validate_interface_versions(interface) - storage_layout = StoredWindowLayout(**interface["storage_layout"]) - resolve_window_view(storage_layout, window_view) + interface, storage = split_interface_storage(interface) + if storage is not None: + resolve_window_view(storage.storage_layout, window_view) + interface_storage[name] = storage authored = { key: value for key, value in interface.items() @@ -219,7 +242,6 @@ def model_execution_from_model_config( "categorical_decoder_special_tokens": interface.get( "categorical_decoder_special_tokens", {} ), - "storage_layout": storage_layout, "window_view": window_view, } ) @@ -234,8 +256,9 @@ def model_execution_from_model_config( config = ModelExecution( ModelSpecModel(backbone=backbone, interfaces=authored_interfaces), resolved_interfaces, - global_spec, + options, device=device, + interface_storage=interface_storage, embedding_layer_names=values.get( "embedding_layer_names", ["backbone.final_norm"] ), @@ -246,7 +269,13 @@ def model_execution_from_model_config( def execution_config(model, interfaces, runtime_settings, *, device="cpu", **options): """Adapt model execution to the existing public resolved training contract.""" return resolved_training_config( - ModelExecution(model, interfaces, runtime_settings, device=device), **options + ModelExecution( + model, + interfaces, + ModelExecutionOptions.from_training(runtime_settings), + device=device, + ), + **options, ) @@ -259,16 +288,20 @@ def resolved_training_config( "model_name": execution.model_name, "seed": execution.seed, "device": execution.device, - "runtime_settings": execution.runtime_settings, + "runtime_settings": TrainingRuntimeSettings( + **execution.options.model_dump(), torch_compile="none" + ), + "model_options": execution.options, "model": execution.model, "interfaces": execution.interfaces, + "interface_storage": execution.interface_storage, "dataset_training": {}, "training_plan": [], - "export_generative_model": True, - "export_embedding_model": False, - "export_onnx": False, - "export_pt": False, - "embedding_layer_names": list(execution.embedding_layer_names), + "publication": PublicationPlan( + exports=ExportSettings( + pt=False, embedding_layer_names=execution.embedding_layer_names + ) + ), } return ResolvedSequifierConfig(**{**values, **options}) diff --git a/src/sequifier/artifacts/model_export.py b/src/sequifier/artifacts/model_export.py index dcaad2c4..03dbb017 100644 --- a/src/sequifier/artifacts/model_export.py +++ b/src/sequifier/artifacts/model_export.py @@ -1,16 +1,17 @@ from typing import Any from sequifier.model.execution import ModelExecution +from sequifier.model.execution_options import DEFAULT_MAX_CONCAT_TENSORS from sequifier.model.execution_schema import ExecutionSchema from sequifier.typechecking import beartype @beartype def model_execution_config(config: Any) -> dict[str, Any]: - """Return only configuration required to reconstruct model execution.""" + """Serialize the execution contract and its optional inference storage metadata.""" execution = ModelExecution.from_config(config) - spec = execution.runtime_settings + spec = execution.options interfaces = {} for name, interface in execution.interfaces.items(): values = interface.model_dump( @@ -27,13 +28,19 @@ def model_execution_config(config: Any) -> dict[str, Any]: interface, spec.context_length ).to_dict() values["depth_layouts"] = interface.depth_layouts.model_dump(mode="json") + if (storage := execution.interface_storage.get(name)) is not None: + values.update(storage.model_dump(mode="python")) interfaces[name] = values if interface.policy is not None: values["policy"] = interface.policy.model_dump(mode="json") return { "training_objective": spec.training_objective, - "forward_attention": spec.forward_attention - or ("bidirectional" if spec.training_objective == "bert" else "causal"), + "forward_attention": spec.attention, + **( + {"max_concat_tensors": spec.max_concat_tensors} + if spec.max_concat_tensors != DEFAULT_MAX_CONCAT_TENSORS + else {} + ), "context_length": spec.context_length, "target_offset": spec.target_offset, "backbone": execution.model.backbone.model_dump( diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 5d70b977..3a19f974 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping +from copy import deepcopy from dataclasses import asdict, dataclass from pathlib import Path from typing import Any @@ -14,6 +15,119 @@ from sequifier.io.atomic import atomic_save +def read_training_plan(saved: dict): + """Translate legacy embedded interface views at the recovery boundary.""" + from sequifier.artifacts.model_config import ( + split_interface_storage, + validate_interface_storage_versions, + ) + from sequifier.config.metadata import validate_persisted_metadata_versions + from sequifier.config.run_capabilities import PublicationPlan + from sequifier.config.train_config import ( + ResolvedSequifierConfig, + TrainingRuntimeSettings, + ) + from sequifier.model.execution_options import ModelExecutionOptions + + values = deepcopy(saved) + interfaces = values["interfaces"] + storage = values.setdefault("interface_storage", {}) + for name, interface in interfaces.items(): + model_interface, legacy_storage = split_interface_storage(interface) + interfaces[name] = model_interface + if legacy_storage is not None: + storage.setdefault(name, legacy_storage.model_dump()) + if name in storage: + validate_interface_storage_versions( + storage[name], model_interface.get("depth_layouts") or {} + ) + datasets = values["dataset_training"] + for dataset in datasets.values(): + if (legacy_view := dataset.pop("interface", None)) is not None: + _, legacy_storage = split_interface_storage(legacy_view) + if legacy_storage is not None: + dataset.setdefault("storage", legacy_storage.model_dump()) + if (dataset_storage := dataset.get("storage")) is not None: + interface = interfaces[dataset["model_interface"]] + validate_interface_storage_versions( + dataset_storage, interface.get("depth_layouts") or {} + ) + for part in dataset["parts"].values(): + validate_persisted_metadata_versions(part["metadata"]) + + if "model_metadata_paths" not in values: + authored_interfaces = (values.get("ownership", {}).get("model") or {}).get( + "interfaces", {} + ) + values["model_metadata_paths"] = { + name: str(Path(values["project_root"]) / path) + for name, spec in authored_interfaces.items() + if (path := spec.get("input_contract", {}).get("metadata_config_path")) + } + if "model_options" not in values: + values["model_options"] = ModelExecutionOptions.from_training( + TrainingRuntimeSettings.model_validate(values["runtime_settings"]) + ) + if "publication" not in values: + values["publication"] = PublicationPlan( + exports={ + "generative_model": values.pop("export_generative_model"), + "embedding_model": values.pop("export_embedding_model"), + "embedding_layer_names": values.pop("embedding_layer_names"), + "onnx": values.pop("export_onnx"), + "pt": values.pop("export_pt"), + "with_dropout": values.pop("export_with_dropout", False), + "batch_size": values["runtime_settings"].get("inference_batch_size", 1), + } + ) + for phase in values["training_plan"]: + if phase["algorithm"] == "supervised": + for source in phase["sources"] + phase.get("evaluation_sources", []): + if "parts" not in source: + source["parts"] = ( + [source["part"]] + if source.get("part") is not None + else list(datasets[source["dataset"]]["parts"]) + ) + return ResolvedSequifierConfig.model_validate(values) + + +def training_plan_identity(config): + """Retain the flat fingerprint contract used by existing coordinated runs.""" + from sequifier.fingerprints import fingerprint + + values = config.model_dump(mode="json") + values.pop("model_options") + values.pop("model_metadata_paths") + storage = values.pop("interface_storage") + for name, interface in values["interfaces"].items(): + interface.update(storage.get(name, {})) + for name, dataset in values["dataset_training"].items(): + dataset["interface"] = { + **config.dataset_interface(name).model_dump(mode="json"), + **config.dataset_storage(name).model_dump(mode="json"), + } + dataset.pop("storage") + exports = values.pop("publication")["exports"] + values.update( + export_generative_model=exports["generative_model"], + export_embedding_model=exports["embedding_model"], + embedding_layer_names=exports["embedding_layer_names"], + export_onnx=exports["onnx"], + export_pt=exports["pt"], + export_with_dropout=exports["with_dropout"], + ) + if values["actor_initialization"] == "attach": + values.pop("actor_initialization") + for phase in values["training_plan"]: + if phase["algorithm"] == "supervised": + for source in phase["sources"] + phase["evaluation_sources"]: + source.pop("parts") + values["runtime_settings"].pop("resume", None) + values.get("ownership", {}).get("run", {}).get("recovery", {}).pop("resume", None) + return fingerprint(values) + + @dataclass(frozen=True) class OptimizationState: optimizer: dict[str, Any] @@ -116,9 +230,6 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": raise ValueError("Expected a supervised checkpoint recovery payload") if payload.get("recovery_type") != "supervised": raise ValueError("Checkpoint belongs to another training runner") - from sequifier.artifacts.model_config import validate_interface_storage_versions - from sequifier.config.metadata import validate_persisted_metadata_versions - from sequifier.config.train_config import ResolvedSequifierConfig require_contract_version( "initialization policy", @@ -131,15 +242,7 @@ def from_state_dict(cls, payload: Mapping[str, Any]) -> "RunCheckpoint": ContractVersions.INITIALIZATION_SEED_DERIVATION, ) model = ModelArtifact.from_state_dict(payload["model"]) - saved_config = payload["training_config"] - for interface in saved_config["interfaces"].values(): - validate_interface_storage_versions(interface) - for dataset in saved_config["dataset_training"].values(): - if "interface" in dataset: - validate_interface_storage_versions(dataset["interface"]) - for part in dataset["parts"].values(): - validate_persisted_metadata_versions(part["metadata"]) - training_config = ResolvedSequifierConfig.model_validate(saved_config) + training_config = read_training_plan(payload["training_config"]) loader_parts = payload["loader_state"]["parts"] checkpoint = cls( format_version=payload["format_version"], diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index 8754ca77..fb0b2e70 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -29,7 +29,7 @@ def _validate_training_candidate(values: dict[str, Any]) -> SequifierConfig: from sequifier.config.train_config import validate_training_contract candidate = SequifierConfig.model_validate(values) - validate_training_contract(candidate) + validate_training_contract(candidate, candidate.training_plan.phases) is_next = candidate.runtime_settings.training_objective == "next_occurrence" for phase in candidate.training_plan.phases: if phase.algorithm != "supervised": diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index 2c97507f..e9f02685 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -131,64 +131,72 @@ def _merge_loaded_execution_values( values[field] = loaded_value -@beartype -def _execution_source( - config: Any, interface: Any -) -> tuple[dict[str, Any], DatasetMetadata]: - global_spec = config.runtime_settings - layout = interface.storage_layout - return ( - { - "model_interface": interface.name, - "input_columns": list(interface.input_columns), - "static_columns": ( +def _execution_values(settings: Any, interface: Any) -> dict[str, Any]: + return { + "model_interface": interface.name, + "input_columns": list(interface.input_columns), + "static_columns": ( + list(interface.static_conditioning.columns) + if interface.static_conditioning + else [] + ), + "target_columns": list(interface.target_columns), + "column_data_types": dict(interface.column_data_types), + "target_column_types": dict(interface.target_column_types), + "training_objective": settings.training_objective, + "context_length": settings.context_length, + "target_offset": settings.target_offset, + "prediction_length": interface.decoder.prediction_length, + "decoder_support_length": interface.decoder.support_length, + } + + +def _execution_metadata(interface: Any, storage: Any) -> DatasetMetadata: + if storage is None: + raise ValueError(f"{interface.name}: stored-window metadata is required") + layout = storage.storage_layout + return DatasetMetadata.model_validate( + dict( + depth_layouts=interface.depth_layouts, + tensor_payload_version=storage.tensor_payload_version, + split_paths=storage.split_paths, + column_data_types=dict(interface.column_data_types), + static_columns=( list(interface.static_conditioning.columns) if interface.static_conditioning else [] ), - "target_columns": list(interface.target_columns), - "column_data_types": dict(interface.column_data_types), - "target_column_types": dict(interface.target_column_types), - "training_objective": global_spec.training_objective, - "context_length": global_spec.context_length, - "target_offset": global_spec.target_offset, - "prediction_length": interface.decoder.prediction_length, - "decoder_support_length": interface.decoder.support_length, - }, - DatasetMetadata.model_validate( - dict( - depth_layouts=interface.depth_layouts, - tensor_payload_version=interface.tensor_payload_version, - split_paths=interface.split_paths, - column_data_types=dict(interface.column_data_types), - static_columns=( - list(interface.static_conditioning.columns) - if interface.static_conditioning - else [] - ), - static_features_path=interface.static_features_path, - image_sources=interface.image_sources, - preconditioning_image=interface.preconditioning_image, - n_classes=dict(interface.n_classes), - id_maps=dict(interface.id_maps), - special_token_ids=dict(interface.special_token_ids), - selected_columns_statistics=dict(interface.selected_columns_statistics), - normalize_real_columns=interface.normalize_real_columns, - prediction_aligned_splits=interface.prediction_aligned_splits, - prediction_alignment_steps=interface.prediction_alignment_steps, - window_lengths=interface.window_lengths, - prediction_length=interface.prediction_length, - target_offset=interface.target_offset, - window_length=( - interface.window_lengths[0] - if interface.window_lengths - else layout.window_length - ), - max_target_offset=layout.max_target_offset, - stored_window_layout_version=layout.version, + static_features_path=storage.static_features_path, + image_sources=interface.image_sources, + preconditioning_image=interface.preconditioning_image, + n_classes=dict(interface.n_classes), + id_maps=dict(interface.id_maps), + special_token_ids=dict(interface.special_token_ids), + selected_columns_statistics=dict(interface.selected_columns_statistics), + normalize_real_columns=interface.normalize_real_columns, + prediction_aligned_splits=storage.prediction_aligned_splits, + prediction_alignment_steps=storage.prediction_alignment_steps, + window_lengths=storage.window_lengths, + prediction_length=storage.prediction_length, + target_offset=storage.target_offset, + window_length=( + storage.window_lengths[0] + if storage.window_lengths + else layout.window_length ), - context={"projected_depth_layouts": True}, + max_target_offset=layout.max_target_offset, + stored_window_layout_version=layout.version, ), + context={"projected_depth_layouts": True}, + ) + + +@beartype +def _execution_source( + settings: Any, interface: Any, storage: Any +) -> tuple[dict[str, Any], DatasetMetadata]: + return _execution_values(settings, interface), _execution_metadata( + interface, storage ) @@ -454,23 +462,28 @@ def load_inferer_config( if dataset_name not in datasets: raise ValueError(f"Unknown inference dataset {dataset_name!r}") interface = training.dataset_interface(dataset_name) + storage = training.dataset_storage(dataset_name) source = f"training config dataset {dataset_name!r}" elif authored_values.get("model_interface") is not None: interface_name = authored_values["model_interface"] if interface_name not in training.interfaces: raise ValueError(f"Unknown model interface {interface_name!r}") interface = training.interface(interface_name) + storage = training.interface_storage[interface_name] source = f"training config interface {interface_name!r}" elif len(datasets) == 1: name = next(iter(datasets)) interface = training.dataset_interface(name) + storage = training.dataset_storage(name) source = f"training config dataset {name!r}" else: raise ValueError( "A dataset or model_interface selection is required for the " "training config" ) - loaded_values, training_metadata = _execution_source(training, interface) + loaded_values, training_metadata = _execution_source( + training.model_options, interface, storage + ) image_selections.append( ( image_views(interface.ingestion), @@ -531,7 +544,11 @@ def load_inferer_config( if branch.type == "autoregressive_transformer" for target in branch.target_columns ) - loaded_values, artifact_metadata = _execution_source(training, interface) + loaded_values, artifact_metadata = _execution_source( + training.options, + interface, + training.interface_storage.get(interface_name), + ) image_selections.append( ( image_views(interface.ingestion), diff --git a/src/sequifier/config/input_contract_resolution.py b/src/sequifier/config/input_contract_resolution.py index 2a76ef48..eeb1868f 100644 --- a/src/sequifier/config/input_contract_resolution.py +++ b/src/sequifier/config/input_contract_resolution.py @@ -1,12 +1,9 @@ """Assert authored interface facts against frozen artifact/source metadata.""" -from pathlib import Path - -def validate_input_contracts(config, resolved): +def validate_input_contracts(config, resolved, declarations): if config.model is None: return - from sequifier.config.metadata import load_dataset_metadata from sequifier.config.train_config import _assert_compatible, _part_signature for name, spec in config.model.interfaces.items(): @@ -16,10 +13,8 @@ def validate_input_contracts(config, resolved): for value in resolved.dataset_training.values() if value.model_interface == name ] - if datasets and declaration.metadata_config_path: - expected = load_dataset_metadata( - str(Path(config.run.project_root) / declaration.metadata_config_path) - ) + if datasets and name in declarations: + expected = declarations[name] for dataset in datasets: for part in dataset.parts.values(): _assert_compatible( diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index 74213fe4..da207b7e 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -111,7 +111,8 @@ def policy_inference_config( route = InferenceRouteConfig.model_validate(values) from sequifier.config.infer_config import ( _assert_metadata_matches_source, - _execution_source, + _execution_metadata, + _execution_values, ) from sequifier.config.metadata import load_dataset_metadata @@ -122,7 +123,8 @@ def policy_inference_config( model_path, selected = selection.path, selection.interface_name request = PolicyInferenceConfig.model_validate(values) execution = selection.resolve(request.device) - policy = execution.interface(selected).policy + interface = execution.interface(selected) + policy = interface.policy if policy is None: raise ValueError("selected interface has no policy") if ( @@ -143,7 +145,7 @@ def policy_inference_config( if field.type == "categorical" ): raise ValueError("action labels must have unique probability column names") - contract, metadata = _execution_source(execution, execution.interface(selected)) + contract = _execution_values(execution.options, interface) for key, actual in contract.items(): if ( key in request.model_fields_set @@ -157,10 +159,9 @@ def policy_inference_config( ) _assert_metadata_matches_source( explicit, - metadata, + _execution_metadata(interface, execution.interface_storage.get(selected)), "policy metadata", - "storage_layout" - in artifacts[model_path].model_config.values["interfaces"][selected], + layout_is_authoritative=True, ) authored = request.model_dump() authored["run"] = request.run.model_dump(mode="python", exclude_unset=True) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 3a38c5c7..e5a00026 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -80,7 +80,7 @@ def validate_location(self): def resolve_sequence_source( - source: SequenceSourceConfig, root: Path + source: SequenceSourceConfig, root: Path, *, metadata=None ) -> SequenceSourceConfig: values = source.model_dump() values["paths"] = [str((root / path).resolve()) for path in source.paths] @@ -92,7 +92,8 @@ def resolve_sequence_source( ): from sequifier.config.metadata import load_dataset_metadata - metadata = load_dataset_metadata(values["metadata_config_path"]) + if metadata is None: + metadata = load_dataset_metadata(values["metadata_config_path"]) prepared = ( source.format == "prepared" or source.split_index is not None diff --git a/src/sequifier/config/run_capabilities.py b/src/sequifier/config/run_capabilities.py new file mode 100644 index 00000000..8e77952d --- /dev/null +++ b/src/sequifier/config/run_capabilities.py @@ -0,0 +1,69 @@ +"""Runner and publication capabilities checked when compiling a training plan.""" + +import os +from typing import Literal + +from pydantic import Field + +from sequifier.config.naming import ConfigModel +from sequifier.config.ownership import ExportSettings, RunSettings + +RunMode = Literal["supervised", "policy", "coordinated"] + + +class PublicationPlan(ConfigModel): + mode: RunMode = "supervised" + exports: ExportSettings = Field(default_factory=ExportSettings) + + +def validate_runner_settings( + run: RunSettings, *, supervised: bool, policy: bool +) -> None: + execution = run.execution + if policy: + # These settings are consumed only by the supervised runner or finalizer. + supervised_only = ( + ( + "run.execution", + execution, + {"distribution", "compilation", "precision", "max_concat_tensors"}, + ), + ( + "run.execution.data_loading", + execution.data_loading, + {"num_workers", "load_full_data_to_ram", "max_ram_gb"}, + ), + ( + "run.export", + run.export, + { + "onnx", + "embedding_model", + "with_dropout", + "generative_model", + "pt", + "batch_size", + }, + ), + ) + for path, settings, fields in supervised_only: + if explicit := fields & settings.model_fields_set: + raise ValueError( + f"policy phases do not support {path} settings: {sorted(explicit)}" + ) + if not supervised and "log_interval_batches" in run.logging.model_fields_set: + raise ValueError("policy phases do not support batch logging intervals") + elif ( + not (run.export.generative_model or run.export.embedding_model) + and os.getenv("SEQUIFIER_PREVENT_EXPORT") is None + ): + raise ValueError("At least one model export must be enabled") + + +def publication_plan(run: RunSettings, phases) -> PublicationPlan: + policy = [phase for phase in phases if phase.algorithm != "supervised"] + if not policy: + return PublicationPlan(mode="supervised", exports=run.export) + if len(policy) == len(phases) and len({p.model_interface for p in policy}) == 1: + return PublicationPlan(mode="policy", exports=run.export) + return PublicationPlan(mode="coordinated", exports=run.export) diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 465076fb..baf69b92 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -9,7 +9,6 @@ import copy import inspect import os -import warnings from dataclasses import dataclass from pathlib import Path from types import SimpleNamespace @@ -70,6 +69,7 @@ ) from sequifier.config.policy_train_config import SequenceSourceConfig from sequifier.config.prediction_config import PredictionDistributionConfig +from sequifier.config.run_capabilities import PublicationPlan from sequifier.config.sources import TrainingSourceSpecModel, phase_sources from sequifier.config.sources import validate_source as _validate_source @@ -83,9 +83,8 @@ resolve_window_view, ) from sequifier.model.embedding import validate_embedding_layer_names +from sequifier.model.execution_options import ExecutionOptions, ModelExecutionOptions from sequifier.objectives import ( - ALLOWED_OBJECTIVE_NAMES, - OBJECTIVE_NAME_MESSAGE, BERTObjective, NextOccurrenceObjective, get_objective_class, @@ -126,22 +125,17 @@ def _unique_columns(value: list[str], usage: str) -> list[str]: return value -class TrainingRuntimeSettings(ConfigModel): +class TrainingRuntimeSettings(ExecutionOptions): """Internal native execution projection; never an authored ownership surface.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") episode_cache_size: int = Field(default=8, gt=0) - training_objective: str | None = None - forward_attention: Literal["causal", "bidirectional"] | None = None - context_length: int | None = Field(default=None, gt=0) - target_offset: int = Field(default=1, ge=0) window_stride: Optional[int] = Field(default=None, gt=0) inference_batch_size: int = Field(default=1, gt=0) bert_spec: Optional[BERTSpecModel] = None next_occurrence_config: Optional[NextOccurrenceConfigModel] = None - max_concat_tensors: int = Field(default=12, gt=0) log_interval_batches: int = Field(default=10, gt=0) log_interval_iterations: int = Field(default=10, gt=0) latest_checkpoint_interval_minutes: Optional[float] = None @@ -153,44 +147,12 @@ class TrainingRuntimeSettings(ConfigModel): world_size: int = Field(default=1, gt=0) num_workers: int = Field(default=0, ge=0) backend: str = "nccl" - layer_type_dtypes: Optional[dict[str, str]] = None layer_autocast: bool = False data_parallelism: Optional[Literal["ddp", "fsdp"]] = None fsdp_cpu_offload: Optional[bool] = None torch_compile: Literal["outer", "inner", "none"] = "outer" float32_matmul_precision: Literal["highest", "high", "medium"] = "highest" - @field_validator("training_objective") - @classmethod - @beartype - def validate_objective(cls, value: str) -> str: - if value not in ALLOWED_OBJECTIVE_NAMES: - raise ValueError( - f"Only {OBJECTIVE_NAME_MESSAGE} are allowed, found {value}" - ) - return value - - @field_validator("layer_type_dtypes") - @classmethod - @beartype - def validate_layer_dtypes(cls, value: Optional[dict[str, str]]): - if value is None: - return value - allowed_keys = {"embedding", "linear", "conv", "norm", "decoder"} - allowed_types = { - "float32", - "float16", - "bfloat16", - "float64", - "float8_e4m3fn", - "float8_e5m2", - } - if invalid := set(value) - allowed_keys: - raise ValueError(f"Invalid layer_type_dtypes keys: {sorted(invalid)}") - if invalid := set(value.values()) - allowed_types: - raise ValueError(f"Invalid layer_type_dtypes values: {sorted(invalid)}") - return value - @model_validator(mode="after") @beartype def validate_distribution(self): @@ -579,24 +541,25 @@ def validate_unique_names(self): return self -def validate_training_contract(config): - if config.model is None and config.initialize_from is None: - raise ValueError("a run requires model or initialize_from") - for phase in config.training_plan.phases: +def validate_training_contract(config, phases): + """Check model and phase requirements without reading training data.""" + for phase in phases: evaluation = phase.evaluation - refs = phase_sources(phase) - for ref in refs: - _validate_source(config, ref.source, f"phase {phase.name!r}") - if phase.algorithm != "supervised" and ( + if phase.algorithm != "supervised": + if any( ref.selection_weight is not None or ref.batches_per_selection is not None + for ref in phase_sources(phase) ): raise ValueError( "policy source references cannot configure supervised interleaving" ) - if phase.algorithm != "supervised": continue - role = phase.optimization_configs()["model"] + role = ( + phase.optimization_configs()["model"] + if isinstance(phase, TrainingPhaseSpecModel) + else phase.optimization["model"] + ) needs_monitor = ( phase.handoff == "best" or role.scheduler.name == "ReduceLROnPlateau" @@ -613,14 +576,9 @@ def validate_training_contract(config): ): raise ValueError("ReduceLROnPlateau requires evaluation_interval_epochs: 1") - if any(phase.algorithm != "supervised" for phase in config.training_plan.phases): - if config.runtime_settings.distributed: - raise ValueError("policy phases currently require local execution") + supervised = [phase for phase in phases if phase.algorithm == "supervised"] + if not supervised: return config - if config.initialize_from is not None: - return ( - config # Model-contract assertions are checked after artifact resolution. - ) model = config.model assert model is not None if ( @@ -639,36 +597,11 @@ def validate_training_contract(config): ), ) - referenced_interfaces = set() - for dataset_name, dataset in config.dataset_training.items(): - if dataset.model_interface not in model.interfaces: - raise ValueError( - f"Dataset {dataset_name!r} references unknown model interface " - f"{dataset.model_interface!r}." - ) - referenced_interfaces.add(dataset.model_interface) - interface = model.interfaces[dataset.model_interface] - learning = config.dataset_learning(dataset_name) - learning.validate_targets(interface.target_columns, f"Dataset {dataset_name!r}") - unreferenced = set(model.interfaces) - referenced_interfaces - if unreferenced: - warnings.warn( - f"Unreferenced model interfaces: {sorted(unreferenced)}", - stacklevel=2, - ) - - for phase in config.training_plan.phases: - for source in phase.sources: - _validate_source(config, source.source, f"training phase {phase.name!r}") objective = get_objective_class(config.runtime_settings.training_objective) is_bert = issubclass(objective, BERTObjective) - if (config.runtime_settings.bert_spec is not None) != is_bert: - raise ValueError("bert_spec must be configured exactly for BERT training") - if ( - len(config.dataset_training) > 1 - and config.runtime_settings.data_parallelism == "fsdp" - ): - raise ValueError("Multi-dataset training does not support FSDP") + for phase in supervised: + if (phase.learning.bert_spec is not None) != is_bert: + raise ValueError("bert_spec must be configured exactly for BERT training") context_length = config.runtime_settings.context_length prefix_positions = int( @@ -700,12 +633,6 @@ def validate_training_contract(config): context_length, usage="training", ) - if ( - not config.run.export.generative_model - and not config.run.export.embedding_model - and os.getenv("SEQUIFIER_PREVENT_EXPORT") is None - ): - raise ValueError("At least one model export must be enabled") return config @@ -735,18 +662,28 @@ class ResolvedInterfaceStorage(ConfigModel): prediction_length: Optional[int] = None target_offset: Optional[int] = None static_features_path: str | None = None - image_sources: dict[str, ImageSourceConfig] = Field(default_factory=dict) - preconditioning_image: PreconditioningImageConfig | None = None storage_layout: StoredWindowLayout @classmethod - def from_interface( - cls, interface: ResolvedModelInterface + def from_metadata( + cls, metadata: DatasetMetadata, layout: StoredWindowLayout | None = None ) -> ResolvedInterfaceStorage: - return cls.model_validate(interface.model_dump(include=set(cls.model_fields))) + return cls( + tensor_payload_version=metadata.tensor_payload_version, + split_paths=metadata.split_paths, + prediction_aligned_splits=metadata.prediction_aligned_splits, + prediction_alignment_steps=metadata.prediction_alignment_steps, + window_lengths=metadata.window_lengths, + prediction_length=metadata.prediction_length, + target_offset=metadata.target_offset, + static_features_path=metadata.static_features_path, + storage_layout=layout or metadata.storage_layout, + ) -class ResolvedModelInterface(ResolvedInterfaceStorage): +class ResolvedModelInterface(ConfigModel): + image_sources: dict[str, ImageSourceConfig] = Field(default_factory=dict) + preconditioning_image: PreconditioningImageConfig | None = None policy: ResolvedPolicyBinding | None = None prediction_distributions: dict[str, PredictionDistributionConfig] = Field( default_factory=dict @@ -803,7 +740,7 @@ class ResolvedDatasetTrainingSpec(ConfigModel): name: str model_interface: str - storage: ResolvedInterfaceStorage | None = Field(default=None, exclude=True) + storage: ResolvedInterfaceStorage | None = None parts: dict[str, ResolvedDatasetPart] criterion: dict[str, str] class_weights: Optional[dict[str, list[float]]] = None @@ -817,6 +754,7 @@ class ResolvedTrainingSource(ConfigModel): source: str dataset: str part: Optional[str] = None + parts: tuple[str, ...] selection_weight: float = 1.0 batches_per_selection: int = 1 @@ -848,7 +786,7 @@ class ResolvedTrainingPhase(PhaseControls): sources: list[ResolvedTrainingSource] -def resolve_supervised_phases(authored_phases, *, seed=0): +def resolve_phases(authored_phases, datasets, *, seed: int): """Resolve role defaults and epoch budgets without accessing datasets.""" phases = [ ResolvedTrainingPhase( @@ -856,7 +794,7 @@ def resolve_supervised_phases(authored_phases, *, seed=0): resolved_seed=seed + phase_index, epochs=phase.epochs, evaluation_sources=[ - _source(ref.source, 1) for ref in phase.evaluation.sources + _source(ref.source, datasets, 1) for ref in phase.evaluation.sources ] if phase.evaluation is not None else [], @@ -882,6 +820,7 @@ def resolve_supervised_phases(authored_phases, *, seed=0): sources=[ _source( source.source, + datasets, phase.accumulation_batches, selection_weight=source.selection_weight, batches_per_selection=( @@ -893,10 +832,14 @@ def resolve_supervised_phases(authored_phases, *, seed=0): for source in phase.sources ], ) + if phase.algorithm == "supervised" + else phase for phase_index, phase in enumerate(authored_phases) ] # A continued epoch scheduler spans the entire contiguous ownership chain. for index, phase in enumerate(phases): + if phase.algorithm != "supervised": + continue role = phase.optimization["model"] if role.scheduler.name != "OneCycleLR" or role.scheduler_step_on != "epoch": continue @@ -918,7 +861,7 @@ def resolve_supervised_phases(authored_phases, *, seed=0): class ResolvedSequifierConfig(ConfigModel): - """Runtime configuration after all dataset parts and interfaces resolve.""" + """Compiled run, model, source, phase, and publication records.""" _prepared_inputs: Any = PrivateAttr(default=None) @@ -935,8 +878,11 @@ class ResolvedSequifierConfig(ConfigModel): device: str seed: int runtime_settings: TrainingRuntimeSettings + model_options: ModelExecutionOptions model: ModelSpecModel interfaces: dict[str, ResolvedModelInterface] = Field(min_length=1) + interface_storage: dict[str, ResolvedInterfaceStorage] = Field(default_factory=dict) + model_metadata_paths: dict[str, str] = Field(default_factory=dict) dataset_training: dict[str, ResolvedDatasetTrainingSpec] authored: SequifierConfig | None = Field(default=None, exclude=True) initialize_from: str | None = None @@ -950,36 +896,7 @@ class ResolvedSequifierConfig(ConfigModel): training_plan: list[ResolvedTrainingPhase | PolicyPhase] curriculum_training: bool = False curriculum_column: Optional[str] = None - - @model_validator(mode="before") - @classmethod - def read_dataset_interfaces(cls, values): - """Read checkpoints with embedded dataset interface views.""" - if not isinstance(values, dict) or not isinstance( - values.get("dataset_training"), dict - ): - return values - datasets = {} - for name, dataset in values["dataset_training"].items(): - if isinstance(dataset, dict) and "interface" in dataset: - dataset = dict(dataset) - interface = ResolvedModelInterface.model_validate( - dataset.pop("interface") - ) - dataset["storage"] = ResolvedInterfaceStorage.from_interface(interface) - datasets[name] = dataset - return {**values, "dataset_training": datasets} - - @model_serializer(mode="wrap") - def serialize_training_config(self, handler, info): - values = handler(self) - if self.actor_initialization == "attach": - values.pop("actor_initialization", None) - for name, dataset in values.get("dataset_training", {}).items(): - dataset["interface"] = self.dataset_interface(name).model_dump( - mode=info.mode - ) - return values + publication: PublicationPlan @property def evaluation_sources(self) -> list[ResolvedTrainingSource]: @@ -993,12 +910,9 @@ def evaluation_sources(self) -> list[ResolvedTrainingSource]: }.values() ) - export_generative_model: bool - export_embedding_model: bool - embedding_layer_names: list[str] - export_onnx: bool - export_pt: bool - export_with_dropout: bool = False + @property + def embedding_layer_names(self) -> list[str]: + return self.publication.exports.embedding_layer_names def validate_next_occurrence_phases(self): objective = get_objective_class(self.runtime_settings.training_objective) @@ -1100,9 +1014,24 @@ def interface(self, name: Optional[str] = None) -> ResolvedModelInterface: def dataset_interface(self, name: str) -> ResolvedModelInterface: dataset = self.dataset_training[name] interface = self.interface(dataset.model_interface) - if dataset.storage is None: + if not dataset.parts: return interface - return interface.model_copy(update=dict(dataset.storage)) + metadata = next(iter(dataset.parts.values())).metadata + return interface.model_copy( + update={ + "image_sources": { + view: metadata.image_sources[view] + for view in interface.image_sources + }, + "preconditioning_image": metadata.preconditioning_image + if interface.preconditioning_image_encoder is not None + else None, + } + ) + + def dataset_storage(self, name: str) -> ResolvedInterfaceStorage: + dataset = self.dataset_training[name] + return dataset.storage or self.interface_storage[dataset.model_interface] @dataclass(frozen=True) @@ -1114,13 +1043,14 @@ class LoadedTrainConfig: @beartype def _source( - source: str, accumulation_batches: Optional[int], **values: Any + source: str, datasets: dict, accumulation_batches: Optional[int], **values: Any ) -> ResolvedTrainingSource: dataset, _, part = source.partition(".") return ResolvedTrainingSource( source=source, dataset=dataset, part=part or None, + parts=(part,) if part else tuple(datasets[dataset].parts), selection_weight=values.get("selection_weight") or 1.0, batches_per_selection=( values.get("batches_per_selection") or accumulation_batches or 1 @@ -1153,27 +1083,6 @@ def _layout_for_part_path( return metadata.storage_layout -@beartype -def _evaluated_parts(config: Any) -> set[str]: - selected: set[str] = set() - sources = [ - ref - for phase in config.training_plan.phases - if phase.algorithm == "supervised" and phase.evaluation is not None - for ref in phase.evaluation.sources - ] - for source in sources: - dataset_name, _, part_name = source.source.partition(".") - if part_name: - selected.add(source.source) - else: - selected.update( - f"{dataset_name}.{name}" - for name in config.dataset_training[dataset_name].parts - ) - return selected - - @beartype def _part_signature( metadata: DatasetMetadata, interface: ModelInterfaceSpecModel @@ -1326,7 +1235,6 @@ def _resolve_interface( spec: ModelInterfaceSpecModel, metadata: DatasetMetadata, global_spec: TrainingRuntimeSettings, - storage_layout: StoredWindowLayout, ) -> ResolvedModelInterface: if global_spec.training_objective is None or global_spec.context_length is None: raise ValueError( @@ -1439,14 +1347,12 @@ def _resolve_interface( objective=global_spec.training_objective, target_offset=target_offset, ) - resolve_window_view(storage_layout, window_view) return ResolvedModelInterface( name=name, policy=None, input_columns=spec.input_columns, prediction_distributions=spec.prediction_distributions, static_conditioning=spec.static_conditioning, - static_features_path=metadata.static_features_path, image_sources={ name: metadata.image_sources[name] for name in sorted(image_views(spec.ingestion)) @@ -1471,13 +1377,6 @@ def _resolve_interface( for column, tokens in categorical_decoder_special_tokens.items() }, depth_layouts=metadata.depth_layouts.relevant_layouts(spec.input_columns), - tensor_payload_version=metadata.tensor_payload_version, - split_paths=metadata.split_paths, - prediction_aligned_splits=metadata.prediction_aligned_splits, - prediction_alignment_steps=metadata.prediction_alignment_steps, - window_lengths=metadata.window_lengths, - prediction_length=metadata.prediction_length, - target_offset=metadata.target_offset, feature_layout=spec.feature_layout, ingestion=spec.ingestion, decoder=spec.decoder, @@ -1505,7 +1404,6 @@ def _resolve_interface( target_decoder_ids=target_decoder_ids, target_n_classes=target_n_classes, target_global_to_decoder=target_global_to_decoder, - storage_layout=storage_layout, window_view=window_view, ) @@ -1519,14 +1417,6 @@ def _interface_semantics(interface: ResolvedModelInterface) -> dict[str, Any]: "ingestion", "decoder", "feature_layout", - "tensor_payload_version", - "split_paths", - "prediction_aligned_splits", - "prediction_alignment_steps", - "window_lengths", - "prediction_length", - "target_offset", - "static_features_path", "image_sources", "preconditioning_image", }, @@ -1555,48 +1445,24 @@ def _interface_semantics(interface: ResolvedModelInterface) -> dict[str, Any]: @beartype -def resolve_sequifier_config( - config: Any, - metadata: DatasetMetadata | dict[str, DatasetMetadata], - *, - execution=None, -) -> ResolvedSequifierConfig: - """Resolve every dataset part, compatibility contract, and source.""" - - validate_training_contract(config) - if config.model is None: - raise ValueError("resolved training requires a model") - if ( - config.runtime_settings.context_length is None - or config.runtime_settings.training_objective is None - ): - raise ValueError( - "resolved training requires training_objective and context_length" - ) - - part_refs = [ - f"{dataset_name}.{part_name}" - for dataset_name, dataset in config.dataset_training.items() - for part_name in dataset.parts - ] - if isinstance(metadata, DatasetMetadata): - if len(part_refs) != 1: - raise ValueError( - "Metadata must be keyed by dataset.part for multiple parts" - ) - metadata_by_part = {part_refs[0]: metadata} - else: - metadata_by_part = metadata - missing_metadata = set(part_refs) - set(metadata_by_part) - if missing_metadata: - raise ValueError(f"Missing metadata for parts: {sorted(missing_metadata)}") - - evaluated = _evaluated_parts(config) +def resolve_supervised_datasets(config, metadata_by_part, phases, dataset_names): + """Resolve prepared dataset parts and their model compatibility contracts.""" + + evaluated = { + f"{source.dataset}.{part}" + for phase in phases + if phase.algorithm == "supervised" + for source in phase.evaluation_sources + for part in source.parts + } resolved_datasets: dict[str, ResolvedDatasetTrainingSpec] = {} - interfaces = {} if execution is None else dict(execution.interfaces) + interfaces = {} + interface_storage = {} dataset_semantics: dict[str, dict[str, Any]] = {} for dataset_name, dataset_spec in config.dataset_training.items(): + if dataset_name not in dataset_names: + continue learning = config.dataset_learning(dataset_name) if dataset_spec.model_interface is None: raise ValueError(f"Dataset {dataset_name!r} requires a model interface") @@ -1608,6 +1474,10 @@ def resolve_sequifier_config( first_metadata = None for part_name, part_spec in dataset_spec.parts.items(): ref = f"{dataset_name}.{part_name}" + if not isinstance(part_spec, DatasetPartSpecModel): + raise ValueError(f"{ref}: supervised sources require prepared parts") + if ref not in metadata_by_part: + raise ValueError(f"Missing metadata for part: {ref}") part_metadata = metadata_by_part[ref] signature = _part_signature(part_metadata, interface_spec) if expected_signature is None: @@ -1660,6 +1530,9 @@ def resolve_sequifier_config( interface_spec.target_columns, first_metadata.column_data_types, ) + learning.validate_targets( + interface_spec.target_columns, f"Dataset {dataset_name!r}" + ) for column in learning.class_share_log_columns: if column not in interface_spec.target_columns: raise ValueError(f"Class-share column {column!r} must be a target") @@ -1696,7 +1569,9 @@ def resolve_sequifier_config( interface_spec, first_metadata, config.runtime_settings, - first_training_layout, + ) + storage = ResolvedInterfaceStorage.from_metadata( + first_metadata, first_training_layout ) for part_name, resolved_part in resolved_parts.items(): ref = f"{dataset_name}.{part_name}" @@ -1744,24 +1619,10 @@ def resolve_sequifier_config( f"{aligned_metadata.prediction_length}, but the selected interface " f"uses {interface.decoder.prediction_length}." ) - semantic_contract = _interface_semantics(interface) - if execution is not None: - canonical = _interface_semantics(execution.interface(interface_name)) - dataset_fields = { - "storage_layout", - "window_view", - "policy", - "prediction_distributions", - } - _assert_compatible( - { - key: value - for key, value in canonical.items() - if key not in dataset_fields - }, - semantic_contract, - f"Dataset {dataset_name!r} for interface {interface_name!r}", - ) + semantic_contract = { + **_interface_semantics(interface), + "storage_layout": first_training_layout, + } prior_contract = dataset_semantics.get(interface_name) if prior_contract is not None: _assert_compatible( @@ -1772,6 +1633,7 @@ def resolve_sequifier_config( else: dataset_semantics[interface_name] = semantic_contract interfaces.setdefault(interface_name, interface) + interface_storage.setdefault(interface_name, storage) for target, criterion in learning.criterion.items(): if ( @@ -1815,7 +1677,7 @@ def resolve_sequifier_config( resolved_datasets[dataset_name] = ResolvedDatasetTrainingSpec( name=dataset_name, model_interface=interface_name, - storage=ResolvedInterfaceStorage.from_interface(interface), + storage=storage, parts=resolved_parts, criterion=learning.criterion, class_weights=learning.class_weights, @@ -1823,63 +1685,28 @@ def resolve_sequifier_config( class_share_log_columns=learning.class_share_log_columns, ) - for name, spec in config.model.interfaces.items(): - if name in interfaces: - continue - path = spec.input_contract.metadata_config_path - if path is None: - raise ValueError(f"{name}: model interface requires metadata") - interface_metadata = load_dataset_metadata( - normalize_path(path, config.run.project_root) - ) - interfaces[name] = _resolve_interface( - name, - spec, - interface_metadata, - config.runtime_settings, - interface_metadata.storage_layout, - ) + return resolved_datasets, interfaces, interface_storage - phases = resolve_supervised_phases( - config.training_plan.phases, seed=config.run.reproducibility.seed - ) - resolved = ResolvedSequifierConfig( - ownership=config.model_dump(mode="json"), - authored=config, - source_validation_fractions={ - f"{name}.{part}": spec.split.validation_fraction - for name, spec in config.dataset_training.items() - for part in spec.parts - }, - project_root=config.run.project_root, - model_name=config.run.model_name, - device=config.run.execution.device, - seed=config.run.reproducibility.seed, - runtime_settings=config.runtime_settings, - model=config.model.architecture_model(), - interfaces=interfaces, - dataset_training=resolved_datasets, - training_plan=phases, - curriculum_training=bool(config.training_plan.phases[0].curriculum), - curriculum_column=config.training_plan.phases[0].curriculum.column - if config.training_plan.phases[0].curriculum - else None, - export_generative_model=config.run.export.generative_model, - export_embedding_model=config.run.export.embedding_model, - embedding_layer_names=config.run.export.embedding_layer_names, - export_onnx=config.run.export.onnx, - export_pt=config.run.export.pt, - export_with_dropout=config.run.export.with_dropout, - ) - from sequifier.config.input_contract_resolution import validate_input_contracts - validate_input_contracts(config, resolved) - from sequifier.io.output_paths import resolve_training_output_paths +def resolve_sequifier_config( + config: SequifierConfig, + metadata: DatasetMetadata | dict[str, DatasetMetadata], +) -> ResolvedSequifierConfig: + """Compile a training plan with caller-provided dataset metadata.""" + from sequifier.config.training_resolution import resolve_training_run - for name in ("model_path", "report_path", "phase_directory"): - setattr(resolved, name, getattr(config.run.outputs, name)) - resolve_training_output_paths(resolved) - return resolved + if isinstance(metadata, DatasetMetadata): + parts = [ + f"{name}.{part}" + for name, dataset in config.dataset_training.items() + for part in dataset.parts + ] + if len(parts) != 1: + raise ValueError( + "Metadata must be keyed by dataset.part for multiple parts" + ) + metadata = {parts[0]: metadata} + return resolve_training_run(config, metadata_by_part=metadata) _INLINE_METADATA_KEYS = { @@ -1995,53 +1822,34 @@ def load_train_config_with_source( raise ValueError(f"Unsupported training CLI overrides: {sorted(args)}") authored = try_catch_excess_keys(config_path, SequifierConfig, raw) - config = authored - if ( - config.initialize_from is not None - or config.policy_bindings - or any(phase.algorithm != "supervised" for phase in config.training_plan.phases) - ): - if metadata_inline_values or skip_metadata: - raise ValueError( - "policy/mixed plans require source declarations instead of inline metadata overrides" - ) - from sequifier.config.training_resolution import resolve_training_run + from sequifier.config.training_resolution import resolve_training_run - return LoadedTrainConfig( - config=authored, resolved=resolve_training_run(config), metadata={} - ) metadata_by_part: dict[str, DatasetMetadata] = {} - all_part_refs = [ - (dataset_name, part_name, part) - for dataset_name, dataset in config.dataset_training.items() - for part_name, part in dataset.parts.items() - ] if skip_metadata: - if len(all_part_refs) != 1 and "metadata_by_part" not in metadata_inline_values: - raise ValueError( - "skip_metadata with multiple parts requires metadata_by_part" - ) + parts = [ + f"{name}.{part}" + for name, dataset in authored.dataset_training.items() + for part in dataset.parts + ] if "metadata_by_part" in metadata_inline_values: - for ref, values in metadata_inline_values["metadata_by_part"].items(): - metadata_by_part[ref] = _inline_metadata( - values, config.runtime_settings - ) - else: - dataset_name, part_name, _ = all_part_refs[0] - metadata_by_part[f"{dataset_name}.{part_name}"] = _inline_metadata( - metadata_inline_values, config.runtime_settings + metadata_by_part = { + ref: _inline_metadata(values, authored.runtime_settings) + for ref, values in metadata_inline_values["metadata_by_part"].items() + } + elif len(parts) == 1: + metadata_by_part[parts[0]] = _inline_metadata( + metadata_inline_values, authored.runtime_settings ) - else: - for dataset_name, part_name, part in all_part_refs: - ref = f"{dataset_name}.{part_name}" - configured_path = part.metadata_config_path - if configured_path is None: - raise ValueError(f"Part {ref!r} has no metadata_config_path") - metadata_by_part[ref] = load_dataset_metadata( - normalize_path(configured_path, config.run.project_root) + else: + raise ValueError( + "skip_metadata with multiple parts requires metadata_by_part" ) - resolved = resolve_sequifier_config(config, metadata_by_part) + resolved = resolve_training_run( + authored, + metadata_by_part=metadata_by_part if skip_metadata else None, + loaded_metadata=metadata_by_part, + ) return LoadedTrainConfig( config=authored, resolved=resolved, metadata=metadata_by_part ) @@ -2071,11 +1879,9 @@ class SelectedInterfaceConfig: model_name: str device: str seed: int - runtime_settings: TrainingRuntimeSettings training_objective: str input_columns: list[str] static_columns: list[str] - static_features_path: str | None image_sources: dict[str, ImageSourceConfig] preconditioning_image: PreconditioningImageConfig | None preconditioning_image_encoder: ImageEncoderConfig | None @@ -2089,22 +1895,28 @@ class SelectedInterfaceConfig: categorical_decoder_special_tokens: dict[str, list[str]] feature_layout: Optional[FeatureLayoutRegistryModel] depth_layouts: DepthLayoutRegistryModel - tensor_payload_version: int n_classes: dict[str, int] id_maps: dict[str, dict[str | int, int]] special_token_ids: dict[str, int] target_decoder_ids: dict[str, list[int]] target_n_classes: dict[str, int] target_global_to_decoder: dict[str, list[int]] - storage_layout: StoredWindowLayout window_view: ModelWindowView model: SelectedModelComponents @dataclass(frozen=True) -class SelectedDatasetPartConfig(SelectedInterfaceConfig): +class SelectedObjectiveConfig(SelectedInterfaceConfig): + runtime_settings: TrainingRuntimeSettings + + +@dataclass(frozen=True) +class SelectedDatasetPartConfig(SelectedObjectiveConfig): """Explicit dataset-part configuration used by data loaders.""" + static_features_path: str | None + tensor_payload_version: int + storage_layout: StoredWindowLayout batch_size: int dataset_training: ResolvedDatasetTrainingSpec part: ResolvedDatasetPart @@ -2122,22 +1934,15 @@ class SelectedDatasetPartConfig(SelectedInterfaceConfig): def interface_build_view( config: Any, interface: ResolvedModelInterface, - *, - runtime_settings: TrainingRuntimeSettings | None = None, ) -> SelectedInterfaceConfig: """Select one interface for model and objective construction.""" - settings = runtime_settings or config.runtime_settings - objective = settings.training_objective - if objective is None: - raise ValueError("Interface construction requires a training objective") return SelectedInterfaceConfig( project_root=config.project_root, model_name=config.model_name, device=config.device, seed=config.seed, - runtime_settings=settings, - training_objective=objective, + training_objective=interface.window_view.objective, static_columns=( interface.static_conditioning.columns if interface.static_conditioning @@ -2171,10 +1976,9 @@ def dataset_part_view( dataset = config.dataset_training[dataset_name] part = dataset.parts[part_name] interface_config = interface_build_view( - config, config.dataset_interface(dataset_name) + config, config.interface(dataset.model_interface) ) interface_values = vars(interface_config).copy() - interface_values["static_features_path"] = part.metadata.static_features_path selected_sources = { name: part.metadata.image_sources[name] for name in interface_config.image_sources @@ -2194,6 +1998,10 @@ def dataset_part_view( ) return SelectedDatasetPartConfig( + runtime_settings=config.runtime_settings, + static_features_path=part.metadata.static_features_path, + tensor_payload_version=part.metadata.tensor_payload_version, + storage_layout=config.dataset_storage(dataset_name).storage_layout, batch_size=batch_size, **interface_values, dataset_training=dataset, @@ -2376,51 +2184,6 @@ def validate_contract(self): self.model.interfaces[interface_name].target_columns, f"{phase.name}: learning.datasets.{name}", ) - if any(phase.algorithm != "supervised" for phase in self.training_plan.phases): - execution = self.run.execution - unsupported_execution = execution.model_fields_set & { - "distribution", - "compilation", - "precision", - "max_concat_tensors", - } - if unsupported_execution: - raise ValueError( - "policy phases do not support run.execution settings: " - f"{sorted(unsupported_execution)}" - ) - if ( - all( - phase.algorithm != "supervised" - for phase in self.training_plan.phases - ) - and "log_interval_batches" in self.run.logging.model_fields_set - ): - raise ValueError("policy phases do not support batch logging intervals") - export = self.run.export - unsupported_export = export.model_fields_set & { - "onnx", - "embedding_model", - "with_dropout", - "generative_model", - "pt", - "batch_size", - } - if unsupported_export: - raise ValueError( - "policy-containing plans do not support run.export settings: " - f"{sorted(unsupported_export)}" - ) - unsupported_loading = execution.data_loading.model_fields_set & { - "num_workers", - "load_full_data_to_ram", - "max_ram_gb", - } - if unsupported_loading: - raise ValueError( - "policy phases do not support run.execution.data_loading settings: " - f"{sorted(unsupported_loading)}" - ) return self def _build_runtime_settings(self): diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index edc4ad36..24db460a 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -7,16 +7,23 @@ SequenceSourceConfig, resolve_sequence_source, ) +from sequifier.config.run_capabilities import publication_plan, validate_runner_settings +from sequifier.config.sources import phase_sources from sequifier.config.train_config import ( DatasetPartSpecModel, OwnedModelSpec, PreferencePartSpecModel, + ResolvedInterfaceStorage, ResolvedSequifierConfig, SequencePartSpecModel, SequifierConfig, TrainingRuntimeSettings, + _assert_compatible, + _interface_semantics, _resolve_interface, - resolve_sequifier_config, + resolve_phases, + resolve_supervised_datasets, + validate_training_contract, ) from sequifier.special_tokens import SPECIAL_TOKEN_IDS @@ -57,31 +64,31 @@ def __init__(self, config, root): for part_name, part in dataset.parts.items() } - def resolve(self, *, prepared=True): + def resolve(self, metadata_by_part): sequences, preferences = {}, {} for ref, (_, part) in self.parts.items(): if isinstance(part, PreferencePartSpecModel): path = self.root / part.manifest_path - preferences[ref] = str(path.resolve() if prepared else path) + preferences[ref] = str(path.resolve()) continue if isinstance(part, SequencePartSpecModel): source = SequenceSourceConfig.model_validate( part.model_dump(exclude={"type"}) ) - elif prepared and isinstance(part, DatasetPartSpecModel): + else: source = SequenceSourceConfig( metadata_config_path=part.metadata_config_path, split_index=0, format="prepared", encoding="model", ) - else: - continue - sequences[ref] = resolve_sequence_source(source, self.root) + sequences[ref] = resolve_sequence_source( + source, self.root, metadata=metadata_by_part.get(ref) + ) return sequences, preferences -def _resolve_phase_sources(config, phase, available, *, strict=True): +def _resolve_phase_sources(config, phase, available): """Expand named sources for both metadata fitting and final run resolution.""" preference = phase.algorithm == "dpo" fields = ( @@ -100,24 +107,22 @@ def _resolve_phase_sources(config, phase, available, *, strict=True): for full, _ in source_parts(config, ref.source) ] if not expanded or any(ref.source not in available for ref in expanded): - if not strict: - return False raise ValueError( f"{phase.name}: select {'preference' if preference else 'sequence'} parts" ) if single and len(expanded) != 1: raise ValueError(f"{phase.name}: select one training preference part") setattr(owner, key, expanded[0] if single else expanded) - return True def phase_interface(config, phase): - names = set() refs = list(getattr(phase, "sources", [])) if phase.algorithm in {"policy_imitation", "iql"}: refs.extend(phase.evaluation.sources or []) - for ref in refs: - names.add(config.dataset_training[ref.source.partition(".")[0]].model_interface) + names = { + config.dataset_training[ref.source.partition(".")[0]].model_interface + for ref in refs + } explicit = getattr(phase, "model_interface", None) if explicit is not None: names.add(explicit) @@ -190,7 +195,7 @@ def _inferred_metadata(config, name, binding): ) -def _history_initialized_metadata(config, name, binding, source): +def _history_initialized_metadata(config, name, binding, source, storage): """Resolve new targets while sourcing observation preprocessing from the artifact.""" metadata = _inferred_metadata(config, name, binding) static = source.static_conditioning @@ -219,18 +224,29 @@ def _history_initialized_metadata(config, name, binding, source): return metadata.model_copy( update={ "static_columns": static.columns if static else [], - "static_features_path": source.static_features_path, + "static_features_path": storage.static_features_path if storage else None, "image_sources": source.image_sources, "preconditioning_image": source.preconditioning_image, "special_token_ids": source.special_token_ids, "depth_layouts": source.depth_layouts, - "tensor_payload_version": source.tensor_payload_version, + "tensor_payload_version": storage.tensor_payload_version + if storage + else metadata.tensor_payload_version, "normalize_real_columns": source.normalize_real_columns, } ) -def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): +def _fit_input_metadata( + config, + name, + binding, + metadata, + sources, + preferences, + source_fractions, + prepared_inputs, +): """Fit missing raw codecs/moments on the same episode-safe training split.""" from types import SimpleNamespace @@ -240,8 +256,7 @@ def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): ( phase for phase in config.training_plan.phases - if phase.algorithm != "supervised" - and phase_interface(config, phase) == name + if phase.algorithm != "supervised" and phase.model_interface == name ), None, ) @@ -270,18 +285,12 @@ def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): context_length=config.runtime_settings.context_length, interface_name=name, ) - sources, preferences = TrainingSources(config, root).resolve(prepared=False) - phase = phase.model_copy(deep=True) - if not _resolve_phase_sources( - config, - phase, - preferences if phase.algorithm == "dpo" else sources, - strict=False, - ): - return metadata # Prepared sources already own frozen preprocessing metadata. + available = preferences if phase.algorithm == "dpo" else sources + if any(ref.source not in available for ref in phase_sources(phase)): + return metadata # Prepared dataset parts already own frozen preprocessing. run = SimpleNamespace( validation_fraction=0.2, - source_validation_fractions=_source_fractions(config), + source_validation_fractions=source_fractions, seed=config.run.reproducibility.seed, preferences=preferences, sources=sources, @@ -307,44 +316,45 @@ def _fit_input_metadata(config, name, binding, metadata, root, prepared_inputs): return DatasetMetadata.model_validate(values) -def _supervised_subset(config): - phases = [ - phase - for phase in config.training_plan.phases - if phase.algorithm == "supervised" - ] - names = {ref.source.partition(".")[0] for phase in phases for ref in phase.sources} - names.update( - ref.source.partition(".")[0] - for phase in phases - if phase.evaluation is not None - for ref in phase.evaluation.sources - ) - return config.model_copy( - update={ - "dataset_training": { - name: dataset - for name, dataset in config.dataset_training.items() - if name in names - }, - "training_plan": config.training_plan.model_copy(update={"phases": phases}), - "policy_bindings": {}, - } - ) - - -def resolve_training_run(authored: SequifierConfig): +def resolve_training_run( + authored: SequifierConfig, + *, + metadata_by_part: dict[str, DatasetMetadata] | None = None, + loaded_metadata: dict[str, DatasetMetadata] | None = None, +) -> ResolvedSequifierConfig: + """Compile the authored run into the records consumed by both runners.""" from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.artifacts.model_config import model_execution_from_model_config from sequifier.model.execution import ModelExecution + from sequifier.model.execution_options import ModelExecutionOptions from sequifier.model.history_transfer import validate_history_transfer from sequifier.rl.attachment import apply_binding, resolve_binding from sequifier.rl.prepared_inputs import PreparedTrainingInputs prepared_inputs = PreparedTrainingInputs() config = authored.model_copy(deep=True) + phases = config.training_plan.phases + supervised = [phase for phase in phases if phase.algorithm == "supervised"] + has_policy = len(supervised) != len(phases) + validate_runner_settings(config.run, supervised=bool(supervised), policy=has_policy) + needs_policy_contracts = has_policy or bool( + config.policy_bindings or config.initialize_from + ) + supervised_names = { + ref.source.partition(".")[0] + for phase in supervised + for ref in phase_sources(phase) + } + if not needs_policy_contracts: + supervised_names.update(config.dataset_training) + if ( + len(supervised_names) > 1 + and config.run.execution.distribution.strategy == "fsdp" + ): + raise ValueError("Multi-dataset training does not support FSDP") root = Path(config.run.project_root).resolve() - config.run.project_root = str(root) + if needs_policy_contracts: + config.run.project_root = str(root) training = config.runtime_settings source = None history_only = config.actor_initialization == "history_only" @@ -366,7 +376,7 @@ def resolve_training_run(authored: SequifierConfig): "target_offset", "forward_attention", ): - actual = getattr(source.runtime_settings, field) + actual = getattr(source.options, field) if ( history_only and field in {"training_objective", "target_offset"} @@ -381,32 +391,33 @@ def resolve_training_run(authored: SequifierConfig): f"model.execution.{field} differs from initialize_from" ) setattr(training, field, actual) - if history_only: - if ( - "layer_type_dtypes" in config.run.execution.precision.model_fields_set - and training.layer_type_dtypes - != source.runtime_settings.layer_type_dtypes - ): - raise ValueError("history transfer differs at layer_type_dtypes") - training.layer_type_dtypes = source.runtime_settings.layer_type_dtypes + source_dtypes = source.options.layer_type_dtypes + if ( + "layer_type_dtypes" in config.run.execution.precision.model_fields_set + and training.layer_type_dtypes != source_dtypes + ): + raise ValueError( + "run.execution.precision.layer_type_dtypes differs from initialize_from" + ) + training.layer_type_dtypes = source_dtypes + if "max_concat_tensors" not in config.run.execution.model_fields_set: + training.max_concat_tensors = source.options.max_concat_tensors assert config.model is not None assert config.model.backbone is not None - for dataset in config.dataset_training.values(): + for dataset_name, dataset in config.dataset_training.items(): if dataset.model_interface is None: if len(config.model.interfaces) != 1: raise ValueError("dataset model_interface is required when ambiguous") dataset.model_interface = next(iter(config.model.interfaces)) if dataset.model_interface not in config.model.interfaces: raise ValueError( - f"unknown dataset model_interface {dataset.model_interface!r}" + f"Dataset {dataset_name!r} references unknown model interface " + f"{dataset.model_interface!r}." ) - training = config.runtime_settings if training.context_length is None: training.context_length = config.model.backbone.architecture.max_context_length if training.training_objective is None: - if any( - phase.algorithm == "supervised" for phase in config.training_plan.phases - ): + if supervised: raise ValueError( "supervised phases require model.execution.training_objective" ) @@ -414,88 +425,149 @@ def resolve_training_run(authored: SequifierConfig): training.target_offset = 0 training = TrainingRuntimeSettings.model_validate(training.model_dump()) config.runtime_settings = training - for field in ( - "context_length", - "training_objective", - "target_offset", - "forward_attention", - ): - setattr(config.model.execution, field, getattr(training, field)) - - # All declared metadata for an interface is an assertion of the same facts. - metadata = {} - declarations = { - name: value.input_contract.metadata_config_path - for name, value in config.model.interfaces.items() - if value.input_contract.metadata_config_path + if needs_policy_contracts: + for field in ( + "context_length", + "training_objective", + "target_offset", + "forward_attention", + ): + setattr(config.model.execution, field, getattr(training, field)) + + # Keep each physical metadata read shared by model assertions and source resolution. + metadata_cache: dict[Path, DatasetMetadata] = {} + + def read_metadata(path): + resolved = (root / path).resolve() + if resolved not in metadata_cache: + metadata_cache[resolved] = load_dataset_metadata(str(resolved)) + return metadata_cache[resolved] + + metadata_paths = { + name: str(root / path) + for name, spec in config.model.interfaces.items() + if (path := spec.input_contract.metadata_config_path) } - for name, path in (declarations or {}).items(): - if name not in config.model.interfaces: - raise ValueError(f"unknown metadata model_interface {name!r}") - metadata[name] = load_dataset_metadata(str(root / path)) + declarations = {name: read_metadata(path) for name, path in metadata_paths.items()} part_metadata = {} for dataset_name, dataset in config.dataset_training.items(): for part_name, part in dataset.parts.items(): + ref = f"{dataset_name}.{part_name}" path = getattr(part, "metadata_config_path", None) - if path: - value = load_dataset_metadata(str(root / path)) - part_metadata[f"{dataset_name}.{part_name}"] = value - existing = metadata.get(dataset.model_interface) - if existing is not None: - for key in ( - "column_data_types", - "n_classes", - "id_maps", - "selected_columns_statistics", - "normalize_real_columns", - ): - if getattr(existing, key) != getattr(value, key): - raise ValueError( - f"{dataset_name}.{part_name}: interface metadata differs at {key}" - ) - metadata[dataset.model_interface] = value - interfaces = {} - history_interface = ( - phase_interface(config, config.training_plan.phases[0]) - if history_only - else None + if metadata_by_part is not None and isinstance(part, DatasetPartSpecModel): + if ref in metadata_by_part: + part_metadata[ref] = metadata_by_part[ref] + elif path: + part_metadata[ref] = read_metadata(path) + if loaded_metadata is not None: + loaded_metadata.update(part_metadata) + + source_fractions = _source_fractions(config) + sources = TrainingSources(config, root) + sequences, preferences = ( + sources.resolve(metadata_by_part=part_metadata) + if needs_policy_contracts + else ({}, {}) ) + fitting_sequences = { + ref: sequence + for ref, sequence in sequences.items() + if isinstance(sources.parts[ref][1], SequencePartSpecModel) + } + + for phase in phases: + if phase.algorithm == "supervised": + continue + phase.optimization = phase.optimization_configs() + phase.model_interface = phase_interface(config, phase) + _resolve_phase_sources( + config, phase, preferences if phase.algorithm == "dpo" else sequences + ) + reference = getattr(phase, "reference", None) + if reference and reference.path: + reference.path = str((root / reference.path).resolve()) + archive = getattr(phase, "archive_directory", None) + if archive: + phase.archive_directory = str((root / archive).resolve()) + + from sequifier.runtime.plan import validate_continuation + + for index, phase in enumerate(phases): + validate_continuation( + phases[index - 1] if index else None, phase, resolved=True + ) + + resolved_phases = resolve_phases( + phases, config.dataset_training, seed=config.run.reproducibility.seed + ) + validate_training_contract(config, resolved_phases) + + datasets, dataset_interfaces, dataset_storage = resolve_supervised_datasets( + config, part_metadata, resolved_phases, supervised_names + ) + metadata = dict(declarations) + if needs_policy_contracts: + for ref, value in part_metadata.items(): + name = config.dataset_training[ref.partition(".")[0]].model_interface + if name in metadata: + for key in ( + "column_data_types", + "n_classes", + "id_maps", + "selected_columns_statistics", + "normalize_real_columns", + ): + if getattr(metadata[name], key) != getattr(value, key): + raise ValueError(f"{ref}: interface metadata differs at {key}") + metadata[name] = value + + interfaces = {} + storage = {} + history_interface = phases[0].model_interface if history_only else None if history_only: assert source is not None if history_interface not in source.interfaces: raise ValueError( f"history transfer requires interface {history_interface!r}" ) - for name, interface in config.model.interfaces.items(): + interface_order = dict.fromkeys( + config.model.interfaces + if needs_policy_contracts + else [*dataset_interfaces, *config.model.interfaces] + ) + for name in interface_order: + spec = config.model.interfaces[name] + value = metadata.get(name) + binding = config.policy_bindings.get(name) if source is not None and not history_only: - interfaces[name] = source.interface(name).model_copy(deep=True) - if name in metadata: - candidate = _resolve_interface( - name, - interface, - metadata[name], - training, - metadata[name].storage_layout, - ) + interface = source.interface(name) + if value is not None: + candidate = _resolve_interface(name, spec, value, training) for key in ( "column_data_types", "id_maps", "selected_columns_statistics", "target_decoder_ids", ): - if getattr(candidate, key) != getattr(interfaces[name], key): + if getattr(candidate, key) != getattr(interface, key): raise ValueError( f"{name}: metadata differs from initialization at {key}" ) + selected_storage = source.interface_storage.get(name) + elif name in dataset_interfaces and not needs_policy_contracts: + interface = dataset_interfaces[name] + selected_storage = dataset_storage[name] else: - value = metadata.get(name) - binding = config.policy_bindings.get(name) if name == history_interface and value is None: assert source is not None if binding is None: raise ValueError(f"{name}: history_only requires a policy binding") value = _history_initialized_metadata( - config, name, binding, source.interface(name) + config, + name, + binding, + source.interface(name), + source.interface_storage.get(name), ) if value is None: if binding is None: @@ -503,12 +575,37 @@ def resolve_training_run(authored: SequifierConfig): value = _inferred_metadata(config, name, binding) if binding is not None and name != history_interface: value = _fit_input_metadata( - config, name, binding, value, root, prepared_inputs + config, + name, + binding, + value, + fitting_sequences, + preferences, + source_fractions, + prepared_inputs, ) - interfaces[name] = _resolve_interface( - name, interface, value, training, value.storage_layout + interface = _resolve_interface(name, spec, value, training) + selected_storage = ResolvedInterfaceStorage.from_metadata(value) + from sequifier.helpers import resolve_window_view + + resolve_window_view(selected_storage.storage_layout, interface.window_view) + if name in dataset_interfaces and interface is not dataset_interfaces[name]: + canonical = _interface_semantics(interface) + for key in ("window_view", "policy", "prediction_distributions"): + canonical.pop(key) + _assert_compatible( + canonical, _interface_semantics(dataset_interfaces[name]), name ) - execution = ModelExecution(config.model.architecture_model(), interfaces, training) + if selected_storage is not None: + storage[name] = selected_storage + interfaces[name] = interface + + execution = ModelExecution( + config.model.architecture_model(), + interfaces, + ModelExecutionOptions.from_training(training), + interface_storage=storage, + ) if history_only: validate_history_transfer(source, execution, history_interface) for name, binding in config.policy_bindings.items(): @@ -516,114 +613,69 @@ def resolve_training_run(authored: SequifierConfig): raise ValueError(f"policy_bindings: unknown interface {name!r}") apply_binding(execution, name, resolve_binding(execution, name, binding)) - sequences, preferences = TrainingSources(config, root).resolve() - for phase in config.training_plan.phases: - phase.optimization = phase.optimization_configs() + for phase in phases: if phase.algorithm == "supervised": - for ref in phase.sources: - for _, part in source_parts(config, ref.source): - if not isinstance(part, DatasetPartSpecModel): - raise ValueError( - f"{phase.name}: supervised sources require prepared parts" - ) continue - name = phase_interface(config, phase) - phase.model_interface = name - if interfaces[name].policy is None: + name = phase.model_interface + policy = interfaces[name].policy + if policy is None: raise ValueError( f"{phase.name}: interface {name!r} requires a policy binding" ) - _resolve_phase_sources( - config, phase, preferences if phase.algorithm == "dpo" else sequences - ) for environment in ( getattr(phase, "environment", None), - getattr(getattr(phase, "evaluation", None), "environment", None), + getattr(phase.evaluation, "environment", None), ): - if environment and environment.units != interfaces[name].policy.units: + if environment and environment.units != policy.units: raise ValueError( f"{phase.name}: environment units differ from policy units" ) - reference = getattr(phase, "reference", None) - if reference and reference.path: - reference.path = str((root / reference.path).resolve()) - archive = getattr(phase, "archive_directory", None) - if archive: - phase.archive_directory = str((root / archive).resolve()) - from sequifier.runtime.plan import validate_continuation - for previous, current in zip( - config.training_plan.phases, config.training_plan.phases[1:] - ): - validate_continuation(previous, current, resolved=True) - if any(phase.algorithm == "supervised" for phase in config.training_plan.phases): - subset = _supervised_subset(config).model_copy( - update={"initialize_from": None, "actor_initialization": "attach"} - ) - resolved = resolve_sequifier_config(subset, part_metadata, execution=execution) - else: - resolved = ResolvedSequifierConfig( - model=execution.model, - interfaces=execution.interfaces, - runtime_settings=training, - dataset_training={}, - training_plan=[], - device=config.run.execution.device, - project_root=str(root), - model_name=config.run.model_name, - seed=config.run.reproducibility.seed, - export_generative_model=config.run.export.generative_model, - export_embedding_model=config.run.export.embedding_model, - embedding_layer_names=config.run.export.embedding_layer_names, - export_onnx=config.run.export.onnx, - export_pt=config.run.export.pt, - ) - resolved.ownership = config.model_dump(mode="json") - resolved.source_validation_fractions = _source_fractions(config) - resolved.normalization_by_interface = { - name: value - for name in interfaces - if (value := _normalization(config, name)) is not None - } - resolved.policy_bindings = { - name: interface.policy - for name, interface in interfaces.items() - if interface.policy is not None - } - resolved._prepared_inputs = prepared_inputs - resolved.authored = config - resolved.initialize_from = config.initialize_from - resolved.actor_initialization = config.actor_initialization + curriculum = supervised[0].curriculum if supervised else None + resolved = ResolvedSequifierConfig( + ownership=config.model_dump(mode="json"), + authored=config, + project_root=config.run.project_root, + model_name=config.run.model_name, + device=config.run.execution.device, + seed=config.run.reproducibility.seed, + runtime_settings=training, + model_options=execution.options, + model=execution.model, + interfaces=interfaces, + interface_storage=storage, + model_metadata_paths=metadata_paths, + dataset_training=datasets, + training_plan=resolved_phases, + initialize_from=config.initialize_from, + actor_initialization=config.actor_initialization, + model_path=config.run.outputs.model_path, + report_path=config.run.outputs.report_path, + phase_directory=config.run.outputs.phase_directory, + sequence_sources=sequences, + preference_sources=preferences, + source_validation_fractions=source_fractions, + policy_bindings={ + name: interface.policy + for name, interface in interfaces.items() + if interface.policy + }, + curriculum_training=curriculum is not None, + curriculum_column=curriculum.column if curriculum else None, + publication=publication_plan(config.run, resolved_phases), + ) + if has_policy: + resolved._prepared_inputs = prepared_inputs + from sequifier.config.input_contract_resolution import validate_input_contracts from sequifier.io.output_paths import resolve_training_output_paths - for name in ("model_path", "report_path", "phase_directory"): - setattr(resolved, name, getattr(config.run.outputs, name)) + validate_input_contracts(config, resolved, declarations) resolve_training_output_paths(resolved) - resolved.sequence_sources = sequences - resolved.preference_sources = preferences - supervised_by_name = {phase.name: phase for phase in resolved.training_plan} - resolved.training_plan = [ - supervised_by_name[phase.name] if phase.algorithm == "supervised" else phase - for phase in config.training_plan.phases - ] - for index, phase in enumerate(resolved.training_plan): - if phase.algorithm == "supervised": - phase.resolved_seed = config.run.reproducibility.seed + index - export = config.run.export - resolved.export_generative_model = export.generative_model - resolved.export_embedding_model = export.embedding_model - resolved.embedding_layer_names = export.embedding_layer_names - resolved.export_onnx = export.onnx - resolved.export_pt = export.pt - resolved.export_with_dropout = export.with_dropout - from sequifier.config.input_contract_resolution import validate_input_contracts - - validate_input_contracts(config, resolved) return resolved def policy_run_config( - config, + config: ResolvedSequifierConfig, phases, *, incoming=None, @@ -632,13 +684,9 @@ def policy_run_config( resuming=None, ): """Select resolved inputs for one native policy lifecycle.""" - from sequifier.artifacts.dependencies import model_metadata_paths - from sequifier.artifacts.model_artifact import read_execution_artifact - from sequifier.artifacts.model_config import model_execution_from_model_config - from sequifier.artifacts.model_export import model_execution_config + from sequifier.artifacts.run_checkpoint import checkpoint_path from sequifier.config.components import ResumeConfig from sequifier.model.execution import ModelExecution - from sequifier.rl.attachment import apply_binding, resolve_binding from sequifier.rl.prepared_inputs import PreparedTrainingInputs from sequifier.rl.run_config import ResolvedPolicyRun @@ -651,32 +699,12 @@ def policy_run_config( incoming if index or incoming is not None else config.initialize_from ) actor_initialization = config.actor_initialization if index == 0 else "attach" - if initialization and actor_initialization == "attach": - execution, _ = model_execution_from_model_config( - read_execution_artifact(initialization).model_config.values, - device=config.device, - interface_name=name, - ) - from sequifier.artifacts.model_config import extend_prediction_distributions - - extend_prediction_distributions(execution, config.model) - apply_binding( - execution, - name, - resolve_binding(execution, name, config.policy_bindings[name]), - ) binding = execution.interface(name).policy if binding is None: raise ValueError("selected interface has no policy binding") training = config.runtime_settings resume = training.resume - root = Path(config.project_root) - checkpoint = ( - (root / resume.checkpoint_path).resolve() - if resume and resume.checkpoint_path - else root - / f"checkpoints/runs/{config.model_name}/{config.model_name}-latest.pt" - ) + checkpoint = checkpoint_path(config) if resuming is not None: checkpoint = ( Path(config.phase_directory) / f"{index:03d}-{phases[0].name}" / "run.pt" @@ -687,12 +715,13 @@ def policy_run_config( device=config.device, seed=config.seed, enforce_determinism=training.enforce_determinism, - model=execution.model, policy=binding, - execution_model=model_execution_config(execution), + execution=execution, policy_interface=name, initialize_from=initialization, - metadata_paths=model_metadata_paths(config, name), + metadata_paths=[config.model_metadata_paths[name]] + if name in config.model_metadata_paths + else [], actor_initialization=actor_initialization, normalization=config.normalization_by_interface.get(name), units=binding.units, diff --git a/src/sequifier/evaluation/compatibility.py b/src/sequifier/evaluation/compatibility.py index 6fb4c8ac..828c7c0d 100644 --- a/src/sequifier/evaluation/compatibility.py +++ b/src/sequifier/evaluation/compatibility.py @@ -29,12 +29,14 @@ def evaluation_fingerprint(config, phase_index): learning = phase.learning.datasets.get(source.dataset) if learning is None: return None - parts = (source.part,) if source.part is not None else tuple(dataset.parts) sources.append( { "source": source.source, "interface": dataset.model_interface, - "interface_contract": interface.model_dump(mode="json"), + "interface_contract": { + **interface.model_dump(mode="json"), + **config.dataset_storage(source.dataset).model_dump(mode="json"), + }, "parts": [ { "name": name, @@ -54,7 +56,7 @@ def evaluation_fingerprint(config, phase_index): name ].preconditioning_image_paths, } - for name in parts + for name in source.parts ], "criterion": learning.criterion, "class_weights": learning.class_weights, diff --git a/src/sequifier/export/preflight.py b/src/sequifier/export/preflight.py index aad929f0..13f092f0 100644 --- a/src/sequifier/export/preflight.py +++ b/src/sequifier/export/preflight.py @@ -49,14 +49,15 @@ def graph_only(value): return [graph_only(item) for item in value] return value + export = config.publication.exports return { "protocol_version": ContractVersions.EXPORT_PREFLIGHT, "representative_batch_policy": "publication_clamped_2_to_4_preflight_2", "execution": graph_only(execution), - "generative": config.export_generative_model, - "embedding": config.export_embedding_model, + "generative": export.generative_model, + "embedding": export.embedding_model, "embedding_sites": config.embedding_layer_names, - "dropout_mode": "stochastic" if config.export_with_dropout else "evaluation", + "dropout_mode": "stochastic" if export.with_dropout else "evaluation", "export_policy": { "dynamo": True, "opset": ContractVersions.ONNX_OPSET, @@ -68,7 +69,7 @@ def graph_only(value): }, "versions": versions, "provider": "CPUExecutionProvider", - "session_optimization": "disabled" if config.export_with_dropout else "all", + "session_optimization": "disabled" if export.with_dropout else "all", "device": { "machine": platform.machine(), "system": platform.platform(), @@ -92,7 +93,8 @@ def graph_only(value): def ensure_export_preflight(config, distributed): - if not config.export_onnx: + export = config.publication.exports + if config.publication.mode != "supervised" or not export.onnx: return execution = ModelExecution.from_config(config) interfaces = { @@ -146,9 +148,9 @@ def ensure_export_preflight(config, distributed): json.dumps( { "execution": contract, - "generative": config.export_generative_model, - "embedding": config.export_embedding_model, - "dropout": config.export_with_dropout, + "generative": export.generative_model, + "embedding": export.embedding_model, + "dropout": export.with_dropout, "seed": PREFLIGHT_SEED, } ) @@ -213,7 +215,7 @@ def _worker(request_path): request_path.parent / f"{index}-generative.onnx", interface_name=name, batch_size=2, - context_length=config.runtime_settings.context_length, + context_length=config.options.context_length, training=request["dropout"], ) if request["embedding"]: @@ -224,7 +226,7 @@ def _worker(request_path): request_path.parent / f"{index}-embedding.onnx", interface_name=None, batch_size=2, - context_length=config.runtime_settings.context_length, + context_length=config.options.context_length, training=request["dropout"], ) print( diff --git a/src/sequifier/export/service.py b/src/sequifier/export/service.py index cc06b630..0f8d3cb2 100644 --- a/src/sequifier/export/service.py +++ b/src/sequifier/export/service.py @@ -66,17 +66,7 @@ def _export( return ExportResult() del network execution = ModelExecution.from_config(self.config) - export_config = replace( - execution, - device="cpu", - runtime_settings=execution.runtime_settings.model_copy( - update={ - "distributed": False, - "data_parallelism": None, - "torch_compile": "none", - } - ), - ) + export_config = replace(execution, device="cpu") export_network = build_transformer_network( export_config, device=torch.device("cpu"), initialize=False ).network @@ -86,7 +76,8 @@ def _export( validate_hash_contracts(export_network, export_config) export_network.eval() paths: list[Path] = [] - if self.config.export_pt and self.config.export_generative_model: + export = self.config.publication.exports + if export.pt and export.generative_model: path = model_artifact_path( self.config.project_root, self.config.model_name, @@ -97,7 +88,7 @@ def _export( export_network, self.config, path, options ) paths.append(exported_path) - if self.config.export_pt and self.config.export_embedding_model: + if export.pt and export.embedding_model: embedding_options = ExportOptions( f"{options.suffix}-embedding", options.epoch ) @@ -112,9 +103,9 @@ def _export( export_network, self.config, path, embedding_options ) ) - if self.config.export_onnx: + if export.onnx: contract = model_execution_config(execution) - settings = execution.runtime_settings + settings = execution.options datasets = self.config.dataset_training dataset_count = len(datasets) for dataset_name, dataset in datasets.items(): @@ -128,7 +119,7 @@ def _export( interface.target_decoder_ids ), } - if self.config.export_generative_model: + if export.generative_model: path = model_artifact_path( self.config.project_root, self.config.model_name, @@ -143,13 +134,13 @@ def _export( interface, path, interface_name=name, - batch_size=settings.inference_batch_size, + batch_size=export.batch_size, context_length=settings.context_length, - training=self.config.export_with_dropout, + training=export.with_dropout, metadata=metadata, ) ) - if self.config.export_embedding_model: + if export.embedding_model: embedding = self.embedding.build(export_network, name, execution) path = model_artifact_path( self.config.project_root, @@ -165,9 +156,9 @@ def _export( interface, path, interface_name=None, - batch_size=settings.inference_batch_size, + batch_size=export.batch_size, context_length=settings.context_length, - training=self.config.export_with_dropout, + training=export.with_dropout, metadata={ **metadata, ONNX_EMBEDDING_LAYER_NAMES_KEY: json.dumps( diff --git a/src/sequifier/model/execution.py b/src/sequifier/model/execution.py index 5cdc86cf..21c0828e 100644 --- a/src/sequifier/model/execution.py +++ b/src/sequifier/model/execution.py @@ -3,22 +3,21 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from sequifier.config.train_config import ( - ModelSpecModel, - ResolvedModelInterface, - ResolvedSequifierConfig, - TrainingRuntimeSettings, - ) +from sequifier.config.train_config import ( + ModelSpecModel, + ResolvedInterfaceStorage, + ResolvedModelInterface, + ResolvedSequifierConfig, +) +from sequifier.model.execution_options import ModelExecutionOptions @dataclass class ModelExecution: model: ModelSpecModel interfaces: dict[str, ResolvedModelInterface] - runtime_settings: TrainingRuntimeSettings + options: ModelExecutionOptions device: str = "cpu" project_root: str = "." model_name: str = "loaded-model" @@ -27,16 +26,19 @@ class ModelExecution: default_factory=lambda: ["backbone.final_norm"] ) + interface_storage: dict[str, ResolvedInterfaceStorage] = field(default_factory=dict) + @classmethod def from_config( cls, config: ResolvedSequifierConfig | ModelExecution ) -> ModelExecution: - if isinstance(config, cls): + if isinstance(config, ModelExecution): return config return cls( model=config.model, interfaces=config.interfaces, - runtime_settings=config.runtime_settings, + options=config.model_options, + interface_storage=config.interface_storage, device=config.device, project_root=config.project_root, model_name=config.model_name, diff --git a/src/sequifier/model/execution_options.py b/src/sequifier/model/execution_options.py new file mode 100644 index 00000000..f6f4df63 --- /dev/null +++ b/src/sequifier/model/execution_options.py @@ -0,0 +1,80 @@ +"""Model-owned options shared by training models and portable artifacts.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import Field, field_validator + +from sequifier.config.naming import ConfigModel + +DEFAULT_MAX_CONCAT_TENSORS = 12 + + +class _SharedExecutionOptions(ConfigModel): + target_offset: int = Field(default=1, ge=0) + forward_attention: Literal["causal", "bidirectional"] | None = None + layer_type_dtypes: dict[str, str] | None = None + max_concat_tensors: int = Field(default=DEFAULT_MAX_CONCAT_TENSORS, gt=0) + + @field_validator("training_objective", check_fields=False) + @classmethod + def validate_objective(cls, value: str | None) -> str | None: + if value is not None: + from sequifier.objectives import get_objective_class + + get_objective_class(value) + return value + + @field_validator("layer_type_dtypes") + @classmethod + def validate_layer_dtypes(cls, value: dict[str, str] | None): + if value is None: + return value + allowed_keys = {"embedding", "linear", "conv", "norm", "decoder"} + allowed_types = { + "float32", + "float16", + "bfloat16", + "float64", + "float8_e4m3fn", + "float8_e5m2", + } + if invalid := set(value) - allowed_keys: + raise ValueError(f"Invalid layer_type_dtypes keys: {sorted(invalid)}") + if invalid := set(value.values()) - allowed_types: + raise ValueError(f"Invalid layer_type_dtypes values: {sorted(invalid)}") + return value + + +class ExecutionOptions(_SharedExecutionOptions): + context_length: int | None = Field(default=None, gt=0) + training_objective: str | None = None + + +class ModelExecutionOptions(_SharedExecutionOptions): + context_length: int = Field(gt=0) + training_objective: str + + @property + def attention(self) -> Literal["causal", "bidirectional"]: + return self.forward_attention or ( + "bidirectional" if self.training_objective == "bert" else "causal" + ) + + @classmethod + def from_training(cls, settings: ExecutionOptions) -> ModelExecutionOptions: + context_length = settings.context_length + training_objective = settings.training_objective + if context_length is None or training_objective is None: + raise ValueError( + "Model execution requires context_length and training_objective" + ) + return cls( + context_length=context_length, + training_objective=training_objective, + target_offset=settings.target_offset, + forward_attention=settings.forward_attention, + layer_type_dtypes=settings.layer_type_dtypes, + max_concat_tensors=settings.max_concat_tensors, + ) diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index d9a78ae4..ac0c3dd4 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -20,7 +20,7 @@ from sequifier.model.layers import RMSNorm from sequifier.model.network import ComposableTransformerNetwork, ModelInterfaceModule from sequifier.model.static_conditioning import StaticConditionEncoder -from sequifier.objectives import BERTObjective, CausalObjective, create_objective +from sequifier.objectives import BERTObjective, CausalObjective from sequifier.special_tokens import resolve_categorical_decoder_ids from sequifier.typechecking import beartype @@ -62,7 +62,6 @@ def target_global_to_decoder(self) -> dict[str, list[int]]: @dataclass class BuiltModel: network: ComposableTransformerNetwork - objectives: dict[str, Any] runtime_metadata: ModelRuntimeMetadata @@ -122,7 +121,7 @@ def compile_unique_layers(layers: nn.ModuleList) -> None: @beartype def _apply_layer_dtypes(network: nn.Module, config: Any) -> None: - layer_config = config.runtime_settings.layer_type_dtypes + layer_config = config.options.layer_type_dtypes if not layer_config: return for name, module in network.named_modules(): @@ -267,16 +266,6 @@ def _build_composable_network( if encoder_interface is None else {encoder_interface: config.interface(encoder_interface)} ) - objectives = {} - for name, interface in interfaces.items(): - objective_view = interface_build_view(config, interface) - # Next-occurrence target construction belongs to each training phase; - # its forward attention policy is the ordinary causal policy. - if config.runtime_settings.training_objective == "next_occurrence": - objectives[name] = CausalObjective(objective_view) - else: - objectives[name] = create_objective(objective_view) - objective = next(iter(objectives.values())) backbone_seed = config.model.backbone.initialization_seed with isolated_initialization(derived_seed(backbone_seed, (), "constructor")): backbone = TransformerBackbone(config.model.backbone.architecture) @@ -297,7 +286,7 @@ def _build_composable_network( direct_real_dtype_provider=lambda: backbone.layers[ 0 ].ff.get_first_layer_dtype(), - max_concat_tensors=config.runtime_settings.max_concat_tensors, + max_concat_tensors=config.options.max_concat_tensors, load_pretrained=load_pretrained, ) from sequifier.model.ingestions import EmbeddingFeatureIngestion @@ -443,23 +432,19 @@ def _build_composable_network( interface.ingestion.initialization, ) - attention_objective = objective - if config.runtime_settings.forward_attention is not None: - attention_objective = ( - CausalObjective - if config.runtime_settings.forward_attention == "causal" - else BERTObjective - ) + attention_objective = ( + CausalObjective if config.options.attention == "causal" else BERTObjective + ) network = ComposableTransformerNetwork( backbone=backbone, interfaces=routes, attention_mask_policy=attention_objective.build_attention_mask_policy( - config.runtime_settings.context_length + config.options.context_length ), attention_mask_policy_with_prefix=attention_objective.build_attention_mask_policy( - config.runtime_settings.context_length + 1 + config.options.context_length + 1 ), - context_length=config.runtime_settings.context_length, + context_length=config.options.context_length, ) if initialize: if len(config.interfaces) == 1: @@ -504,7 +489,6 @@ def _build_composable_network( network.to(device) return BuiltModel( network=network, - objectives=objectives, runtime_metadata=ModelRuntimeMetadata( interfaces=runtime_metadata, device=device, diff --git a/src/sequifier/model/history_transfer.py b/src/sequifier/model/history_transfer.py index 62fbad20..e5976681 100644 --- a/src/sequifier/model/history_transfer.py +++ b/src/sequifier/model/history_transfer.py @@ -27,7 +27,7 @@ def _history_contract(config, interface_name): if interface_name not in execution.interfaces: raise ValueError(f"history transfer requires interface {interface_name!r}") interface = execution.interface(interface_name) - settings = execution.runtime_settings + settings = execution.options static = interface.static_conditioning columns = interface.input_columns + (static.columns if static else []) media_fields = { @@ -41,8 +41,7 @@ def _history_contract(config, interface_name): "backbone.architecture": execution.model.backbone.architecture, "layer_type_dtypes": settings.layer_type_dtypes, "context_length": settings.context_length, - "forward_attention": settings.forward_attention - or ("bidirectional" if settings.training_objective == "bert" else "causal"), + "forward_attention": settings.attention, "inputs": ExecutionSchema.from_interface( interface, settings.context_length ).inputs, diff --git a/src/sequifier/rl/attachment.py b/src/sequifier/rl/attachment.py index fc91aa3a..a44fcc0b 100644 --- a/src/sequifier/rl/attachment.py +++ b/src/sequifier/rl/attachment.py @@ -7,14 +7,10 @@ def resolve_binding(execution, interface_name, binding): interface = execution.interface(interface_name) - schema = ExecutionSchema.from_interface( - interface, execution.runtime_settings.context_length - ) + schema = ExecutionSchema.from_interface(interface, execution.options.context_length) if ( -binding.decision_position - > execution.runtime_settings.context_length - - interface.decoder.support_length - + 1 + > execution.options.context_length - interface.decoder.support_length + 1 ): raise ValueError("decision_position lies outside the source decoder output") values = binding.model_dump() @@ -75,7 +71,7 @@ def resolve_binding(execution, interface_name, binding): for name, stats in interface.selected_columns_statistics.items() if name in set(temporal) | set(static) and interface.normalize_real_columns }, - context_length=execution.runtime_settings.context_length, + context_length=execution.options.context_length, interface_name=interface_name, ) for column in binding.previous_action_inputs: @@ -102,12 +98,18 @@ def attach_policy(network, execution, interface_name, binding): policy = resolve_binding(execution, interface_name, binding) apply_binding(execution, interface_name, policy) actor = Policy.from_network(network, execution, interface_name) - actor.transfer_report = { + actor.transfer_report = attachment_report(actor, original_parameters) + return actor + + +def attachment_report(actor, original_parameters): + policy = actor.config + return { "mode": "adapt" - if binding.previous_action_inputs + if policy.previous_action_inputs or any( field.lower is not None or field.upper is not None - for field in binding.actions.values() + for field in policy.actions.values() ) or any( field.type == "categorical" @@ -116,15 +118,15 @@ def attach_policy(network, execution, interface_name, binding): for name, field in policy.actions.items() ) else "attach", - "interface": interface_name, + "interface": actor.interface_name, "reused_parameters": [ name - for name, _ in network.named_parameters() + for name, _ in actor.network.named_parameters() if name in original_parameters ], "new_parameters": [ name - for name, _ in network.named_parameters() + for name, _ in actor.network.named_parameters() if name not in original_parameters ], "source_preprocessing": "preserved", @@ -133,4 +135,3 @@ def attach_policy(network, execution, interface_name, binding): for name, field in policy.actions.items() }, } - return actor diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 3b980212..70f2222c 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -13,7 +13,7 @@ from sequifier.artifacts.reports import phase_report, publish_run_report, run_report from sequifier.fingerprints import fingerprint from sequifier.helpers import configure_determinism, configure_logger -from sequifier.rl.checkpoint import load_policy, save_policy, validate_policy_weights +from sequifier.rl.checkpoint import save_policy, validate_policy_weights from sequifier.rl.preparation import ( build_actor, preflight_environments, @@ -52,7 +52,15 @@ def _plan_contract(config): + from pydantic_core import to_jsonable_python + + from sequifier.artifacts.model_export import model_execution_config + values = config.model_dump(mode="json") + values["execution_model"] = to_jsonable_python( + model_execution_config(config.execution) + ) + values["model"] = config.execution.model.model_dump(mode="json") values.pop("resume") values.pop("checkpoint_path", None) if config.actor_initialization == "attach": @@ -79,16 +87,6 @@ def _publish_phase_report(record, checkpoint): record.commit.publish(artifacts, _phase_report(result, checkpoint)) -def _new_actor(config): - return build_actor( - config, - config.policy, - config.execution_model["interfaces"][config.policy_interface].get( - "selected_columns_statistics", {} - ), - ) - - def _initialize_history_actor(config): from sequifier.artifacts.model_artifact import load_execution_model from sequifier.model.history_transfer import ( @@ -101,7 +99,7 @@ def _initialize_history_actor(config): if not config.initialize_from: raise ValueError("history_only requires initialize_from") - actor = _new_actor(config) + actor = build_actor(config) name = actor.interface_name decoder = actor.network.resolve_interface(name).decoder history_ids = history_parameter_ids(actor.network, name) @@ -137,24 +135,16 @@ def _initialize_actor(config): if config.actor_initialization == "history_only": return _initialize_history_actor(config) if config.initialize_from: - source = load_policy( - config.initialize_from, - device=config.device, - interface_name=config.policy_interface, - binding=config.policy, - model=config.model, - ) - validate_declared_policy(config.policy, source.config, "initialize_from.policy") + from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.artifacts.model_config import validate_execution_attachment - from sequifier.artifacts.model_export import model_execution_config + from sequifier.rl.attachment import attachment_report - validate_execution_attachment( - config.execution_model, model_execution_config(source.execution_config) - ) - source.execution_config.embedding_layer_names = config.embedding_layer_names - source.units = dict(config.units) - return source - return _new_actor(config) + artifact = read_execution_artifact(config.initialize_from) + validate_execution_attachment(artifact.model_config.values, config.execution) + actor = build_actor(config, artifact=artifact) + actor.transfer_report = attachment_report(actor, artifact.model_state_dict) + return actor + return build_actor(config) def run_policy_training( @@ -233,9 +223,7 @@ def _execute_policy(config, prepared, group, identity, on_checkpoint): representation = recovery.representation validate_declared_policy(config.policy, representation.policy, "resume.policy") data = prepare_data(config, representation.policy, inputs) - actor = build_actor( - config, representation.policy, representation.values["statistics"] - ) + actor = build_actor(config, representation.policy) recovery.validate_actor(actor, config.units) else: data = prepare_data(config, config.policy, inputs) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index 462eb1a5..ddec4f67 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -22,31 +22,6 @@ from sequifier.model.network import ComposableTransformerNetwork -def policy_execution_config( - config, - device="cpu", - *, - execution_model=None, - project_root=".", - model_name="loaded-model", - seed=0, -): - """Select policy alignment over canonical model-owned execution facts.""" - from sequifier.artifacts.model_config import model_execution_from_model_config - from sequifier.rl.attachment import apply_binding - - if execution_model is None: - raise ValueError("joint policies require a resolved model execution contract") - execution, _ = model_execution_from_model_config( - execution_model, device=device, interface_name=config.interface_name - ) - apply_binding(execution, config.interface_name, config) - execution.project_root = project_root - execution.model_name = model_name - execution.seed = seed - return execution - - class Policy(nn.Module): """A convenience view; the network owns all trainable weights and execution.""" diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index f7216e07..102652bc 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -1,5 +1,6 @@ """Read-only policy run preparation and whole-plan preflight validation.""" +from dataclasses import replace from pathlib import Path import torch @@ -11,7 +12,7 @@ validate_path_conflicts, ) from sequifier.io.prepared_policy_data import PreparedPolicyData -from sequifier.model.factory import build_transformer_network +from sequifier.model.factory import build_transformer_network, validate_hash_contracts from sequifier.rl.checkpoint import phase_actor_path, validate_policy_weights from sequifier.rl.environment import ( adapter_identity, @@ -19,7 +20,7 @@ require_exact_snapshot, require_group_start, ) -from sequifier.rl.policy import Policy, policy_execution_config +from sequifier.rl.policy import Policy from sequifier.rl.ppo_evaluation import require_evaluation_timer from sequifier.rl.prepared_inputs import PreparedTrainingInputs, RunInputs # noqa: F401 from sequifier.rl.run_state import require_equal @@ -86,29 +87,33 @@ def validate_declared_policy(declared, fitted, field): require_equal(expected, actual, field) -def build_actor(config, fitted, statistics): - execution = policy_execution_config( - fitted, - config.device, - execution_model=config.execution_model, - project_root=config.project_root, - model_name=config.model_name, - seed=config.seed, +def build_actor(config, binding=None, *, artifact=None): + execution = config.execution + name = config.policy_interface + # A runner owns its mutable policy view. Recovery supplies its saved binding. + interface = execution.interface(name).model_copy( + update={"policy": binding or config.policy} ) - execution.embedding_layer_names = config.embedding_layer_names - actor = Policy.from_network( - build_transformer_network( - execution, - device=torch.device(config.device), - restored_history_interface=( - config.policy_interface - if config.actor_initialization == "history_only" - else None - ), - ).network, + # Exact recovery depends on the selected route being built first. + interfaces = {name: interface} + interfaces.update( + (other, value) for other, value in execution.interfaces.items() if other != name + ) + execution = replace(execution, interfaces=interfaces) + network = build_transformer_network( execution, - config.policy_interface, - ).eval() + device=torch.device(config.device), + initialize=artifact is None, + restored_history_interface=name + if config.actor_initialization == "history_only" + else None, + ).network + if artifact is not None: + from sequifier.artifacts.model_config import load_attached_model_weights + + load_attached_model_weights(network, artifact) + validate_hash_contracts(network, execution) + actor = Policy.from_network(network, execution, name).eval() actor.units = dict(config.units) validate_policy_weights(actor) return actor diff --git a/src/sequifier/rl/references.py b/src/sequifier/rl/references.py index 8a49eb61..e9c9da74 100644 --- a/src/sequifier/rl/references.py +++ b/src/sequifier/rl/references.py @@ -65,7 +65,7 @@ def _artifact(self, path): device="cpu", interface_name=self.config.policy_interface, binding=self.config.policy, - model=self.config.model, + model=self.config.execution.model, ) self.recovery.representation.validate_actor( source, self.config.units, f"reference.{path}" diff --git a/src/sequifier/rl/run_config.py b/src/sequifier/rl/run_config.py index cef4b167..807e6518 100644 --- a/src/sequifier/rl/run_config.py +++ b/src/sequifier/rl/run_config.py @@ -9,7 +9,7 @@ from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_plan import PolicyPhase from sequifier.config.policy_train_config import SequenceSourceConfig -from sequifier.config.train_config import ModelSpecModel +from sequifier.model.execution import ModelExecution class ResolvedPolicyRun(BaseModel): @@ -22,9 +22,8 @@ class ResolvedPolicyRun(BaseModel): device: str seed: int enforce_determinism: bool = False - model: ModelSpecModel policy: ResolvedPolicyBinding - execution_model: dict + execution: ModelExecution = Field(exclude=True) policy_interface: str initialize_from: str | None = None metadata_paths: list[str] = Field(default_factory=list) diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 3c1776b2..6969b6f8 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -66,6 +66,8 @@ def from_actor(cls, actor, units): from sequifier.artifacts.model_export import model_execution_config + model = actor.execution_config.model.model_dump(mode="json") + model["backbone"]["repository"] = None return cls( json.dumps( { @@ -73,7 +75,7 @@ def from_actor(cls, actor, units): model_execution_config(actor.execution_config) ), "policy": actor.config.model_dump(mode="json"), - "model": actor.execution_config.model.model_dump(mode="json"), + "model": model, "statistics": actor.interface.selected_columns_statistics, "units": dict(getattr(actor, "units", units)), }, diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index c53c9219..954c149c 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -187,13 +187,9 @@ def build( from sequifier.artifacts.run_checkpoint import ( checkpoint_path as output_checkpoint_path, ) - from sequifier.io.output_paths import ( - resolve_training_output_paths, - validate_training_destinations, - ) + from sequifier.io.output_paths import validate_training_destinations config = config.model_copy() - resolve_training_output_paths(config) precision = PrecisionPolicy.from_settings( config.runtime_settings, execution.device ) @@ -266,13 +262,13 @@ def build( network = built.network if loaded is None and initial_artifact is not None: from sequifier.artifacts.model_artifact import read_execution_artifact - from sequifier.artifacts.model_export import model_execution_config + from sequifier.model.execution import ModelExecution artifact = read_execution_artifact(initial_artifact) from sequifier.artifacts.model_config import validate_execution_attachment validate_execution_attachment( - artifact.model_config.values, model_execution_config(config) + artifact.model_config.values, ModelExecution.from_config(config) ) from sequifier.artifacts.model_config import load_attached_model_weights @@ -345,7 +341,6 @@ def dataset_factory(view: SupervisedPhaseView) -> DatasetRuntimeRegistry: parts=parts, metrics=dataset_metrics, runtime_metadata=built.runtime_metadata, - default_objectives=built.objectives, masks=freezing_plans[view.index].bind(network), ) diff --git a/src/sequifier/train.py b/src/sequifier/train.py index b86e8d34..3b152c50 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -295,7 +295,7 @@ def _tensor_batches( column_data_types: dict[str, torch.dtype], ): categorical = set(model.categorical_columns) - layer_types = model.config.runtime_settings.layer_type_dtypes or {} + layer_types = model.config.options.layer_type_dtypes or {} reference_dtype = get_torch_dtype(layer_types.get("linear", "float32")) for index, features in enumerate(x): values = { diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index b5bc91a6..cdf22d28 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -104,6 +104,8 @@ def strip_provenance(value: Any) -> Any: phases = [phase.model_dump(mode="python") for phase in config.training_plan] for phase in phases: phase.pop("trainability_summary", None) + for source in phase["sources"] + phase["evaluation_sources"]: + source.pop("parts", None) for role in phase["optimization"].values(): if role["scheduler"]["name"] == "OneCycleLR": # Recomputed from the phase/data budget before optimizer construction. diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index c1d1253a..3bddf941 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -28,10 +28,9 @@ ) from sequifier.artifacts.phase import PhaseArtifacts, PhaseCommit, verify_artifact from sequifier.artifacts.reports import phase_report, publish_run_report, run_report -from sequifier.artifacts.run_checkpoint import checkpoint_path +from sequifier.artifacts.run_checkpoint import checkpoint_path, training_plan_identity from sequifier.config.components import ResumeConfig from sequifier.contract_versions import ContractVersions, require_contract_version -from sequifier.fingerprints import fingerprint from sequifier.io.atomic import atomic_path, atomic_save from sequifier.io.output_paths import validate_training_destinations from sequifier.runtime.plan import ( @@ -40,7 +39,6 @@ continuation_groups, new_run_id, run_phases, - validate_continuation, ) from sequifier.training.phase_runners import ( PortableGroup, @@ -70,13 +68,10 @@ def coordinate_training( ): """Choose native execution and recovery services for the resolved plan.""" from sequifier.config.train_config import ResolvedSequifierConfig - from sequifier.io.output_paths import resolve_training_output_paths if not isinstance(config, ResolvedSequifierConfig): raise TypeError("Training requires a canonical resolved config.") - resolve_training_output_paths(config) - phases = config.training_plan - if all(phase.algorithm == "supervised" for phase in phases): + if config.publication.mode == "supervised": return launch_supervised( config, integration_specs=integration_specs, @@ -87,9 +82,7 @@ def coordinate_training( raise ValueError( "policy phases do not support custom integrations or parameter grouping" ) - if all(phase.algorithm != "supervised" for phase in phases): - return run_policy_plan(config) - return run_learning_plan(config) + return run_policy_plan(config) def run_policy_plan(config): @@ -97,7 +90,7 @@ def run_policy_plan(config): from sequifier.config.training_resolution import policy_run_config from sequifier.rl.learner import run_policy_training - if len({phase.model_interface for phase in config.training_plan}) == 1: + if config.publication.mode == "policy": return run_policy_training(policy_run_config(config, config.training_plan)) return run_learning_plan(config) @@ -168,7 +161,7 @@ def _prepare_plan(config, prepared, groups, revision, state): for index, phases in groups: phase = phases[0] if phase.algorithm == "supervised": - native = supervised_group_config(config, phases, index, None, False) + native = supervised_group_config(config, phases, index, False) manifest.merge(supervised_dependencies(native)) supervised_runs.append(native) continue @@ -209,19 +202,10 @@ def _prepare_plan(config, prepared, groups, revision, state): def _run_learning_plan(config, prepared): phases = config.training_plan - if not phases: - raise ValueError("A learning plan requires at least one phase") - for index, phase in enumerate(phases): - validate_continuation( - phases[index - 1] if index else None, phase, resolved=True - ) groups = [(index, tuple(group)) for index, group in continuation_groups(phases)] resume = config.runtime_settings.resume or ResumeConfig() checkpoint = checkpoint_path(config) - plan = config.model_dump(mode="json") - plan["runtime_settings"].pop("resume", None) - plan.get("ownership", {}).get("run", {}).get("recovery", {}).pop("resume", None) - plan_identity = fingerprint(plan) + plan_identity = training_plan_identity(config) state = { "artifact_type": "sequifier_learning_run", "format_version": ContractVersions.LEARNING_RUN_STATE, diff --git a/src/sequifier/training/phase_runners.py b/src/sequifier/training/phase_runners.py index ec4f4fb6..30e5c1dc 100644 --- a/src/sequifier/training/phase_runners.py +++ b/src/sequifier/training/phase_runners.py @@ -1,11 +1,10 @@ """Adapt resolved phases to native execution and exact recovery.""" from collections.abc import Callable -from dataclasses import dataclass, replace +from dataclasses import dataclass import torch -from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.artifacts.phase import PhaseArtifacts from sequifier.runtime.plan import RunIdentity @@ -20,31 +19,17 @@ class PortableGroup: on_checkpoint: Callable -def supervised_group_config(config, phases, index, incoming, resume): +def supervised_group_config(config, phases, index, resume): phase = phases[0] - from sequifier.artifacts.model_config import ( - extend_prediction_distributions, - model_execution_from_model_config, - validate_execution_attachment, - ) - from sequifier.artifacts.model_export import model_execution_config from sequifier.config.components import ResumeConfig - from sequifier.model.execution import ModelExecution - - training = config.runtime_settings.model_copy(deep=True) - training.bert_spec = phase.learning.bert_spec - training.next_occurrence_config = phase.learning.next_occurrence_config - training.window_stride = phase.learning.window_stride - if training.distributed: - raise ValueError("mixed learning plans currently require local phase execution") + native = PhaseArtifacts.for_phase( config.phase_directory, index, phase.name ).model.parent - training.resume = ResumeConfig( + resume = ResumeConfig( policy="required" if resume else "never", checkpoint_path=str(native / "run.pt"), ) - training.log_interval_iterations = 10 names = { ref.dataset for item in phases @@ -55,45 +40,21 @@ def supervised_group_config(config, phases, index, incoming, resume): for name, dataset in config.dataset_training.items() if name in names } - if datasets.keys() != names or any( - not dataset.parts for dataset in datasets.values() - ): - raise ValueError(f"{phase.name}: supervised phase requires resolved datasets") - - model = config.model - interfaces = config.interfaces - artifact = read_execution_artifact(incoming) if incoming is not None else None - if artifact is not None: - source, _ = model_execution_from_model_config( - artifact.model_config.values, device=config.device - ) - extend_prediction_distributions(source, config.model) - model = source.model - interfaces = source.interfaces - training.forward_attention = source.runtime_settings.forward_attention or ( - "bidirectional" - if source.runtime_settings.training_objective == "bert" - else "causal" - ) - planned = replace(ModelExecution.from_config(config), runtime_settings=training) - validate_execution_attachment( - artifact.model_config.values, model_execution_config(planned) - ) - resolved = config.model_copy( update={ - "model": model, - "interfaces": interfaces, "dataset_training": datasets, - "training_plan": [item.model_copy(deep=True) for item in phases], - "runtime_settings": training, - "seed": config.seed + index, + "training_plan": list(phases), + "runtime_settings": config.runtime_settings.model_copy( + update={"resume": resume} + ), + "seed": phase.resolved_seed + if phase.resolved_seed is not None + else config.seed + index, "model_name": f"{config.model_name}-phase-{index:03d}-{phase.name}", "initialize_from": None, "actor_initialization": "attach", "curriculum_training": bool(phase.curriculum), "curriculum_column": phase.curriculum.column if phase.curriculum else None, - "export_pt": True, "model_path": str(native / "native/model.pt"), "report_path": str(native / "native/report.json"), "phase_directory": str(native / "native/phases"), @@ -110,7 +71,7 @@ def run_supervised_group(config, request: PortableGroup, backbone_revision): from sequifier.training.engine import TrainingEngine resolved = supervised_group_config( - config, request.phases, request.index, request.incoming, request.resuming + config, request.phases, request.index, request.resuming ) torch.set_float32_matmul_precision( resolved.runtime_settings.float32_matmul_precision diff --git a/src/sequifier/training/plan.py b/src/sequifier/training/plan.py index b334b97c..04b50b0f 100644 --- a/src/sequifier/training/plan.py +++ b/src/sequifier/training/plan.py @@ -88,7 +88,13 @@ def from_config(cls, config: ResolvedSequifierConfig) -> "SupervisedRunPlan": "Native supervised training requires supervised phases" ) phases.append( - SupervisedPhaseView.from_spec(index, config.seed + index, spec) + SupervisedPhaseView.from_spec( + index, + spec.resolved_seed + if spec.resolved_seed is not None + else config.seed + index, + spec, + ) ) return cls(tuple(phases)) diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index 1d4d86e1..e528392a 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -18,6 +18,7 @@ ResolvedSequifierConfig, ResolvedTrainingPhase, ResolvedTrainingSource, + SelectedObjectiveConfig, dataset_part_view, interface_build_view, ) @@ -37,7 +38,7 @@ ) from sequifier.model.network import ComposableTransformerNetwork from sequifier.model.parameter_catalog import ParameterCatalog -from sequifier.objectives import Objective, create_objective +from sequifier.objectives import CausalObjective, Objective, create_objective from sequifier.training.accurate_cross_entropy import AccurateCrossEntropyLoss from sequifier.training.plan import ( PhaseLoading, @@ -501,7 +502,7 @@ def build_dataset_parts( (source.dataset, part) for phase in plan.phases for source in phase.spec().sources - for part in source_part_names(source, config.dataset_training[source.dataset]) + for part in source.parts } return { name: { @@ -525,7 +526,6 @@ def build_dataset_runtimes( parts: dict[str, dict[str, DatasetPartRuntime]], metrics: dict[str, DatasetMetrics], runtime_metadata: Any, - default_objectives: dict[str, Objective], masks: dict[str, frozenset[int]], ) -> DatasetRuntimeRegistry: spec = phase.spec() @@ -534,12 +534,14 @@ def build_dataset_runtimes( for name, base in config.dataset_training.items(): dataset = dataset_learning_view(base, spec) interface = config.interface(dataset.model_interface) + objective_view = SelectedObjectiveConfig( + **vars(interface_build_view(config, interface)), runtime_settings=settings + ) objective = ( - create_objective( - interface_build_view(config, interface, runtime_settings=settings) - ) - if name in spec.learning.datasets - else default_objectives[dataset.model_interface] + CausalObjective(objective_view) + if name not in spec.learning.datasets + and objective_view.training_objective == "next_occurrence" + else create_objective(objective_view) ) runtimes[name] = DatasetRuntime( name=name, @@ -558,21 +560,13 @@ def build_dataset_runtimes( return DatasetRuntimeRegistry(runtimes) -def source_part_names( - source: ResolvedTrainingSource, dataset: ResolvedDatasetTrainingSpec -) -> tuple[str, ...]: - return (source.part,) if source.part is not None else tuple(dataset.parts) - - def build_source_runtime( source: ResolvedTrainingSource, datasets: DatasetRuntimeRegistry, phase: SupervisedPhaseView, ) -> TrainingSourceRuntime: dataset = datasets.resolve(source.dataset) - return TrainingSourceRuntime( - source, dataset, source_part_names(source, dataset.config), phase - ) + return TrainingSourceRuntime(source, dataset, source.parts, phase) class SourceScheduler: diff --git a/src/sequifier/training/scheduler_budgets.py b/src/sequifier/training/scheduler_budgets.py index 50299196..70bfe92e 100644 --- a/src/sequifier/training/scheduler_budgets.py +++ b/src/sequifier/training/scheduler_budgets.py @@ -10,7 +10,7 @@ ) from sequifier.io.batch import SequifierBatch from sequifier.training.plan import SupervisedRunPlan -from sequifier.training.runtime import RuntimeBatch, SourceScheduler, source_part_names +from sequifier.training.runtime import RuntimeBatch, SourceScheduler @dataclass @@ -48,7 +48,7 @@ def _phase_update_budget(config, parts, view): for source in phase.sources: dataset = config.dataset_training[source.dataset] batches = 0 - for part in source_part_names(source, dataset): + for part in source.parts: key = (source.dataset, part) if key not in counts: counts[key] = parts[source.dataset][ From 4f08da21db92b104a4b0038bab73328bd0ba7afb Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 19:13:19 +0100 Subject: [PATCH 59/71] refactor phase execution --- src/sequifier/artifacts/run_checkpoint.py | 19 +- src/sequifier/config/training_resolution.py | 38 ++-- src/sequifier/io/episode_corpus.py | 49 +++-- src/sequifier/io/episode_features.py | 18 +- src/sequifier/io/iql_dataset.py | 3 - src/sequifier/io/output_paths.py | 6 +- src/sequifier/io/policy_dataset.py | 46 +++-- src/sequifier/io/preference_dataset.py | 54 +++--- src/sequifier/io/prepared_policy_data.py | 2 - src/sequifier/rl/checkpoint.py | 15 +- src/sequifier/rl/dpo.py | 2 +- src/sequifier/rl/group_collection.py | 4 +- src/sequifier/rl/iql.py | 24 +-- src/sequifier/rl/learner.py | 88 +++++---- src/sequifier/rl/offline_data.py | 36 ++-- src/sequifier/rl/phase_optimization.py | 83 +++++++++ src/sequifier/rl/ppo.py | 14 +- src/sequifier/rl/preparation.py | 71 +++----- src/sequifier/rl/prepared_inputs.py | 127 ++++++------- src/sequifier/rl/references.py | 12 +- src/sequifier/rl/run_checkpoint.py | 39 ++-- src/sequifier/rl/run_config.py | 8 +- src/sequifier/rl/run_state.py | 182 ++++++++++--------- src/sequifier/rl/training.py | 2 +- src/sequifier/runtime/builder.py | 10 +- src/sequifier/training/checkpoint_service.py | 4 +- src/sequifier/training/coordinator.py | 28 +-- src/sequifier/training/engine.py | 11 +- src/sequifier/training/phase_runners.py | 5 +- src/sequifier/training/state.py | 3 +- 30 files changed, 554 insertions(+), 449 deletions(-) create mode 100644 src/sequifier/rl/phase_optimization.py diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 3a19f974..7dec0a01 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -181,7 +181,10 @@ def phase_boundary(self): state = self.run_state completed = state["phase_commits"] active = ( - {"index": state["phase_index"], "transaction": state["optimizer_step"]} + { + "index": state.get("phase_offset", 0) + state["phase_index"], + "transaction": state["optimizer_step"], + } if len(completed) <= state["phase_index"] else None ) @@ -273,21 +276,29 @@ def __init__(self, training_config: Any, model_name: str) -> None: self.phase_directory = training_config.phase_directory self.phases = training_config.training_plan - def path_for(self, suffix: str | None, phase_index: int = 0) -> Path: + def path_for( + self, suffix: str | None, phase_index: int = 0, *, phase_offset: int = 0 + ) -> Path: if suffix in {None, "latest"}: return self.latest_path from sequifier.artifacts.phase import PhaseArtifacts directory = ( PhaseArtifacts.for_phase( - self.phase_directory, phase_index, self.phases[phase_index].name + self.phase_directory, + phase_offset + phase_index, + self.phases[phase_index].name, ).model.parent / "checkpoints" ) return directory / f"{self.model_name}-{suffix}.pt" def save(self, checkpoint: RunCheckpoint, suffix: str | None) -> Path: - output_path = self.path_for(suffix, checkpoint.run_state["phase_index"]) + output_path = self.path_for( + suffix, + checkpoint.run_state["phase_index"], + phase_offset=checkpoint.run_state.get("phase_offset", 0), + ) output_path.parent.mkdir(parents=True, exist_ok=True) write_run_checkpoint(checkpoint, output_path, self.latest_path) return output_path diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 24db460a..5fad07a2 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -298,21 +298,23 @@ def _fit_input_metadata( ) inputs = prepared_inputs.resolve(run) inputs.revalidate(inputs.identity, f"interfaces.{name}.fitting_sources") - data = prepared_inputs.prepare_phase(run, phase, provisional, inputs, fitting=True) - fitting = data.fitting_dataset() - fitted = fitting.fit_categorical_config() - values = metadata.model_dump() - values["n_classes"].update(fitted.categorical_n_classes) - values["id_maps"].update(fitted.id_maps) - if ( - normalization is None - and not metadata.depth_layouts - and not metadata.image_sources - ): - fitting.use_config(fitted) - for column, statistics in fitting.normalization_statistics().items(): - values["selected_columns_statistics"].setdefault(column, statistics) - prepared_inputs.measure() + from sequifier.io.episode_features import EpisodeFeatures + + with prepared_inputs.fitting_phase(run, phase, provisional, inputs) as data: + fitting = data.fitting_dataset() + fitted = fitting.fit_categorical_config() + values = metadata.model_dump() + values["n_classes"].update(fitted.categorical_n_classes) + values["id_maps"].update(fitted.id_maps) + if ( + normalization is None + and not metadata.depth_layouts + and not metadata.image_sources + ): + features = EpisodeFeatures(fitting.episodes, fitted, fitting.lengths) + features.validate_features() + for column, statistics in features.normalization_statistics().items(): + values["selected_columns_statistics"].setdefault(column, statistics) return DatasetMetadata.model_validate(values) @@ -687,7 +689,6 @@ def policy_run_config( from sequifier.artifacts.run_checkpoint import checkpoint_path from sequifier.config.components import ResumeConfig from sequifier.model.execution import ModelExecution - from sequifier.rl.prepared_inputs import PreparedTrainingInputs from sequifier.rl.run_config import ResolvedPolicyRun names = {phase.model_interface for phase in phases} @@ -709,11 +710,12 @@ def policy_run_config( checkpoint = ( Path(config.phase_directory) / f"{index:03d}-{phases[0].name}" / "run.pt" ) - run = ResolvedPolicyRun( + return ResolvedPolicyRun( project_root=config.project_root, model_name=config.model_name, device=config.device, seed=config.seed, + phase_offset=index, enforce_determinism=training.enforce_determinism, policy=binding, execution=execution, @@ -750,5 +752,3 @@ def policy_run_config( if destination is None else str(Path(destination).parent / "native"), ) - run._prepared_inputs = PreparedTrainingInputs.for_run(config) - return run diff --git a/src/sequifier/io/episode_corpus.py b/src/sequifier/io/episode_corpus.py index 2c9f30c5..1f246500 100644 --- a/src/sequifier/io/episode_corpus.py +++ b/src/sequifier/io/episode_corpus.py @@ -29,13 +29,31 @@ class EpisodeMetadata: fingerprint: str +@dataclass(frozen=True) +class EpisodeRecords: + """Indexed source readers and metadata for one episode interpretation.""" + + sources: tuple + metadata: tuple[EpisodeMetadata, ...] + + class EpisodeCorpus: - """Own validated episode metadata and one bounded storage cache.""" + """Own one bounded decoded cache over immutable source records.""" - def __init__(self, sources, config, cache_size=8, *, cache_owner=None): + def __init__( + self, sources, config, cache_size=8, *, cache_owner=None, records=None + ): if cache_size < 1: raise ValueError("episode_cache_size must be positive") - self.sources = [ + self.cache = ( + cache_owner.episode_cache(cache_size) + if cache_owner is not None + else EpisodeCache(cache_size) + ) + self.records = records if records is not None else self._index(sources, config) + + def _index(self, sources, config): + sources = tuple( episode for source in sources for episode in ( @@ -43,17 +61,11 @@ def __init__(self, sources, config, cache_size=8, *, cache_owner=None): if isinstance(source, (SequenceSourceConfig, ResolvedSequenceSource)) else [source] ) - ] - self.cache_size = cache_size - self.cache = ( - cache_owner.episode_cache(cache_size) - if cache_owner is not None - else EpisodeCache(cache_size) ) - self.metadata = [] + metadata = [] seen = set() - for index in range(len(self.sources)): - episode = self[index] + for index in range(len(sources)): + episode = self.cache.get(index, lambda i: _load(sources[i])) if episode.episode_id in seen: raise ValueError(f"duplicate episode ID {episode.episode_id!r}") seen.add(episode.episode_id) @@ -67,11 +79,7 @@ def __init__(self, sources, config, cache_size=8, *, cache_owner=None): action_index(value, config) if episode.previous_action is not None: action_index(episode.previous_action, config) - for column, values in episode.observations.items(): - pass - for column, value in episode.static_inputs.items(): - pass - self.metadata.append( + metadata.append( EpisodeMetadata( episode.episode_id, episode.group_id, @@ -80,15 +88,16 @@ def __init__(self, sources, config, cache_size=8, *, cache_owner=None): fingerprint(episode.model_dump(mode="json")), ) ) + return EpisodeRecords(sources, tuple(metadata)) def __len__(self): - return len(self.sources) + return len(self.records.sources) def __getitem__(self, index): return self.cache.get(index, self._load_episode) def _load_episode(self, index): - return _load(self.sources[index]) + return _load(self.records.sources[index]) def view(self): return EpisodeView([(self, i) for i in range(len(self))]) @@ -112,7 +121,7 @@ def __getitem__(self, index): def metadata(self, index): corpus, position = self.entries[index] - return corpus.metadata[position] + return corpus.records.metadata[position] def subset(self, indices): return EpisodeView([self.entries[i] for i in indices]) diff --git a/src/sequifier/io/episode_features.py b/src/sequifier/io/episode_features.py index 1b5d6727..6d4b47a1 100644 --- a/src/sequifier/io/episode_features.py +++ b/src/sequifier/io/episode_features.py @@ -32,7 +32,8 @@ def __init__(self, episodes, config, lengths=None): if lengths is not None else [episodes.metadata(i).length for i in range(len(episodes))] ) - self._validated_config = self._validated_statistics = None + self._features_validated = False + self._validated_statistics = None ids = [episodes.metadata(i).episode_id for i in range(len(episodes))] if len(ids) != len(set(ids)): raise ValueError("duplicate episode IDs in fitting view") @@ -86,17 +87,10 @@ def fit_categorical_config(self) -> ResolvedPolicyBinding | PolicyFittingSpec: {**self.config.model_dump(), **values} ) - def rebind(self, config): - """Refresh feature views without changing episode membership or validating windows.""" - if self.config == config: + def validate_features(self) -> None: + if self._features_validated: return - self.config = config - self._validated_config = self._validated_statistics = None - - def use_config(self, config: ResolvedPolicyBinding | PolicyFittingSpec) -> None: - if self._validated_config == config: - return - self.rebind(config) + config = self.config # Validate held-out values against the fixed training codec before updates. for episode, length in zip(self.episodes, self.lengths): for column, values in episode.observations.items(): @@ -105,7 +99,7 @@ def use_config(self, config: ResolvedPolicyBinding | PolicyFittingSpec) -> None: for column, value in episode.static_inputs.items(): encode_feature(value, column, config) - self._validated_config = deepcopy(config) + self._features_validated = True def validate_normalization(self, statistics): """Check FP32 transforms episode-by-episode without constructing windows.""" diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py index 6b65bd94..cd600f3f 100644 --- a/src/sequifier/io/iql_dataset.py +++ b/src/sequifier/io/iql_dataset.py @@ -57,9 +57,6 @@ def __init__(self, decisions): if bootstrap and successor is None: raise ValueError(f"{location}: bootstrap needs the true successor") - def use_config(self, config): - self.view = PolicyWindowView(config) - def validate_normalization(self, actor): # Interior successors are ordinary observations, already checked by the # policy dataset. Only the final successor can contain additional values. diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py index 6a70d5c8..3063d3d0 100644 --- a/src/sequifier/io/output_paths.py +++ b/src/sequifier/io/output_paths.py @@ -40,7 +40,7 @@ def require_directory_destinations(paths): raise ValueError(f"archive requires a directory: {path}") -def validate_training_destinations(config, checkpoint, protected): +def validate_training_destinations(config, checkpoint, protected, *, phase_offset=0): """Reserve generated trees as well as explicit files before any publication.""" checkpoint = Path(checkpoint).resolve() files = unique_paths( @@ -60,7 +60,7 @@ def validate_training_destinations(config, checkpoint, protected): from sequifier.artifacts.phase import PhaseArtifacts generated = [] - for index, phase in enumerate(phases): + for index, phase in enumerate(phases, phase_offset): artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) for path in ( artifacts.model, @@ -97,7 +97,7 @@ def validate_training_destinations(config, checkpoint, protected): ).model.parent / "checkpoints" ).resolve() - for index, phase in enumerate(phases) + for index, phase in enumerate(phases, phase_offset) } resume = getattr(getattr(config, "runtime_settings", None), "resume", None) snapshot_resume = ( diff --git a/src/sequifier/io/policy_dataset.py b/src/sequifier/io/policy_dataset.py index e124ad13..a4f250a0 100644 --- a/src/sequifier/io/policy_dataset.py +++ b/src/sequifier/io/policy_dataset.py @@ -93,10 +93,13 @@ def __len__(self): def sampler(self, generator): return EpisodeWindowSampler(self, generator) - def rebind(self, config): - if self.config != config: - self._labels.clear() - super().rebind(config) + def subset(self, indices): + return type(self)( + self.episodes.subset(indices), + self.config, + episode_cache_size=self.episode_cache_size, + cache_owner=self.cache_owner, + ) def _episode_labels(self, index: int): return self._labels.get(index, self._load_labels) @@ -146,7 +149,7 @@ def __iter__(self): yield offset + step -def split_by_episode( +def episode_partition_indices( dataset, validation_fraction=0.2, seed=1010, *, group_identical=False ): def keys(index): @@ -159,17 +162,22 @@ def keys(index): result.append(("content", dataset.demonstration_keys[index])) return result - partitions = split_components( - range(len(dataset.episodes)), keys, validation_fraction, seed + return tuple( + tuple(indices) + for indices in split_components( + range(len(dataset.episodes)), keys, validation_fraction, seed + ) ) + + +def split_by_episode( + dataset, validation_fraction=0.2, seed=1010, *, group_identical=False +): return tuple( - type(dataset)( - dataset.episodes.subset(indices), - dataset.config, - episode_cache_size=dataset.episode_cache_size, - cache_owner=dataset.cache_owner, + dataset.subset(indices) + for indices in episode_partition_indices( + dataset, validation_fraction, seed, group_identical=group_identical ) - for indices in partitions ) @@ -183,11 +191,13 @@ def __init__( cache_size, *, preference_manifests=None, + episode_records=None, cache_owner=None, ): self.sources, self.config, self.cache_size = sources, config, cache_size self.cache_owner = cache_owner self.corpora = {} + self.episode_records = episode_records if episode_records is not None else {} self.datasets = {} self.preference_manifests = ( preference_manifests if preference_manifests is not None else {} @@ -200,22 +210,20 @@ def preference_pairs(self, path): self.preference_manifests[path] = load_preferences(path).pairs return self.preference_manifests[path] - def rebind(self, config): - self.config = config - for dataset in self.datasets.values(): - dataset.rebind(config) - def dataset(self, names): key = tuple(names) if key not in self.datasets: for name in key: if name not in self.corpora: - self.corpora[name] = EpisodeCorpus( + corpus = EpisodeCorpus( [self.sources[name]], self.config, self.cache_size, cache_owner=self.cache_owner, + records=self.episode_records.get(name), ) + self.corpora[name] = corpus + self.episode_records[name] = corpus.records self.datasets[key] = EpisodeDecisions( EpisodeView.combine([self.corpora[name].view() for name in key]), self.config, diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index da88b945..4b363d90 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -24,15 +24,11 @@ def contract(self): def fitting_dataset(self): return self.training.fitting_dataset() - def rebind(self, config): - for partition in (self.training, self.validation): - partition.rebind(config) - def validate(self, actor): for partition in (self.training, self.validation): - partition.use_config(actor.config) + partition.validate_features() fitting = partition.fitting_dataset() - fitting.use_config(actor.config) + fitting.validate_features() fitting.validate_normalization(actor.interface.selected_columns_statistics) @@ -44,7 +40,7 @@ def __init__(self, pairs, corpora, config): for name, data in corpora.items() } self._fitting = None - self._validated_config = None + self._features_validated = False self.contract = [] for pair in self.pairs: entries = [] @@ -128,18 +124,10 @@ def fitting_dataset(self): ) return self._fitting - def rebind(self, config): - if self.config == config: - return - self.config = config - self._validated_config = None - if self._fitting is not None: - self._fitting.rebind(config) - - def use_config(self, config): - if self._validated_config == config: + def validate_features(self): + if self._features_validated: return - self.rebind(config) + config = self.config view = PolicyWindowView(config) contexts = {} for pair in self.pairs: @@ -193,10 +181,10 @@ def use_config(self, config): f"pair {pair.pair_id}: initial {mapping} differ under frozen codec" ) - self._validated_config = config.model_copy(deep=True) + self._features_validated = True -def prepare_preferences(run, phase, config, catalog, cache): +def prepare_preferences(run, phase, catalog, cache, membership): from sequifier.rl.offline_data import phase_validation_fraction key = ( @@ -206,6 +194,7 @@ def prepare_preferences(run, phase, config, catalog, cache): ) if key in cache: return cache[key] + config = catalog.config pairs = catalog.preference_pairs(run.preferences[phase.preferences.source]) validation = ( [ @@ -226,18 +215,21 @@ def prepare_preferences(run, phase, config, catalog, cache): f"preferences reference unknown sources: {sorted(names - run.sources.keys())}" ) corpora = {name: catalog.dataset([name]) for name in sorted(names)} - data = PreferenceDataset(all_pairs, corpora, config) - if validation is not None: - left = {key for pair in pairs for key in data.components(pair)} - right = {key for pair in validation for key in data.components(pair)} - if left & right: - raise ValueError( - f"preference train/validation components overlap: {next(iter(left & right))}" + if key not in membership: + data = PreferenceDataset(all_pairs, corpora, config) + if validation is not None: + left = {key for pair in pairs for key in data.components(pair)} + right = {key for pair in validation for key in data.components(pair)} + if left & right: + raise ValueError( + f"preference train/validation components overlap: {next(iter(left & right))}" + ) + else: + pairs, validation = split_components( + pairs, data.components, phase_validation_fraction(run, phase), run.seed ) - else: - pairs, validation = split_components( - pairs, data.components, phase_validation_fraction(run, phase), run.seed - ) + membership[key] = tuple(pairs), tuple(validation) + pairs, validation = membership[key] result = PreferenceData( PreferenceDataset(pairs, corpora, config), PreferenceDataset(validation, corpora, config), diff --git a/src/sequifier/io/prepared_policy_data.py b/src/sequifier/io/prepared_policy_data.py index c0703eb4..c7609bce 100644 --- a/src/sequifier/io/prepared_policy_data.py +++ b/src/sequifier/io/prepared_policy_data.py @@ -18,6 +18,4 @@ def contract(self) -> list: ... def fitting_dataset(self) -> EpisodeFeatures: ... - def rebind(self, config) -> None: ... - def validate(self, actor: "Policy") -> None: ... diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 087469df..40591b29 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -34,9 +34,17 @@ def save_policy( def policy_model_artifact( - policy: Policy, *, metadata: dict | None = None + policy: Policy, + *, + metadata: dict | None = None, + weights: dict[str, torch.Tensor] | None = None, ) -> ModelArtifact: validate_policy_weights(policy) + if weights is None: + weights = { + name: value.detach().cpu() + for name, value in policy.network.state_dict().items() + } return build_model_artifact( policy.network, policy.execution_config, @@ -44,10 +52,7 @@ def policy_model_artifact( **(metadata or {}), "transfer": getattr(policy, "transfer_report", None), }, - state_dict={ - name: value.detach().cpu() - for name, value in policy.network.state_dict().items() - }, + state_dict=weights, ) diff --git a/src/sequifier/rl/dpo.py b/src/sequifier/rl/dpo.py index c69da68e..72c7949a 100644 --- a/src/sequifier/rl/dpo.py +++ b/src/sequifier/rl/dpo.py @@ -73,7 +73,7 @@ def run_dpo_phase(context: PhaseContext[DPOPhaseConfig, PreferenceData]): batches = BatchOrder( len(data.training), phase.batch_size * phase.accumulation_batches, - context.run.seed + context.index, + context.identity.sampling_seed, state=state["batch_order"] if state else None, ) if ( diff --git a/src/sequifier/rl/group_collection.py b/src/sequifier/rl/group_collection.py index 0216bbbf..fa6e7ce2 100644 --- a/src/sequifier/rl/group_collection.py +++ b/src/sequifier/rl/group_collection.py @@ -101,9 +101,7 @@ def __init__( > self.phase.max_buffer_bytes ): raise ValueError("complete GRPO groups exceed the minimum buffer estimate") - self.seed = stream_seed( - context.run.seed, f"grpo-phase-{context.index}-restart-{restart_stream}", 0 - ) + self.seed = context.identity.grpo_seed(restart_stream) self.used, self.executed = 0, 0 self.initial = {} self.cursors = set() diff --git a/src/sequifier/rl/iql.py b/src/sequifier/rl/iql.py index cd42f362..0146683d 100644 --- a/src/sequifier/rl/iql.py +++ b/src/sequifier/rl/iql.py @@ -1,12 +1,13 @@ """Fixed-dataset categorical IQL as a phase-local actor/value/twin-Q runtime.""" import math +from typing import cast import torch +from torch import nn from sequifier.config.policy_plan import IQLPhaseConfig from sequifier.io.iql_dataset import IQLTransitionDataset -from sequifier.rl.models import critic_history_options, make_iql_models from sequifier.rl.offline_data import OfflineData from sequifier.rl.run_state import PhaseContext, PhaseSession from sequifier.rl.window_storage import collated_windows @@ -62,23 +63,10 @@ def _evaluate( def run_iql_phase(context: PhaseContext[IQLPhaseConfig, OfflineData]): actor, phase = context.actor.eval(), context.phase data = context.data.prepare_transitions() - value, q1, q2, targets = make_iql_models( - actor.execution_config, - actor.interface_name, - actor.config.actions, - **critic_history_options( - actor, - phase, - restore=context.resume_state is not None, - ), - ) - for module in (value, q1, q2, *targets): - module.eval() - session = PhaseSession( - context, - modules={"value": value, "critic": torch.nn.ModuleList([q1, q2])}, - frozen={"target_q1": targets[0], "target_q2": targets[1]}, - ) + session = PhaseSession(context) + value = session.modules["value"] + q1, q2 = cast(nn.ModuleList, session.modules["critic"]) + targets = session.networks["target_q1"], session.networks["target_q2"] if not 0 <= session.transaction <= phase.iterations: raise ValueError("IQL checkpoint exceeds phase update budget") if any(steps != session.transaction for steps in session.steps.values()): diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 70f2222c..8f624e93 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -18,7 +18,6 @@ build_actor, preflight_environments, preflight_optimization, - prepare_data, resolve_normalization, validate_data, validate_declared_policy, @@ -32,6 +31,7 @@ from sequifier.rl.run_state import ( ActorRepresentation, PhaseContext, + PolicyPhaseIdentity, PolicyRunResult, clone_weights, require_equal, @@ -57,6 +57,8 @@ def _plan_contract(config): from sequifier.artifacts.model_export import model_execution_config values = config.model_dump(mode="json") + if config.phase_offset: + values["phase_offset"] = config.phase_offset values["execution_model"] = to_jsonable_python( model_execution_config(config.execution) ) @@ -149,10 +151,12 @@ def _initialize_actor(config): def run_policy_training( config: ResolvedPolicyRun | ResolvedSequifierConfig | SequifierConfig, + *, + prepared: PreparedTrainingInputs | None = None, ) -> PolicyRunResult | LearningRunResult: """Validate the whole run, execute phase-local lifecycles, and export its final actor.""" if isinstance(config, ResolvedPolicyRun): - return _run_policy(config) + return _run_policy(config, prepared or PreparedTrainingInputs()) from sequifier.config.train_config import ResolvedSequifierConfig, SequifierConfig from sequifier.config.training_resolution import resolve_training_run @@ -167,18 +171,22 @@ def run_policy_training( return run_policy_plan(config) -def run_policy_group(config: ResolvedPolicyRun, identity: RunIdentity, on_checkpoint): +def run_policy_group( + config: ResolvedPolicyRun, + identity: RunIdentity, + on_checkpoint, + *, + prepared: PreparedTrainingInputs, +): """Run a group of policy phases with a shared actor and optimizer lifecycle.""" - _run_policy(config, group=True, identity=identity, on_checkpoint=on_checkpoint) + _run_policy( + config, prepared, group=True, identity=identity, on_checkpoint=on_checkpoint + ) -def _run_policy(config, *, group=False, identity=None, on_checkpoint=None): - prepared = PreparedTrainingInputs.for_run(config) +def _run_policy(config, prepared, *, group=False, identity=None, on_checkpoint=None): with prepared.run_scope(): - try: - return _execute_policy(config, prepared, group, identity, on_checkpoint) - finally: - config._metrics_service = None + return _execute_policy(config, prepared, group, identity, on_checkpoint) def _execute_policy(config, prepared, group, identity, on_checkpoint): @@ -188,6 +196,10 @@ def _execute_policy(config, prepared, group, identity, on_checkpoint): validate_continuation( config.phases[index - 1] if index else None, phase, resolved=True ) + phase_identities = tuple( + PolicyPhaseIdentity.resolve(index, config.seed) + for index, _ in enumerate(config.phases, config.phase_offset) + ) plan = _plan_contract(config) inputs = prepared.resolve(config) inputs.revalidate(inputs.identity, "run.source_identity") @@ -222,11 +234,11 @@ def _execute_policy(config, prepared, group, identity, on_checkpoint): ) representation = recovery.representation validate_declared_policy(config.policy, representation.policy, "resume.policy") - data = prepare_data(config, representation.policy, inputs) + data = prepared.prepare(config, representation.policy, inputs) actor = build_actor(config, representation.policy) recovery.validate_actor(actor, config.units) else: - data = prepare_data(config, config.policy, inputs) + data = prepared.prepare(config, config.policy, inputs) actor = _initialize_actor(config) representation = ActorRepresentation.from_actor(actor, config.units) if config.normalization is not None: @@ -266,10 +278,10 @@ def _execute_policy(config, prepared, group, identity, on_checkpoint): identity = identity or RunIdentity.start(recovery.run_id) recovery.run_id = identity.run_id - config._metrics_service = PolicyMetrics(config, identity) + metrics = PolicyMetrics(config, identity) references = PolicyReferences(config, recovery, resuming=resuming) saved_random = capture_rng_state(actor.device) - step_budgets = preflight_optimization(config, actor, data) + optimizations = preflight_optimization(config, actor, data) restore_rng_state(actor.device, recovery.random if resuming else saved_random) def save_checkpoint(state=None): @@ -288,7 +300,9 @@ def replay(): _publish_phase_report(record, checkpoint) def run_phase(transition): - phase_index, phase = transition.index, transition.phase + position, phase = transition.index, transition.phase + phase_identity = phase_identities[position] + phase_index = phase_identity.index current_state = recovery.phase_state dependencies.revalidate() incoming_weight_hash = weight_fingerprint(actor.network.state_dict()) @@ -311,7 +325,7 @@ def run_phase(transition): if phase_index == 0 and config.actor_initialization == "history_only": initialization["actor_initialization"] = recovery.initialization actor.eval() - reference, reference_identity = references.resolve(actor) + reference, reference_identity = references.resolve(actor, phase_index) def commit(state): representation.validate_actor( @@ -319,33 +333,39 @@ def commit(state): ) save_checkpoint({**state, "telemetry_path": str(artifacts.telemetry)}) - if transition.continues_optimizer: - actor._phase_continuation = recovery.continuation - artifacts = PhaseArtifacts.for_phase( config.phase_directory, phase_index, phase.name ) - position = current_state["telemetry_position"] if current_state else None - with TelemetryJournal.open(artifacts.telemetry, position) as telemetry: + telemetry_position = ( + current_state["telemetry_position"] if current_state else None + ) + with TelemetryJournal.open( + artifacts.telemetry, telemetry_position + ) as telemetry: context = PhaseContext( config, - phase_index, + phase_identity, phase, actor, - data[phase_index] if data else None, + data[position] if data else None, reference, reference_identity, current_state, commit, - step_budgets[phase_index], + optimizations[position], telemetry, + metrics=metrics, + continuation=recovery.continuation + if transition.continues_optimizer + else None, preserve_optimizer=transition.preserves_optimizer, ) - selected, result = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) - position = telemetry.commit() + execution = BUILTIN_ALGORITHMS[phase.algorithm].runtime(context) + telemetry_position = telemetry.commit() representation.validate_actor( - selected, config.units, f"phases.{phase.name}.selected_actor" + execution.actor, config.units, f"phases.{phase.name}.selected_actor" ) + result = execution.result result = replace( result, provenance={ @@ -360,19 +380,21 @@ def commit(state): "source_provenance": data_identity, }, ) - return PhaseOutcome((selected, result, position), result.stop_scope) + return PhaseOutcome( + (replace(execution, result=result), telemetry_position), result.stop_scope + ) def commit_phase(transition, outcome): nonlocal actor - actor, result, position = outcome.result - recovery.continuation = getattr(actor, "_phase_continuation", None) - recovery.finish_phase(actor, result, position) + execution, position = outcome.result + actor = execution.actor + recovery.finish_phase(execution, position) save_checkpoint() return recovery.completed[-1] def publish(record): _publish_phase_report(record, checkpoint) - if not group and record.result.index == last_data_phase: + if not group and record.result.index == config.phase_offset + last_data_phase: release_phase_data() last_data_phase = max( @@ -405,7 +427,7 @@ def release_phase_data(): checkpoint_path=checkpoint, phases=[_phase_report(result, checkpoint) for result in results], run_id=recovery.run_id, - session_id=config._metrics_service.session_id, + session_id=identity.session_id, status="complete", completion_reason=completion_reason, exports=[config.model_path], diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index ddb5e143..add81c1f 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from sequifier.io.iql_dataset import IQLTransitionDataset -from sequifier.io.policy_dataset import EpisodeDecisions, split_by_episode +from sequifier.io.policy_dataset import EpisodeDecisions, episode_partition_indices @dataclass @@ -33,31 +33,19 @@ def contract(self): for part in (self.training, self.validation) ] - def use_config(self, config): - self.rebind(config) - for part in (self.training, self.validation): - part.use_config(config) - - def rebind(self, config): - for part in (self.training, self.validation): - part.rebind(config) - if self.transitions is not None: - for transitions in self.transitions: - transitions.use_config(config) - def fitting_dataset(self): return self.training def validate(self, actor): - self.use_config(actor.config) for part in (self.training, self.validation): + part.validate_features() part.validate_normalization(actor.interface.selected_columns_statistics) if self.transitions is not None: for transitions in self.transitions: transitions.validate_normalization(actor) -def prepare_offline(run, phase, config, catalog, cache): +def prepare_offline(run, phase, catalog, cache, membership): # The cache belongs to one run: budgets and optimizers do not affect partitions. key = ( "offline", @@ -71,12 +59,14 @@ def prepare_offline(run, phase, config, catalog, cache): if phase.evaluation.sources: validation = corpus([ref.source for ref in phase.evaluation.sources]) else: - training, validation = split_by_episode( - training, - phase_validation_fraction(run, phase), - run.seed, - group_identical=True, - ) + if key not in membership: + membership[key] = episode_partition_indices( + training, + phase_validation_fraction(run, phase), + run.seed, + group_identical=True, + ) + training, validation = (training.subset(indices) for indices in membership[key]) if not len(training) or not len(validation): raise ValueError( f"phases.{phase.name}: nonempty train and validation partitions required" @@ -93,8 +83,8 @@ def prepare_offline(run, phase, config, catalog, cache): return data -def prepare_iql(run, phase, config, catalog, cache): - data = prepare_offline(run, phase, config, catalog, cache) +def prepare_iql(run, phase, catalog, cache, membership): + data = prepare_offline(run, phase, catalog, cache, membership) data.prepare_transitions() return data diff --git a/src/sequifier/rl/phase_optimization.py b/src/sequifier/rl/phase_optimization.py new file mode 100644 index 00000000..42a0a508 --- /dev/null +++ b/src/sequifier/rl/phase_optimization.py @@ -0,0 +1,83 @@ +"""Resolve policy optimizer roles and masks before executing a phase.""" + +from dataclasses import dataclass + +from torch import nn + +from sequifier.config.policy_optimization import OptimizationConfig +from sequifier.rl.models import HistoryCritic, critic_history_options, make_iql_models + + +@dataclass(frozen=True) +class PhaseRoles: + trainable: dict[str, nn.Module] + frozen: dict[str, nn.Module] + + @classmethod + def build(cls, actor, phase, *, metadata_only=False, restoring=False): + trainable, frozen = {"actor": actor.network}, {} + if phase.algorithm in {"ppo", "iql"}: + history = ( + {"metadata_only": True} + if metadata_only + else critic_history_options(actor, phase, restore=restoring) + ) + if phase.algorithm == "ppo": + trainable["value"] = HistoryCritic.ppo_value(actor, **history).eval() + else: + value, q1, q2, targets = make_iql_models( + actor.execution_config, + actor.interface_name, + actor.config.actions, + **history, + ) + trainable.update( + value=value.eval(), critic=nn.ModuleList([q1, q2]).eval() + ) + frozen = { + f"target_q{i}": target.eval() for i, target in enumerate(targets, 1) + } + return cls(trainable, frozen) + + +@dataclass(frozen=True) +class PhaseOptimization: + step_budget: int | None + configs: dict[str, OptimizationConfig] + masks: dict[str, list[str]] + + @classmethod + def resolve(cls, phase, roles, training_size): + budget = phase.optimizer_step_budget(training_size) + masks = {} + for role, module in roles.trainable.items(): + frozen = phase.trainability.policy(role).resolve(module) + masks[role] = sorted( + name + for name, parameter in module.named_parameters() + if parameter.requires_grad and id(parameter) not in frozen + ) + if not masks[role]: + raise ValueError(f"{phase.name}: {role} leaves no trainable parameters") + configs = { + role: config.resolve_budget(budget) + for role, config in phase.optimization_configs().items() + } + return cls(budget, configs, masks) + + def build(self, roles): + if roles.trainable.keys() != self.configs.keys(): + raise ValueError("phase optimization roles disagree with trainable modules") + for role, module in roles.trainable.items(): + parameters = dict(module.named_parameters()) + mask = set(self.masks[role]) + if mask - parameters.keys(): + raise ValueError( + f"{role}: trainable parameter names disagree with preflight" + ) + for name, parameter in parameters.items(): + parameter.requires_grad_(name in mask) + return { + role: self.configs[role].build(module, step_budget=self.step_budget) + for role, module in roles.trainable.items() + } diff --git a/src/sequifier/rl/ppo.py b/src/sequifier/rl/ppo.py index f9c423c5..5e1489b0 100644 --- a/src/sequifier/rl/ppo.py +++ b/src/sequifier/rl/ppo.py @@ -9,7 +9,6 @@ load_online_adapter, require_exact_snapshot, ) -from sequifier.rl.models import HistoryCritic, critic_history_options from sequifier.rl.ppo_collector import PPOCollector from sequifier.rl.ppo_update import update_ppo from sequifier.rl.run_state import PhaseContext, PhaseSession, require_equal @@ -19,15 +18,8 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): actor, phase = context.actor.eval(), context.phase - value = HistoryCritic.ppo_value( - actor, - **critic_history_options( - actor, - phase, - restore=context.resume_state is not None, - ), - ).eval() - session = PhaseSession(context, modules={"value": value}) + session = PhaseSession(context) + value = session.modules["value"] state = session.state env_steps = state.get("env_steps", 0) if ( @@ -44,7 +36,7 @@ def run_ppo_phase(context: PhaseContext[PPOPhaseConfig, None]): require_exact_snapshot(adapter) identity = adapter_identity(adapter) collector = PPOCollector( - actor, value, adapter, phase, context.run.seed + context.index + actor, value, adapter, phase, context.identity.sampling_seed ) if state: require_equal(state["adapter_identity"], identity, "ppo.adapter_identity") diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 102652bc..94fda96a 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -11,7 +11,6 @@ unique_paths, validate_path_conflicts, ) -from sequifier.io.prepared_policy_data import PreparedPolicyData from sequifier.model.factory import build_transformer_network, validate_hash_contracts from sequifier.rl.checkpoint import phase_actor_path, validate_policy_weights from sequifier.rl.environment import ( @@ -35,7 +34,7 @@ def validate_output_paths(config, dependencies): Path(config.report_path), Path(config.checkpoint_path), ] - for i, phase in enumerate(config.phases): + for i, phase in enumerate(config.phases, config.phase_offset): actor = phase_actor_path(config.phase_directory, i, phase.name) outputs += [actor, actor.with_name("report.json")] message = "policy outputs and phase archives must have distinct paths" @@ -52,7 +51,9 @@ def validate_output_paths(config, dependencies): validate_path_conflicts(outputs, protected, label="policy output") from sequifier.io.output_paths import validate_training_destinations - validate_training_destinations(config, config.checkpoint_path, protected) + validate_training_destinations( + config, config.checkpoint_path, protected, phase_offset=config.phase_offset + ) require_directory_destinations(archives) validate_path_conflicts(archives, protected | set(outputs), label="archive") @@ -119,10 +120,6 @@ def build_actor(config, binding=None, *, artifact=None): return actor -def prepare_data(config, policy, inputs) -> list[PreparedPolicyData | None]: - return PreparedTrainingInputs.for_run(config).prepare(config, policy, inputs) - - def validate_data(data, actor): validated = set() for item in data: @@ -138,7 +135,7 @@ def preflight_environments(config): saved_random = capture_rng_state(device) try: identities = {} - for index, phase in enumerate(config.phases): + for index, phase in enumerate(config.phases, config.phase_offset): if not hasattr(phase, "environment"): continue require_evaluation_timer() @@ -171,55 +168,31 @@ def preflight_environments(config): def preflight_optimization(config, actor, data): """Resolve constructors and authored scheduler budgets before creating outputs.""" - from sequifier.rl.models import HistoryCritic, make_iql_critics + from sequifier.rl.phase_optimization import PhaseOptimization, PhaseRoles baseline = { name: parameter.requires_grad for name, parameter in actor.network.named_parameters() } - budgets, previous_masks = [], None + + def restore_trainability(): + for name, parameter in actor.network.named_parameters(): + parameter.requires_grad_(baseline[name]) + + plans = [] try: for phase, part in zip(config.phases, data): - for name, parameter in actor.network.named_parameters(): - parameter.requires_grad_(baseline[name]) - modules = {"actor": actor.network} - if phase.algorithm == "iql": - value, q1, q2 = make_iql_critics( - actor.execution_config, - actor.interface_name, - actor.config.actions, - metadata_only=True, - ) - modules.update(value=value, critic=torch.nn.ModuleList([q1, q2])) - elif phase.algorithm == "ppo": - modules["value"] = HistoryCritic.ppo_value(actor, metadata_only=True) - masks = {} - for role, module in modules.items(): - frozen = phase.trainability.policy(role).resolve(module) - for parameter in module.parameters(): - if id(parameter) in frozen: - parameter.requires_grad_(False) - masks[role] = sorted( - name - for name, parameter in module.named_parameters() - if parameter.requires_grad - ) - if not masks[role]: - raise ValueError( - f"{phase.name}: {role} leaves no trainable parameters" - ) - if phase.optimizer_state == "continue" and masks != previous_masks: + restore_trainability() + roles = PhaseRoles.build(actor, phase, metadata_only=True) + plan = PhaseOptimization.resolve( + phase, roles, len(part.training) if part is not None else None + ) + if phase.optimizer_state == "continue" and plan.masks != plans[-1].masks: raise ValueError( "optimizer continuation requires identical resolved parameter masks" ) - previous_masks = masks - size = len(part.training) if part is not None else None - budget = phase.optimizer_step_budget(size) - for role, optimization in phase.optimization_configs().items(): - optimization.validate_budget(budget) - optimization.build(modules[role], step_budget=budget) - budgets.append(budget) + plan.build(roles) + plans.append(plan) finally: - for name, parameter in actor.network.named_parameters(): - parameter.requires_grad_(baseline[name]) - return budgets + restore_trainability() + return plans diff --git a/src/sequifier/rl/prepared_inputs.py b/src/sequifier/rl/prepared_inputs.py index 55cfca35..e18eedc2 100644 --- a/src/sequifier/rl/prepared_inputs.py +++ b/src/sequifier/rl/prepared_inputs.py @@ -1,9 +1,9 @@ """Run-owned source identities, episode catalogs and policy partitions.""" from contextlib import contextmanager +from copy import deepcopy from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING from sequifier.config.policy_config import PolicyBinding from sequifier.config.policy_inputs import PolicyFittingSpec @@ -17,10 +17,6 @@ resolve_source, ) -if TYPE_CHECKING: - from sequifier.io.policy_dataset import PolicyDataCatalog - from sequifier.io.prepared_policy_data import PreparedPolicyData - @dataclass(frozen=True) class RunInputs: @@ -37,7 +33,8 @@ def __post_init__(self): @classmethod def resolve(cls, config): - return PreparedTrainingInputs.for_run(config).resolve(config) + prepared = getattr(config, "_prepared_inputs", None) or PreparedTrainingInputs() + return prepared.resolve(config) @property def identity(self): @@ -69,30 +66,20 @@ def _data_policy_contract(policy): } -@dataclass -class _PolicyPartitions: - catalog: "PolicyDataCatalog" - prepared: dict[tuple, "PreparedPolicyData"] = field(default_factory=dict) - - def rebind(self, policy): - self.catalog.rebind(policy) - for data in self.prepared.values(): - data.rebind(policy) - - class PreparedTrainingInputs: """Private, in-memory preparation shared from metadata fitting through execution. - Source interpretation and split settings key reusable partitions. A changed - policy interpretation builds new views; changed codecs refresh existing views - without selecting the partitions again. Callers still revalidate dependencies - before executing a run and at every committed phase boundary. + Source interpretation keys episode records and split membership. Each + execution codec owns its views and caches for the run. Fitting views are + released as soon as metadata has been fitted. """ def __init__(self): self._sources = {} self._checksums = {} self._inputs = {} + self._episode_records = {} + self._memberships = {} self._partitions = {} self._catalogs = {} self._preference_manifests = {} @@ -111,6 +98,8 @@ def measure(self): self._inputs, self._sources, self._checksums, + self._episode_records, + self._memberships, self._partitions, self._catalogs, self._preference_manifests, @@ -122,6 +111,8 @@ def _clear_data(self): for cache in self._episode_caches: cache.clear() self._episode_caches.clear() + self._episode_records.clear() + self._memberships.clear() self._partitions.clear() self._catalogs.clear() self._preference_manifests.clear() @@ -227,78 +218,88 @@ def resolve(self, config): self._inputs[key] = RunInputs(sources, checksums) return self._inputs[key] - def prepare_phase(self, config, phase, policy, inputs, *, fitting=False): + @contextmanager + def fitting_phase(self, config, phase, policy, inputs): + start = len(self._episode_caches) + try: + yield self._prepare_phase(config, phase, policy, inputs, fitting=True) + finally: + for cache in self._episode_caches[start:]: + cache.clear() + del self._episode_caches[start:] + self.measure() + + def _prepare_phase(self, config, phase, policy, inputs, *, fitting=False): from sequifier.io.policy_dataset import PolicyDataCatalog from sequifier.rl.offline_data import prepare_offline from sequifier.rl.registry import BUILTIN_ALGORITHMS - entry = BUILTIN_ALGORITHMS[phase.algorithm] - prepare = ( - prepare_offline - if fitting and phase.algorithm == "iql" - else entry.prepare_data - ) + prepare = BUILTIN_ALGORITHMS[phase.algorithm].prepare_data if prepare is None: return None + if fitting and phase.algorithm == "iql": + prepare = prepare_offline sources, preferences, checksums = self._phase_dependencies( config, phase, inputs ) - key = fingerprint( + catalog_key = fingerprint( { "sources": { name: { "source": source.source.model_dump(mode="json"), "paths": [str(path) for path in source.paths], "kind": source.kind, + "checksums": { + str(path): value for path, value in source.checksums.items() + }, } for name, source in sources.items() }, - "identity": checksums, - "preferences": preferences, "policy": _data_policy_contract(policy), + "episode_cache_size": config.episode_cache_size, + } + ) + partition_key = fingerprint( + { + "catalog": catalog_key, + "preferences": preferences, + "identity": checksums, "validation_fraction": config.validation_fraction, "source_validation_fractions": getattr( config, "source_validation_fractions", {} ), "seed": config.seed, - "episode_cache_size": config.episode_cache_size, } ) - if key not in self._partitions: - catalog_key = fingerprint( - { - "sources": { - name: { - "source": source.source.model_dump(mode="json"), - "paths": [str(path) for path in source.paths], - "kind": source.kind, - "checksums": { - str(path): value - for path, value in source.checksums.items() - }, - } - for name, source in sources.items() - }, - "policy": _data_policy_contract(policy), - "episode_cache_size": config.episode_cache_size, - } + if fitting: + catalog_views, partitions = {}, {} + view_key = catalog_key + else: + codec_key = fingerprint(policy.model_dump(mode="json")) + catalog_views = self._catalogs + view_key = (catalog_key, codec_key) + partitions = self._partitions.setdefault((partition_key, codec_key), {}) + if view_key not in catalog_views: + catalog_views[view_key] = PolicyDataCatalog( + sources, + deepcopy(policy), + config.episode_cache_size, + preference_manifests=self._preference_manifests, + episode_records=self._episode_records.setdefault(catalog_key, {}), + cache_owner=self, ) - if catalog_key not in self._catalogs: - self._catalogs[catalog_key] = PolicyDataCatalog( - sources, - policy, - config.episode_cache_size, - preference_manifests=self._preference_manifests, - cache_owner=self, - ) - self._partitions[key] = _PolicyPartitions(self._catalogs[catalog_key]) - partitions = self._partitions[key] - partitions.rebind(policy) - return prepare(config, phase, policy, partitions.catalog, partitions.prepared) + return prepare( + config, + phase, + catalog_views[view_key], + partitions, + self._memberships.setdefault(partition_key, {}), + ) def prepare(self, config, policy, inputs): data = [ - self.prepare_phase(config, phase, policy, inputs) for phase in config.phases + self._prepare_phase(config, phase, policy, inputs) + for phase in config.phases ] self.measure() return data diff --git a/src/sequifier/rl/references.py b/src/sequifier/rl/references.py index e9c9da74..92b24ede 100644 --- a/src/sequifier/rl/references.py +++ b/src/sequifier/rl/references.py @@ -36,7 +36,9 @@ def __init__(self, config, recovery, *, resuming): continue if Path(reference.path).is_file(): _, identity = self._artifact(reference.path) - expected = recovery.reference_contract.setdefault(str(index), identity) + expected = recovery.reference_contract.setdefault( + str(config.phase_offset + index), identity + ) require_equal( expected, identity, f"phases.{phase.name}.reference.weights" ) @@ -75,7 +77,7 @@ def _artifact(self, path): self.checksums[path] = checksum return self.artifacts[path] - def resolve(self, actor): + def resolve(self, actor, index): recovery = self.recovery declaration = getattr( self.config.phases[recovery.phase_index], "reference", None @@ -90,7 +92,7 @@ def resolve(self, actor): elif declaration.source == "artifact": weights, identity = self._artifact(declaration.path) require_equal( - recovery.reference_contract[str(recovery.phase_index)], + recovery.reference_contract[str(index)], identity, "reference.artifact.weights", ) @@ -102,7 +104,7 @@ def resolve(self, actor): return reference, identity -def validate_reference_state(phase, recovery): +def validate_reference_state(phase, recovery, index): state = recovery.phase_state if state is not None: reference = getattr(phase, "reference", None) @@ -116,6 +118,6 @@ def validate_reference_state(phase, recovery): expected = ( weight_fingerprint(recovery.incoming_actor) if reference.source == "phase_start" - else recovery.reference_contract[str(recovery.phase_index)] + else recovery.reference_contract[str(index)] ) require_equal(expected, identity, "resume.reference.weights") diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index ead55d17..3d6fb50a 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -24,8 +24,9 @@ from sequifier.rl.references import validate_reference_state from sequifier.rl.run_state import ( ActorRepresentation, + OptimizerContinuation, + PhaseExecution, PolicyPhaseResult, - clone_weights, require_equal, weight_fingerprint, ) @@ -67,7 +68,7 @@ class RunCheckpoint: reference_contract: dict = field(default_factory=dict) completed: list[CompletedPhase] = field(default_factory=list) phase_state: dict | None = None - continuation: dict | None = None + continuation: OptimizerContinuation | None = None stopped: bool = False run_id: str = field(default_factory=new_run_id) @@ -75,6 +76,10 @@ class RunCheckpoint: def phase_index(self): return len(self.completed) + @property + def next_phase_index(self): + return self.plan.get("phase_offset", 0) + self.phase_index + @classmethod def load(cls, path, config, plan, data_identity=None, *, run_id=None): path = Path(path) @@ -141,9 +146,10 @@ def validate(self, config): validate_finite_state(self.incoming_actor, "incoming_actor") # Parsing the owned representation also validates the persisted policy schema. self.representation.policy - for index, record in enumerate(self.completed): + for position, record in enumerate(self.completed): result = record.result - phase = config.phases[index] + phase = config.phases[position] + index = config.phase_offset + position require_equal( (index, phase.name, phase.algorithm, phase.handoff), (result.index, result.name, result.algorithm, result.handoff), @@ -185,7 +191,7 @@ def validate(self, config): ) artifacts = PhaseArtifacts.for_phase( config.phase_directory, - self.phase_index, + self.next_phase_index, config.phases[self.phase_index].name, ) require_equal( @@ -215,7 +221,11 @@ def validate(self, config): raise ValueError( "checkpoint best actor, metric and transaction must be recorded together" ) - validate_reference_state(config.phases[self.phase_index], self) + validate_reference_state( + config.phases[self.phase_index], + self, + self.next_phase_index, + ) validate_finite_state(state, "phase") def validate_actor(self, actor, units): @@ -246,7 +256,7 @@ def phase_boundary(self): active = None if state is not None: active = { - "index": self.phase_index, + "index": self.next_phase_index, "transaction": state["transaction"], "sidecars": { "telemetry": { @@ -294,11 +304,12 @@ def save(self, path, device, state=None): ), ) - def finish_phase(self, actor, result, telemetry_position): - require_equal(self.phase_index, result.index, "handoff.phase_index") + def finish_phase(self, execution: PhaseExecution, telemetry_position): + actor, result = execution.actor, execution.result + require_equal(self.next_phase_index, result.index, "handoff.phase_index") require_equal( result.selected_weight_hash, - weight_fingerprint(actor.network.state_dict()), + weight_fingerprint(execution.weights), "handoff.selected_actor", ) artifacts = PhaseArtifacts(Path(result.provenance["actor_path"])) @@ -306,15 +317,17 @@ def finish_phase(self, actor, result, telemetry_position): commit = PhaseCommit.stage( artifacts, result_data, - model=policy_model_artifact(actor, metadata=result_data), + model=policy_model_artifact( + actor, metadata=result_data, weights=execution.weights + ), telemetry={"path": str(artifacts.telemetry), "position": telemetry_position} if telemetry_position is not None else None, ) - weights = clone_weights(actor.network) self.completed.append(CompletedPhase(commit)) self.stopped = result.stop_scope == "end_plan" - self.incoming_actor = weights + self.incoming_actor = execution.weights + self.continuation = execution.continuation self.phase_state = None diff --git a/src/sequifier/rl/run_config.py b/src/sequifier/rl/run_config.py index 807e6518..9fba87dd 100644 --- a/src/sequifier/rl/run_config.py +++ b/src/sequifier/rl/run_config.py @@ -1,8 +1,8 @@ """Resolved inputs for the native policy runner; never an authored config.""" -from typing import Any, Literal +from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from pydantic import BaseModel, ConfigDict, Field from sequifier.config.components import ResumeConfig from sequifier.config.normalization_config import FrozenNormalizationConfig @@ -13,14 +13,12 @@ class ResolvedPolicyRun(BaseModel): - _prepared_inputs: Any = PrivateAttr(default=None) - _metrics_service: Any = PrivateAttr(default=None) - model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) project_root: str model_name: str device: str seed: int + phase_offset: int = Field(default=0, ge=0, exclude=True) enforce_determinism: bool = False policy: ResolvedPolicyBinding execution: ModelExecution = Field(exclude=True) diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 6969b6f8..8548e692 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -4,14 +4,15 @@ import json import math import time -from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Callable, Generic, TypeVar +from dataclasses import asdict, dataclass, replace +from typing import TYPE_CHECKING, Callable, Generic, TypedDict, TypeVar import torch from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.policy_plan import PhaseConfig from sequifier.contract_versions import ContractVersions +from sequifier.rl.phase_optimization import PhaseOptimization, PhaseRoles from sequifier.rl.policy import Policy from sequifier.rl.telemetry import TelemetryJournal from sequifier.runtime.plan import metric_improved, select_handoff, stop_scope @@ -19,6 +20,7 @@ from sequifier.runtime.stopping import StoppingState if TYPE_CHECKING: + from sequifier.rl.metrics import PolicyMetrics from sequifier.rl.run_config import ResolvedPolicyRun PhaseType = TypeVar("PhaseType", bound=PhaseConfig) @@ -135,10 +137,43 @@ class PolicyRunResult: checkpoint_path: str +class OptimizerContinuation(TypedDict): + algorithm: str + networks: dict[str, dict[str, torch.Tensor]] + optimization: dict[str, dict] + trainable_parameters: dict[str, list[str]] + + +@dataclass(frozen=True) +class PhaseExecution: + actor: Policy + weights: dict[str, torch.Tensor] + result: PolicyPhaseResult + continuation: OptimizerContinuation | None + + +@dataclass(frozen=True) +class PolicyPhaseIdentity: + index: int + run_seed: int + sampling_seed: int + + @classmethod + def resolve(cls, index: int, run_seed: int): + return cls(index, run_seed, run_seed + index) + + def grpo_seed(self, restart): + from sequifier.rl.collection import stream_seed + + return stream_seed( + self.run_seed, f"grpo-phase-{self.index}-restart-{restart}", 0 + ) + + @dataclass(frozen=True) class PhaseContext(Generic[PhaseType, DataType]): run: "ResolvedPolicyRun" - index: int + identity: PolicyPhaseIdentity phase: PhaseType actor: Policy data: DataType @@ -146,10 +181,16 @@ class PhaseContext(Generic[PhaseType, DataType]): reference_identity: str | None resume_state: dict | None commit: Callable[[dict], None] - step_budget: int | None + optimization: PhaseOptimization telemetry: TelemetryJournal + metrics: "PolicyMetrics | None" = None + continuation: OptimizerContinuation | None = None preserve_optimizer: bool = False + @property + def index(self): + return self.identity.index + class Selection: """Strict comparisons retain the earliest finite best on ties.""" @@ -188,10 +229,6 @@ def state_dict(self): def finish(self, context, transaction: int, clocks, provenance): actor = context.actor.eval() - if self.phase.handoff == "best" and self.best_weights is None: - raise ValueError( - f"phases.{self.phase.name}: handoff best requires a finite {self.monitor}" - ) selected, metric = select_handoff( self.phase.handoff, (transaction, self.last_metric), @@ -200,7 +237,14 @@ def finish(self, context, transaction: int, clocks, provenance): if selected is None: raise ValueError(f"{self.phase.name}: no selected transaction") if self.phase.handoff == "best": - actor.network.load_state_dict(self.best_weights, strict=True) + if self.best_weights is None: + raise ValueError( + f"phases.{self.phase.name}: handoff best requires a finite {self.monitor}" + ) + weights = self.best_weights + actor.network.load_state_dict(weights, strict=True) + else: + weights = clone_weights(actor.network) result = PolicyPhaseResult( context.index, self.phase.name, @@ -210,64 +254,33 @@ def finish(self, context, transaction: int, clocks, provenance): self.phase.handoff, metric, selected, - weight_fingerprint(actor.network.state_dict()), + weight_fingerprint(weights), clocks, (), provenance, ) - return actor, result + return actor, weights, result class PhaseSession: """Own common progress and optimization; algorithms own their update loops.""" - def __init__(self, context, *, modules=None, frozen=None): + def __init__(self, context): self.context = context - actor = context.actor - if not hasattr(actor, "_phase_trainability"): - actor._phase_trainability = { - name: p.requires_grad for name, p in actor.network.named_parameters() - } - for name, parameter in actor.network.named_parameters(): - parameter.requires_grad_(actor._phase_trainability[name]) - self.modules = {"actor": context.actor.network, **(modules or {})} - self.mask_ids = {} - for role, module in self.modules.items(): - eligible = {id(p) for p in module.parameters() if p.requires_grad} - mask = context.phase.trainability.policy(role).resolve(module) - if not eligible - mask: + roles = PhaseRoles.build( + context.actor, context.phase, restoring=context.resume_state is not None + ) + self.modules = roles.trainable + self.networks = {**self.modules, **roles.frozen} + self.mask_ids = context.optimization.masks + self.configs = context.optimization.configs + self.optimization = context.optimization.build(roles) + if context.phase.optimizer_state == "continue" and context.resume_state is None: + continuation = context.continuation + if continuation is None: raise ValueError( - f"{context.phase.name}: {role} leaves no trainable parameters" + "continued phase requires the preceding optimizer state" ) - for parameter in module.parameters(): - if id(parameter) in mask: - parameter.requires_grad_(False) - self.mask_ids[role] = sorted( - name for name, p in module.named_parameters() if p.requires_grad - ) - self.networks = {**self.modules, **(frozen or {})} - self.configs = { - role: config.resolve_budget(context.step_budget) - for role, config in context.phase.optimization_configs().items() - } - if self.modules.keys() != self.configs.keys(): - raise ValueError("phase optimization roles disagree with trainable modules") - self.optimization = { - role: self.configs[role].build(module, step_budget=context.step_budget) - for role, module in self.modules.items() - } - continuation = getattr(actor, "_phase_continuation", None) - if ( - context.phase.optimizer_state == "continue" - and not context.resume_state - and continuation is None - ): - raise ValueError("continued phase requires the preceding optimizer state") - if ( - context.phase.optimizer_state == "continue" - and not context.resume_state - and continuation - ): require_equal( context.phase.algorithm, continuation["algorithm"], @@ -275,7 +288,7 @@ def __init__(self, context, *, modules=None, frozen=None): ) self._restore_networks_and_optimizers(continuation, "optimization") self.steps = dict.fromkeys(self.modules, 0) - self.sampler = torch.Generator().manual_seed(context.run.seed + context.index) + self.sampler = torch.Generator().manual_seed(context.identity.sampling_seed) self.state = context.resume_state or {} self.transaction = self.state.get("transaction", 0) phase = context.phase @@ -288,7 +301,7 @@ def __init__(self, context, *, modules=None, frozen=None): else (phase.total_groups + phase.groups_per_iteration - 1) // phase.groups_per_iteration if hasattr(phase, "total_groups") - else context.step_budget or 1 + else context.optimization.step_budget or 1 ) self.stopping = StoppingState(**self.state.get("stopping", {})) self.initial_evaluated = self.state.get("initial_evaluated", False) @@ -305,7 +318,7 @@ def __init__(self, context, *, modules=None, frozen=None): self._restore_networks_and_optimizers(self.state, "phase") for role, saved in self.state["optimization"].items(): steps = saved["steps"] - budget = context.step_budget + budget = context.optimization.step_budget if ( (not isinstance(steps, int)) or steps < self.transaction @@ -403,9 +416,9 @@ def should_evaluate(self, final): ) def observe(self, metric, **record): - if self.context.run._metrics_service is not None: + if self.context.metrics is not None: values = {**record, self.selection.monitor: metric} - self.context.run._metrics_service.record( + self.context.metrics.record( self, values, validation=True, @@ -427,8 +440,8 @@ def record_evaluation(self, metric, **record): def record_training(self, *, final=False, **record): self.telemetry.append("metrics", {"transaction": self.transaction, **record}) - if self.context.run._metrics_service is not None: - self.context.run._metrics_service.record(self, record, final=final) + if self.context.metrics is not None: + self.context.metrics.record(self, record, final=final) def evaluate_validation(self, evaluator, **record): values = evaluator() @@ -570,21 +583,9 @@ def checkpoint(self, final, **extra): self.last_snapshot_time = self.last_checkpoint_time def finish(self, clocks, provenance): - if self.context.run._metrics_service is not None: - self.context.run._metrics_service.flush(self) - if self.context.preserve_optimizer: - self.context.actor._phase_continuation = { - "algorithm": self.context.phase.algorithm, - "networks": { - name: clone_weights(module) - for name, module in self.networks.items() - }, - "optimization": self.optimization_state(), - "trainable_parameters": self.mask_ids, - } - elif hasattr(self.context.actor, "_phase_continuation"): - del self.context.actor._phase_continuation - actor, result = self.selection.finish( + if self.context.metrics is not None: + self.context.metrics.flush(self) + actor, weights, result = self.selection.finish( self.context, self.transaction, clocks, @@ -592,15 +593,26 @@ def finish(self, clocks, provenance): **provenance, "trainable_parameters": self.mask_ids, "resolved_seeds": { - "run": self.context.run.seed, - "phase_sampling": self.context.run.seed + self.context.index, + "run": self.context.identity.run_seed, + "phase_sampling": self.context.identity.sampling_seed, }, }, ) - - from dataclasses import replace - - return actor, replace( - result, - stop_scope=stop_scope(self.context.phase.stopping, self.stopping.triggered), + scope = stop_scope(self.context.phase.stopping, self.stopping.triggered) + continuation: OptimizerContinuation | None = None + if self.context.preserve_optimizer and scope != "end_plan": + continuation = { + "algorithm": self.context.phase.algorithm, + "networks": { + name: weights if name == "actor" else clone_weights(module) + for name, module in self.networks.items() + }, + "optimization": self.optimization_state(), + "trainable_parameters": self.mask_ids, + } + return PhaseExecution( + actor, + weights, + replace(result, stop_scope=scope), + continuation, ) diff --git a/src/sequifier/rl/training.py b/src/sequifier/rl/training.py index bed22d19..7e89b3ac 100644 --- a/src/sequifier/rl/training.py +++ b/src/sequifier/rl/training.py @@ -42,7 +42,7 @@ def run_imitation_phase(context: PhaseContext[PolicyImitationPhaseConfig, Offlin batches = BatchOrder( len(training), phase.batch_size * phase.accumulation_batches, - context.run.seed + context.index, + context.identity.sampling_seed, sample=training.sampler, state=state["batch_order"] if state else None, ) diff --git a/src/sequifier/runtime/builder.py b/src/sequifier/runtime/builder.py index 954c149c..ee94a1cc 100644 --- a/src/sequifier/runtime/builder.py +++ b/src/sequifier/runtime/builder.py @@ -179,6 +179,7 @@ def build( integrations: IntegrationManager, *, initial_artifact=None, + phase_offset: int = 0, identity: RunIdentity | None = None, on_checkpoint=None, backbone_revision=None, @@ -226,7 +227,10 @@ def build( ) add_backbone_revision(dependencies, config, revision_id) validate_training_destinations( - config, output_checkpoint_path(config), dependencies.protected_paths + config, + output_checkpoint_path(config), + dependencies.protected_paths, + phase_offset=phase_offset, ) if loaded is not None: restorer.validate_for_restore(loaded, config, dependencies) @@ -238,7 +242,9 @@ def build( config = config.model_copy(update={"interfaces": interfaces}) state = RunState.from_state_dict(loaded.checkpoint.run_state) else: - state = RunState() + state = RunState(phase_offset=phase_offset) + if state.phase_offset != phase_offset: + raise ValueError("Supervised recovery has a different phase offset") if identity is None: identity = strategy.gather_objects( RunIdentity.start(state.run_id) if strategy.rank == 0 else None diff --git a/src/sequifier/training/checkpoint_service.py b/src/sequifier/training/checkpoint_service.py index cdf22d28..67705f34 100644 --- a/src/sequifier/training/checkpoint_service.py +++ b/src/sequifier/training/checkpoint_service.py @@ -198,7 +198,9 @@ def __init__( self.compatibility = compatibility or CheckpointCompatibility() def save(self, request: CheckpointRequest, run: Any) -> Path | None: - path = self.store.path_for(request.suffix, run.state.phase_index) + path = self.store.path_for( + request.suffix, run.state.phase_index, phase_offset=run.state.phase_offset + ) access = run.optimization.access(run.network) run.integrations.emit( CheckpointSaving(access=access, identity=request.identity, path=path) diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 3bddf941..4cb5bf43 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -89,9 +89,13 @@ def run_policy_plan(config): """Use native policy recovery when the plan shares an actor representation.""" from sequifier.config.training_resolution import policy_run_config from sequifier.rl.learner import run_policy_training + from sequifier.rl.prepared_inputs import PreparedTrainingInputs if config.publication.mode == "policy": - return run_policy_training(policy_run_config(config, config.training_plan)) + return run_policy_training( + policy_run_config(config, config.training_plan), + prepared=PreparedTrainingInputs.for_run(config), + ) return run_learning_plan(config) @@ -106,6 +110,8 @@ def _snapshot_recovery(source, destination): def _portable_commit(config, index, phase, checkpoint, values): commit = PhaseCommit(**values) + if commit.result["index"] != index: + raise ValueError("native phase commit disagrees with the original phase index") artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) details = dict(commit.result) for key in ( @@ -120,7 +126,6 @@ def _portable_commit(config, index, phase, checkpoint, values): details.pop(key, None) details["provenance"] = {**details["provenance"], **artifacts.describe()} if phase.algorithm == "supervised": - details["resolved_seed"] = config.seed + index details["optimizer_steps"] = details["clocks"]["optimizer_steps"] return commit.with_result( phase_report( @@ -143,11 +148,7 @@ def run_learning_plan(config) -> LearningRunResult: def _prepare_plan(config, prepared, groups, revision, state): - from sequifier.rl.preparation import ( - preflight_environments, - prepare_data, - validate_output_paths, - ) + from sequifier.rl.preparation import preflight_environments, validate_output_paths manifest = DependencyManifest() add_initialization(manifest, config) @@ -163,7 +164,7 @@ def _prepare_plan(config, prepared, groups, revision, state): if phase.algorithm == "supervised": native = supervised_group_config(config, phases, index, False) manifest.merge(supervised_dependencies(native)) - supervised_runs.append(native) + supervised_runs.append((index, native)) continue artifacts = PhaseArtifacts.for_phase(config.phase_directory, index, phase.name) native = policy_group_config( @@ -190,12 +191,15 @@ def _prepare_plan(config, prepared, groups, revision, state): validate_output_paths(native, dependencies) manifest.merge(dependencies) preflight_environments(native) - for offset, data in enumerate(prepare_data(native, native.policy, inputs)): + for offset, data in enumerate(prepared.prepare(native, native.policy, inputs)): if data is not None: last_data_phase = index + offset - for native in supervised_runs: + for index, native in supervised_runs: validate_training_destinations( - native, checkpoint_path(native), manifest.protected_paths + native, + checkpoint_path(native), + manifest.protected_paths, + phase_offset=index, ) return manifest, last_data_phase @@ -398,7 +402,7 @@ def save_progress(path, records): if first.algorithm == "supervised": run_supervised_group(config, request, revision if revision else {}) else: - run_portable_policy_group(config, request) + run_portable_policy_group(config, request, prepared) stop = completed[-1].result["stop_scope"] if stop != "end_plan" and len(completed) != index + len(group): raise ValueError("native phase group returned before committing its phases") diff --git a/src/sequifier/training/engine.py b/src/sequifier/training/engine.py index 269a5f00..a6fef712 100644 --- a/src/sequifier/training/engine.py +++ b/src/sequifier/training/engine.py @@ -128,7 +128,7 @@ def _evaluation_context( return EvaluationContext( run_id=run.state.run_id, session_id=run.state.session_id, - phase_index=phase_index, + phase_index=run.state.phase_offset + phase_index, phase_epoch=phase_epoch, epoch=epoch, training_batch=batch, @@ -262,10 +262,11 @@ def _manifest( run.config.report_path, ) - def _stage_phase(self, run, index, phase, outcome): + def _stage_phase(self, run, index, view, outcome): """Stage the selected model for the next checkpoint publication.""" from sequifier.rl.run_state import weight_fingerprint + phase = view.spec() last = run.distributed.capture_model_state(run.network) if run.distributed.rank != 0: return @@ -278,6 +279,7 @@ def _stage_phase(self, run, index, phase, outcome): index=index, name=phase.name, algorithm="supervised", + resolved_seed=view.seed, model_path=artifacts.model, checkpoint_path=run.checkpoints.store.latest_path, handoff=phase.handoff, @@ -845,7 +847,10 @@ def commit_phase(transition, outcome): nonlocal committing committing = True commit = self._stage_phase( - run, transition.index, transition.phase.spec(), outcome + run, + state.phase_offset + transition.index, + transition.phase, + outcome, ) commit = run.distributed.gather_objects(commit)[0] state.phase_commits.append(commit.as_dict()) diff --git a/src/sequifier/training/phase_runners.py b/src/sequifier/training/phase_runners.py index 30e5c1dc..87a1c44f 100644 --- a/src/sequifier/training/phase_runners.py +++ b/src/sequifier/training/phase_runners.py @@ -91,6 +91,7 @@ def run_supervised_group(config, request: PortableGroup, backbone_revision): environment, integrations, initial_artifact=request.incoming, + phase_offset=request.index, identity=request.identity, on_checkpoint=request.on_checkpoint, backbone_revision=backbone_revision, @@ -114,7 +115,7 @@ def policy_group_config(config, phases, index, incoming, destination, resume): ) -def run_portable_policy_group(config, request: PortableGroup): +def run_portable_policy_group(config, request: PortableGroup, prepared): from sequifier.rl.learner import run_policy_group destination = PhaseArtifacts.for_phase( @@ -128,4 +129,4 @@ def run_portable_policy_group(config, request: PortableGroup): destination, request.resuming, ) - run_policy_group(run, request.identity, request.on_checkpoint) + run_policy_group(run, request.identity, request.on_checkpoint, prepared=prepared) diff --git a/src/sequifier/training/state.py b/src/sequifier/training/state.py index 029bd599..8e3ae968 100644 --- a/src/sequifier/training/state.py +++ b/src/sequifier/training/state.py @@ -26,7 +26,8 @@ class RunState: best_optimizer_step: int | None = None plan_stopped: bool = False initial_evaluated_phases: list[int] = field(default_factory=list) - phase_index: int = 0 + phase_index: int = 0 # Position within the native plan. + phase_offset: int = 0 # Original index of the first phase in the plan. phase_epoch: int = 0 phase_epoch_complete: bool = False source_index: int = 0 From 9d4923fafa626f67bcac72212b639597161d9cd8 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Wed, 18 Nov 2026 19:27:14 +0100 Subject: [PATCH 60/71] unify action sampling and replay --- src/sequifier/io/actions.py | 8 +- src/sequifier/rl/action_plan.py | 108 +++++++++++- src/sequifier/rl/collection.py | 8 +- src/sequifier/rl/inference.py | 10 +- src/sequifier/rl/joint_distribution.py | 181 +++++++++----------- src/sequifier/rl/parallel_grpo_collector.py | 18 +- src/sequifier/rl/policy.py | 6 +- 7 files changed, 212 insertions(+), 127 deletions(-) diff --git a/src/sequifier/io/actions.py b/src/sequifier/io/actions.py index b07b8594..e7927ef7 100644 --- a/src/sequifier/io/actions.py +++ b/src/sequifier/io/actions.py @@ -123,8 +123,10 @@ def action_record(actions, index=0): def decode_action(action, config): return { - name: field.values[int(action[name])] - if field.type == "categorical" - else float(action[name]) + name: decode_action_value(action[name], field) for name, field in config.actions.items() } + + +def decode_action_value(value, field): + return field.values[int(value)] if field.type == "categorical" else float(value) diff --git a/src/sequifier/rl/action_plan.py b/src/sequifier/rl/action_plan.py index 702253f4..9a478966 100644 --- a/src/sequifier/rl/action_plan.py +++ b/src/sequifier/rl/action_plan.py @@ -1,19 +1,39 @@ """Compile the policy binding against the model's decoder branches.""" -from dataclasses import dataclass +from collections.abc import Iterator, Mapping +from dataclasses import dataclass, field import torch +from torch import Tensor from sequifier.config.policy_config import ActionFieldConfig from sequifier.io.actions import ( action_component_key, action_latent_key, + decode_action_value, inactive_action, ) +from sequifier.io.preprocessing import normalize_values from sequifier.io.values import DomainValue from sequifier.model.decoders import AutoregressiveTransformerDecoderBranch +@dataclass +class ActionDecision: + """Batched executable values and the real-valued data needed to replay them.""" + + values: dict[str, Tensor] = field(default_factory=dict) + latents: dict[str, Tensor] = field(default_factory=dict) + components: dict[str, Tensor] = field(default_factory=dict) + + def cpu(self) -> "ActionDecision": + return ActionDecision( + {name: value.cpu() for name, value in self.values.items()}, + {name: value.cpu() for name, value in self.latents.items()}, + {name: value.cpu() for name, value in self.components.items()}, + ) + + @dataclass(frozen=True) class ActionField: spec: ActionFieldConfig @@ -30,8 +50,8 @@ def __init__(self, actor, decoder): actions = actor.config.actions dependencies = {name: set(field.active_when) for name, field in actions.items()} self.independent = {} - self.autoregressive = {} self.prefix_steps = {} + self.has_autoregression = False for branch_name, branch in decoder.branches.items(): targets = tuple(name for name in branch.target_columns if name in actions) if not targets: @@ -41,7 +61,7 @@ def __init__(self, actor, decoder): raise ValueError( "autoregressive actions must include all preceding decoder targets" ) - self.autoregressive[branch_name] = targets + self.has_autoregression |= len(targets) > 1 previous = None for name in targets: self.prefix_steps[name] = (branch_name, previous) @@ -70,9 +90,6 @@ def __init__(self, actor, decoder): self.order = tuple(order) self.position = actor.config.decision_position - self.has_autoregression = any( - len(targets) > 1 for targets in self.autoregressive.values() - ) self.fields = {} record_fields = set(actions) for name in self.order: @@ -123,8 +140,87 @@ def __init__(self, actor, decoder): component_key, ) self.record_fields = frozenset(record_fields) + self.action_names = tuple(actions) self._indices = {} + def from_record(self, actions: Mapping[str, Tensor]) -> ActionDecision: + """Translate a table or trajectory batch into the internal decision.""" + if actions.keys() - self.record_fields: + raise ValueError("joint action contains unknown fields") + return ActionDecision( + {name: actions[name] for name in self.action_names if name in actions}, + { + name: actions[field.latent_key] + for name, field in self.fields.items() + if field.latent_key is not None and field.latent_key in actions + }, + { + name: actions[field.component_key] + for name, field in self.fields.items() + if field.component_key is not None and field.component_key in actions + }, + ) + + def to_record(self, decision: ActionDecision) -> dict[str, Tensor]: + """Pack a decision in the existing flat action record format.""" + actions = {name: decision.values[name] for name in self.order} + for name in self.order: + field = self.fields[name] + if field.latent_key is not None and name in decision.latents: + actions[field.latent_key] = decision.latents[name] + if field.component_key is not None and name in decision.components: + actions[field.component_key] = decision.components[name] + return actions + + def domain_action( + self, decision: ActionDecision, index: int = 0 + ) -> dict[str, DomainValue]: + return { + name: decode_action_value( + decision.values[name][index].item(), self.fields[name].spec + ) + for name in self.action_names + } + + def outputs( + self, decoder, context, valid, statistics, decision: ActionDecision + ) -> Iterator[tuple[str, Tensor]]: + """Yield each decoder output using the values chosen at earlier steps.""" + support, _ = self.indices(context.device) + outputs = {} + for branch_name, targets in self.independent.items(): + branch = decoder.branches[branch_name] + outputs.update(branch.decode_targets(context, targets)) + prefixes = {} + for name in self.order: + if name not in self.prefix_steps: + yield name, outputs[name] + continue + branch_name, previous = self.prefix_steps[name] + branch = decoder.branches[branch_name] + if previous is None: + prefix = branch.start_prefix(context) + else: + teacher = self._teacher( + decision.values.get(previous), + support.get(previous), + valid, + statistics.get(previous), + ) + prefix = branch.append_prefix(prefixes[branch_name], previous, teacher) + prefixes[branch_name] = prefix + yield name, branch.decode_prefix(prefix, (name,))[name] + + @staticmethod + def _teacher(value, support, valid, stats): + if value is None: + dtype = torch.long if support is not None else torch.float32 + return valid.new_zeros(valid.shape, dtype=dtype) + if support is not None: + return support[value.long()].masked_fill(~valid, 0) + teacher = normalize_values(value, stats) if stats is not None else value + return teacher.float().expand(valid.shape[0]) + def indices(self, device): # The shared network can move independently of its policy view. if device not in self._indices: diff --git a/src/sequifier/rl/collection.py b/src/sequifier/rl/collection.py index c4729017..c0aed1fb 100644 --- a/src/sequifier/rl/collection.py +++ b/src/sequifier/rl/collection.py @@ -7,7 +7,7 @@ import torch from sequifier.contract_versions import ContractVersions -from sequifier.io.actions import action_record, decode_action +from sequifier.io.actions import action_record from sequifier.rl.distributions import action_generator from sequifier.rl.environment import validate_reset, validate_step from sequifier.rl.history import PolicyHistory @@ -118,10 +118,10 @@ def decide(self, value=None, *, deterministic=False, compute_entropy=True): ) if not torch.isfinite(estimate).all(): raise ValueError("non-finite collection value") - index = action_record(selected.actions) + decision = selected.decision.cpu() return PolicyDecision( - decode_action(index, actor.config), - index, + actor.action_plan.domain_action(decision), + action_record(actor.action_plan.to_record(decision)), float(selected.log_prob[0]), float(distribution.entropy(self.generator)[0]) if compute_entropy else 0.0, float(estimate[0]), diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 071235db..059a549d 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -10,7 +10,7 @@ from sequifier.config.policy_infer_config import ResolvedPolicyInferenceConfig from sequifier.helpers import configure_determinism, normalize_path, write_data -from sequifier.io.actions import action_record, decode_action, encode_action +from sequifier.io.actions import encode_action from sequifier.io.batch import SequifierInputs, collate_model_inputs from sequifier.io.policy_window import PolicyWindowView from sequifier.io.sequence import ordered_sequence_rows @@ -149,17 +149,17 @@ def predict_policy(policy, data, config): selected = distribution.select_and_score( deterministic, generator, include_probabilities=True ) - sampled = selected.actions + sampled = selected.decision scores, probabilities = selected.log_prob, selected.probabilities else: - sampled = distribution.select(deterministic, generator) + sampled = distribution.select_decision(deterministic, generator) scores, probabilities = None, {} - actions = [action_record(sampled, i) for i in range(len(rows))] + actions = distribution.domain_actions(sampled) for index, (row_index, action) in enumerate(zip(rows, actions)): output = { row_key: row_index, **identities[index], - **decode_action(action, spec), + **action, } if scores is not None: output["action_log_probability"] = float(scores[index]) diff --git a/src/sequifier/rl/joint_distribution.py b/src/sequifier/rl/joint_distribution.py index ca685757..32c348bc 100644 --- a/src/sequifier/rl/joint_distribution.py +++ b/src/sequifier/rl/joint_distribution.py @@ -6,17 +6,22 @@ import torch from torch import Tensor -from sequifier.io.actions import action_record, decode_action -from sequifier.io.preprocessing import denormalize_values, normalize_values +from sequifier.io.preprocessing import denormalize_values from sequifier.model.prediction_distribution import RealPredictionDistribution +from sequifier.rl.action_plan import ActionDecision, ActionExecutionPlan from sequifier.rl.distributions import CategoricalActionDistribution @dataclass(frozen=True) class ScoredSelection: - actions: dict[str, Tensor] + decision: ActionDecision log_prob: Tensor probabilities: dict[str, Tensor] + _plan: ActionExecutionPlan + + @property + def actions(self) -> dict[str, Tensor]: + return self._plan.to_record(self.decision) class JointActionDistribution: @@ -63,37 +68,12 @@ def row(self, index): ), ) - def _independent_parameters(self): - outputs = {} - for branch_name, targets in self.plan.independent.items(): - branch = self.decoder.branches[branch_name] - outputs.update(branch.decode_targets(self.context, targets)) - return outputs - - def _teacher(self, name, action): - field = self.plan.fields[name] - if action is None: - teacher = self.context.new_zeros( - self.batch, dtype=torch.long if field.support else torch.float32 - ) - elif field.support: - teacher = self.support[name][action.long()].masked_fill(~self.valid, 0) - else: - stats = self.statistics.get(name) - teacher = normalize_values(action, stats) if stats is not None else action - teacher = teacher.float().expand(self.batch) - return teacher - - def _parameters(self, actions): - outputs = self._independent_parameters() - for branch_name, targets in self.plan.autoregressive.items(): - branch = self.decoder.branches[branch_name] - teachers = { - name: self._teacher(name, actions.get(name)) for name in targets[:-1] - } - prefix = branch.start_prefix(self.context, teachers) - outputs.update(branch.decode_prefix(prefix, targets)) - return outputs + def _distributions(self, decision: ActionDecision, *, categorical_only=False): + for name, output in self.plan.outputs( + self.decoder, self.context, self.valid, self.statistics, decision + ): + if not categorical_only or name in self.support: + yield name, self._field(name, output) def _field(self, name, output): field = self.plan.fields[name].spec @@ -113,73 +93,61 @@ def _field(self, name, output): ) return RealPredictionDistribution(location, scale, field) - def _active(self, name, actions): + def _active(self, name, decision: ActionDecision): active = None for predecessor, allowed in self.activity.get(name, {}).items(): - if predecessor not in actions: + if predecessor not in decision.values: raise ValueError( "conditional activity requires a preceding action field" ) - matches = torch.isin(actions[predecessor], allowed) + matches = torch.isin(decision.values[predecessor], allowed) active = matches if active is None else active & matches return active def _select(self, deterministic, generator): - actions, fields, prefixes = {}, {}, {} - outputs = self._independent_parameters() - for name in self.plan.order: - if name in self.plan.prefix_steps: - branch_name, previous = self.plan.prefix_steps[name] - branch = self.decoder.branches[branch_name] - prefix = ( - branch.start_prefix(self.context) - if previous is None - else branch.append_prefix( - prefixes[branch_name], - previous, - self._teacher(previous, actions[previous]), - ) - ) - prefixes[branch_name] = prefix - outputs.update(branch.decode_prefix(prefix, (name,))) - distribution = self._field(name, outputs[name]) + decision, fields = ActionDecision(), {} + for name, distribution in self._distributions(decision): value = ( distribution.mode() if deterministic else distribution.sample(generator) ) field = self.plan.fields[name] - active = self._active(name, actions) - actions[name] = ( + active = self._active(name, decision) + decision.values[name] = ( value if active is None else torch.where(active, value, value.new_tensor(field.inactive)) ) if isinstance(distribution, RealPredictionDistribution): - actions[field.latent_key] = ( + decision.latents[name] = ( distribution.location if deterministic else distribution.sampled_latent ) - actions[field.component_key] = ( + decision.components[name] = ( torch.full_like(value, -1, dtype=torch.long) if deterministic else distribution.sampled_component ) fields[name] = distribution, active - return actions, fields + return decision, fields + + def select_decision(self, deterministic, generator) -> ActionDecision: + decision, _ = self._select(deterministic, generator) + return decision def select(self, deterministic, generator): - actions, _ = self._select(deterministic, generator) - return actions + return self.plan.to_record(self.select_decision(deterministic, generator)) def select_and_score( self, deterministic, generator, *, include_probabilities=False ): """Score the selected action with the distributions used to select it.""" - actions, fields = self._select(deterministic, generator) + decision, fields = self._select(deterministic, generator) return ScoredSelection( - actions, - self._score(actions, fields), + decision, + self._score(decision, fields), self._probabilities(fields) if include_probabilities else {}, + self.plan, ) def sample(self, generator=None): @@ -188,29 +156,28 @@ def sample(self, generator=None): def mode(self): return self.select(True, None) - def _score(self, actions, fields): + def _score(self, decision: ActionDecision, fields): score = self.context.new_zeros( self.batch, dtype=torch.float32 if self.context.device.type == "mps" else torch.float64, ) - for name in self.plan.order: - distribution, active = fields[name] + for name, (distribution, active) in fields.items(): field = self.plan.fields[name] - value = actions[name] + value = decision.values[name] if active is not None: if ((value != field.inactive) & ~active).any(): raise ValueError( f"{name}: inactive action differs from its point mass" ) value = torch.where(active, value, distribution.mode().detach()) - latent = actions.get(field.latent_key) - if ( - isinstance(distribution, RealPredictionDistribution) - and latent is not None - ): - component = actions.get(field.component_key) + if isinstance(distribution, RealPredictionDistribution): + latent = decision.latents.get(name) + component = decision.components.get(name) if active is not None: - latent = torch.where(active, latent, distribution.location.detach()) + if latent is not None: + latent = torch.where( + active, latent, distribution.location.detach() + ) if component is not None: component = torch.where(active, component, -1) log_prob = distribution.log_prob( @@ -223,24 +190,35 @@ def _score(self, actions, fields): ) return score.float() - def log_prob(self, actions): - if not isinstance(actions, Mapping) or self.plan.fields.keys() - actions.keys(): + def _decision( + self, actions: ActionDecision | Mapping[str, Tensor] + ) -> ActionDecision: + if isinstance(actions, ActionDecision): + return actions + if isinstance(actions, Mapping): + return self.plan.from_record(actions) + raise ValueError("joint likelihood requires named action fields") + + def log_prob(self, actions: ActionDecision | Mapping[str, Tensor]): + decision = self._decision(actions) + if self.plan.fields.keys() - decision.values.keys(): raise ValueError("joint likelihood requires every action field") - if actions.keys() - self.plan.record_fields: + if decision.values.keys() - self.plan.fields.keys(): raise ValueError("joint action contains unknown fields") for name in self.plan.order: - if actions[name].shape != (self.batch,): + if decision.values[name].shape != (self.batch,): raise ValueError(f"{name}: expected one action per decision") - outputs = self._parameters(actions) - fields = { - name: (self._field(name, outputs[name]), self._active(name, actions)) - for name in self.plan.order - } - return self._score(actions, fields) + return self._score( + decision, + { + name: (distribution, self._active(name, decision)) + for name, distribution in self._distributions(decision) + }, + ) def _probabilities(self, fields): result = {} - for name in self.actor.config.actions: + for name in self.plan.action_names: if name not in fields: continue distribution, active = fields[name] @@ -255,11 +233,13 @@ def _probabilities(self, fields): def categorical_probabilities(self, actions): """Per-field conditional probabilities at the supplied action prefix.""" - outputs = self._parameters(actions) + decision = self._decision(actions) return self._probabilities( { - name: (self._field(name, outputs[name]), self._active(name, actions)) - for name in self.support + name: (distribution, self._active(name, decision)) + for name, distribution in self._distributions( + decision, categorical_only=True + ) } ) @@ -270,17 +250,16 @@ def entropy_is_exact(self): and field.lower is None and field.upper is None and not field.atoms - for field in self.actor.config.actions.values() + for field in (action.spec for action in self.plan.fields.values()) ) def entropy(self, generator=None): if self.entropy_is_exact: - parameters = self._parameters({}) return ( torch.stack( [ - self._field(name, parameters[name]).entropy() - for name in self.plan.order + distribution.entropy() + for _, distribution in self._distributions(ActionDecision()) ] ) .sum(dim=0) @@ -298,11 +277,11 @@ def kl(self, reference, generator=None): "KL requires matching action measures and physical transforms" ) if not self.plan.has_autoregression and not self.activity: - current, frozen = self._parameters({}), reference._parameters({}) + current = dict(self._distributions(ActionDecision())) + frozen = dict(reference._distributions(ActionDecision())) values = [] for name, field in self.actor.config.actions.items(): - p = self._field(name, current[name]) - q = reference._field(name, frozen[name]) + p, q = current[name], frozen[name] if isinstance(p, CategoricalActionDistribution) and isinstance( q, CategoricalActionDistribution ): @@ -322,12 +301,10 @@ def kl(self, reference, generator=None): ) return torch.stack(values).sum(dim=0).float() selected = self.select_and_score(False, generator) - delta = selected.log_prob - reference.log_prob(selected.actions) + delta = selected.log_prob - reference.log_prob(selected.decision) surrogate = (delta.detach() + 1) * selected.log_prob return delta.detach() + surrogate - surrogate.detach() def domain_actions(self, actions): - return [ - decode_action(action_record(actions, i), self.actor.config) - for i in range(self.batch) - ] + decision = self._decision(actions).cpu() + return [self.plan.domain_action(decision, i) for i in range(self.batch)] diff --git a/src/sequifier/rl/parallel_grpo_collector.py b/src/sequifier/rl/parallel_grpo_collector.py index f5b74b8c..3b6ecf1f 100644 --- a/src/sequifier/rl/parallel_grpo_collector.py +++ b/src/sequifier/rl/parallel_grpo_collector.py @@ -5,7 +5,7 @@ import torch -from sequifier.io.actions import action_record, decode_action +from sequifier.io.actions import action_record from sequifier.rl.collection import PolicyDecision from sequifier.rl.environment import validate_start_token from sequifier.rl.group_collection import GroupCollection @@ -24,16 +24,20 @@ def _decide(actor, reference, cursors): # Decode and validate distributions across the full environment batch. # Only random draws use individual generators; no per-row decoder calls. selected = distribution.select_and_score(False, generators) - actions = selected.actions scores = ( torch.stack( - [selected.log_prob, reference.evaluate_actions(inputs, actions)], dim=1 + [ + selected.log_prob, + reference.evaluate_actions(inputs, selected.decision), + ], + dim=1, ) .cpu() .tolist() ) del inputs, distribution - host_actions = {name: value.cpu() for name, value in actions.items()} + host_decision = selected.decision.cpu() + host_actions = actor.action_plan.to_record(host_decision) decisions = [] for index, (old, ref) in enumerate(scores): if not math.isfinite(old) or not math.isfinite(ref): @@ -42,7 +46,11 @@ def _decide(actor, reference, cursors): decisions.append( ( PolicyDecision( - decode_action(action, actor.config), action, old, 0.0, 0.0 + actor.action_plan.domain_action(host_decision, index), + action, + old, + 0.0, + 0.0, ), ref, ) diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index ddec4f67..eb227a51 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -15,7 +15,7 @@ from sequifier.model.execution import ModelExecution from sequifier.model.execution_schema import ExecutionSchema from sequifier.model.prediction_distribution import PredictionScales -from sequifier.rl.action_plan import ActionExecutionPlan +from sequifier.rl.action_plan import ActionDecision, ActionExecutionPlan from sequifier.rl.joint_distribution import JointActionDistribution if TYPE_CHECKING: @@ -154,7 +154,9 @@ def act(self, inputs: SequifierInputs, *, deterministic: bool, generator): return selected.actions, selected.log_prob def evaluate_actions( - self, inputs: SequifierInputs, actions: Tensor | dict[str, Tensor] + self, + inputs: SequifierInputs, + actions: ActionDecision | Mapping[str, Tensor], ) -> Tensor: return self.distribution(inputs).log_prob(actions) From 80aa5153f2b21c2fa567941a784da718fba9a623 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 10:09:05 +0100 Subject: [PATCH 61/71] prevent outputs overwriting inputs --- src/sequifier/config/inference_common.py | 29 +++++++++++++++++++++++- src/sequifier/infer.py | 27 +++++++++++++++------- src/sequifier/io/output_paths.py | 21 ++++++++++++++++- src/sequifier/rl/inference.py | 4 ++++ 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index 54d0a201..0ceada0c 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -2,12 +2,15 @@ import json from dataclasses import dataclass +from pathlib import Path from typing import Literal from pydantic import ConfigDict, Field, field_validator, model_validator from sequifier.config.command_ownership import CommandRunSettings from sequifier.config.naming import ConfigModel +from sequifier.helpers import normalize_path +from sequifier.io.output_paths import validate_inference_destination class InferenceExecutionConfig(ConfigModel): @@ -44,8 +47,33 @@ def validate_resources(self): self.run.validate_for("infer") if self.output_path is not None and len(getattr(self, "model_paths", ())) != 1: raise ValueError("output_path requires exactly one model path") + rollout = getattr(self, "rollout", None) + if rollout is not None: + self.validate_output_destination( + normalize_path(rollout.trajectory_path, self.project_root), + label="trajectory_path", + ) + elif self.output_path is not None: + destination = ( + normalize_path(self.output_path, self.project_root) + if getattr(self, "task", None) == "policy" + else Path(self.project_root) / self.output_path + ) + self.validate_output_destination(destination, label="output_path") return self + def validate_output_destination( + self, destination: str | Path, *, label: str + ) -> None: + inputs = list(getattr(self, "model_paths", ())) + if (data_path := getattr(self, "data_path", None)) is not None: + inputs.append(data_path) + validate_inference_destination( + destination, + (normalize_path(path, self.project_root) for path in inputs), + label=label, + ) + class InferenceRouteConfig(ConfigModel): """Validate artifact and interface selection before route-specific options.""" @@ -118,7 +146,6 @@ def resolve(self, device): def inference_model_config(path, project_root, artifacts): """Read the interface contract once for PT artifacts, or from ONNX metadata.""" from sequifier.artifacts.model_artifact import read_execution_artifact - from sequifier.helpers import normalize_path normalized = normalize_path(path, project_root) if path.lower().endswith(".pt"): diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index 07115076..aa1c3a8b 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -14,6 +14,7 @@ from sequifier.artifacts.model_artifact import read_execution_artifact from sequifier.config.infer_config import ResolvedInferenceConfig, load_inferer_config +from sequifier.config.inference_common import InferenceExecutionConfig from sequifier.helpers import ( PANDAS_TO_TORCH_TYPES, configure_determinism, @@ -850,21 +851,27 @@ def inference_output_path( data_id: int, target_column: Optional[str] = None, output_path: Optional[str] = None, + *, + config: InferenceExecutionConfig | None = None, ) -> str: - """Return a canonical inference output path and create its directory.""" + """Prepare an inference output path, checking inputs before creating directories.""" if output_path is not None and artifact_type in {"predictions", "embeddings"}: destination = Path(project_root) / output_path if data_id: destination = destination.with_name( f"{destination.stem}-part-{data_id:03d}{destination.suffix}" ) - destination.parent.mkdir(parents=True, exist_ok=True) - return str(destination) - output_dir = Path(project_root) / "outputs" / artifact_type / model_id - if target_column is not None: - output_dir /= target_column - output_dir.mkdir(parents=True, exist_ok=True) - return str(output_dir / f"part-{data_id:03d}.{write_format}") + label = "output_path" + else: + output_dir = Path(project_root) / "outputs" / artifact_type / model_id + if target_column is not None: + output_dir /= target_column + destination = output_dir / f"part-{data_id:03d}.{write_format}" + label = f"{artifact_type} output" + if config is not None: + config.validate_output_destination(destination, label=label) + destination.parent.mkdir(parents=True, exist_ok=True) + return str(destination) @beartype @@ -988,6 +995,7 @@ def infer_embedding( model_id, data_id, output_path=config.output_path, + config=config, ) logger.info(f"Writing embeddings to '{embeddings_path}'") write_data( @@ -1154,6 +1162,7 @@ def infer_generative( model_id, data_id, target_column, + config=config, ) logger.info(f"Writing probabilities to '{probabilities_path}'") # Step 5: Finalize Output and I/O (write_data now handles Polars DF) @@ -1193,6 +1202,7 @@ def infer_generative( model_id, data_id, output_path=config.output_path, + config=config, ) logger.info(f"Writing predictions to '{predictions_path}'") write_data( @@ -1315,6 +1325,7 @@ def predict(history): }, ) path = normalize_path(rollout.trajectory_path, config.project_root) + config.validate_output_destination(path, label="trajectory_path") save_trajectory(trajectory, path) logger.info( f"Wrote {len(trajectory.transitions)} rollout transitions to '{path}'; return={trajectory.total_return}" diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py index 3063d3d0..648a61b4 100644 --- a/src/sequifier/io/output_paths.py +++ b/src/sequifier/io/output_paths.py @@ -1,4 +1,4 @@ -"""Shared path geometry and destination checks for training outputs.""" +"""Shared path geometry and destination checks.""" from pathlib import Path @@ -28,6 +28,25 @@ def validate_path_conflicts(paths, protected, *, label, check_each_other=True): raise ValueError(f"{label} {path} conflicts with input/output {other}") +def validate_inference_destination(destination, inputs, *, label): + entry = Path(destination) + # Atomic replacement acts on the directory entry even if it is a symlink. + entry = entry.parent.resolve() / entry.name + resolved = entry.resolve() + for input_path in inputs: + source = Path(input_path).resolve() + conflict = paths_overlap(resolved, source) or paths_overlap(entry, source) + if not conflict: + try: + conflict = resolved.samefile(source) + except (FileNotFoundError, NotADirectoryError): + pass + if conflict: + raise ValueError( + f"{label} {destination!s} conflicts with input {input_path}" + ) + + def require_file_destinations(paths): for path in paths: if path.is_dir() or any(p.exists() and not p.is_dir() for p in path.parents): diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 059a549d..88a70d92 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -59,9 +59,11 @@ def infer_policy(config: ResolvedPolicyInferenceConfig) -> None: "predictions", Path(config.model_paths[0]).stem, 0, + config=config, ) else: destination = normalize_path(config.output_path, config.project_root) + config.validate_output_destination(destination, label="output_path") Path(destination).parent.mkdir(parents=True, exist_ok=True) write_data(predictions, destination, config.write_format) logger.info(f"Wrote {len(predictions)} policy predictions to '{destination}'") @@ -115,6 +117,7 @@ def infer_rollout(policy, data, config): } ) destination = normalize_path(rollout.trajectory_path, config.project_root) + config.validate_output_destination(destination, label="trajectory_path") save_trajectory(trajectory, destination) logger.info( f"Wrote {len(trajectory.transitions)} policy transitions to '{destination}'" @@ -188,6 +191,7 @@ def predict_policy(policy, data, config): Path(config.model_paths[0]).stem, 0, name, + config=config, ) write_data( pl.DataFrame(records).sort(row_key).drop(row_key), From 52154f8396dc540dc270ca2cafbedeafe8ab7eb0 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 10:12:37 +0100 Subject: [PATCH 62/71] fix metadata path handling --- src/sequifier/artifacts/dependencies.py | 5 +++-- src/sequifier/config/train_config.py | 6 +++++- src/sequifier/config/training_resolution.py | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/sequifier/artifacts/dependencies.py b/src/sequifier/artifacts/dependencies.py index 49dca3d5..e3382831 100644 --- a/src/sequifier/artifacts/dependencies.py +++ b/src/sequifier/artifacts/dependencies.py @@ -131,11 +131,12 @@ def supervised_dependencies(config, incoming=None): manifest.add(path) for name, dataset in config.dataset_training.items(): for part_name, part in dataset.parts.items(): - manifest.add(root / part.metadata_config_path) + if part.metadata_from_file: + manifest.add(root / part.metadata_config_path) for split in (part.training_data_path, part.validation_data_path): if split is None: continue - path = root / split + path = Path(split) if part.storage_form == "file": manifest.add(path) else: diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index baf69b92..c6d1fcd2 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -642,6 +642,7 @@ class ResolvedDatasetPart(ConfigModel): name: str metadata_config_path: str metadata: DatasetMetadata + metadata_from_file: bool image_paths: MediaPath = None preconditioning_image_paths: MediaPath = None training_data_path: str @@ -1445,7 +1446,9 @@ def _interface_semantics(interface: ResolvedModelInterface) -> dict[str, Any]: @beartype -def resolve_supervised_datasets(config, metadata_by_part, phases, dataset_names): +def resolve_supervised_datasets( + config, metadata_by_part, phases, dataset_names, *, metadata_from_file: bool +): """Resolve prepared dataset parts and their model compatibility contracts.""" evaluated = { @@ -1517,6 +1520,7 @@ def resolve_supervised_datasets(config, metadata_by_part, phases, dataset_names) read_format=part_spec.read_format, metadata_config_path=part_spec.metadata_config_path, metadata=part_metadata, + metadata_from_file=metadata_from_file, image_paths=part_spec.image_paths, preconditioning_image_paths=part_spec.preconditioning_image_paths, training_data_path=training_path, diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 5fad07a2..d9a3d00e 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -505,7 +505,11 @@ def read_metadata(path): validate_training_contract(config, resolved_phases) datasets, dataset_interfaces, dataset_storage = resolve_supervised_datasets( - config, part_metadata, resolved_phases, supervised_names + config, + part_metadata, + resolved_phases, + supervised_names, + metadata_from_file=metadata_by_part is None, ) metadata = dict(declarations) if needs_policy_contracts: From 33db60680a95d3a63575efa7e91cc2062af5919c Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 10:22:42 +0100 Subject: [PATCH 63/71] make random run resumable --- src/sequifier/artifacts/run_checkpoint.py | 35 ++++++++++++++++++--- src/sequifier/config/ownership.py | 14 ++++++--- src/sequifier/config/training_resolution.py | 14 ++++++--- src/sequifier/training/coordinator.py | 1 + 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/sequifier/artifacts/run_checkpoint.py b/src/sequifier/artifacts/run_checkpoint.py index 7dec0a01..7f960e4a 100644 --- a/src/sequifier/artifacts/run_checkpoint.py +++ b/src/sequifier/artifacts/run_checkpoint.py @@ -323,8 +323,9 @@ def write_run_checkpoint( atomic_save(latest_path, payload) -def checkpoint_path(training_config: Any) -> Path: - resume = training_config.runtime_settings.resume +def checkpoint_path(training_config: Any, *, resume=None) -> Path: + if resume is None: + resume = training_config.runtime_settings.resume configured_path = resume.checkpoint_path if resume is not None else None if configured_path: path = Path(configured_path) @@ -341,13 +342,37 @@ def checkpoint_path(training_config: Any) -> Path: return path.resolve() -def select_run_checkpoint(training_config: Any) -> Path | None: - resume = training_config.runtime_settings.resume +def select_run_checkpoint(training_config: Any, *, resume=None) -> Path | None: + if resume is None: + resume = training_config.runtime_settings.resume if resume is None or resume.policy == "never": return None - path = checkpoint_path(training_config) + path = checkpoint_path(training_config, resume=resume) if path.is_file(): return path if resume.policy == "required": raise FileNotFoundError(f"Required run checkpoint does not exist: {path}") return None + + +def read_resume_seed(run) -> int | None: + """Read the saved run seed before resolving seed-dependent inputs.""" + from sequifier.artifacts.checkpoint_envelope import validate_checkpoint_envelope + + path = select_run_checkpoint(run, resume=run.recovery.resume) + if path is None: + return None + envelope = torch.load(path, map_location="cpu", weights_only=False) + validate_checkpoint_envelope(envelope) + recovery = envelope["recovery"] + if envelope["recovery_type"] == "supervised": + recovery = recovery["training_config"] + elif envelope["recovery_type"] == "policy": + recovery = recovery["plan"] + seed = recovery.get("seed") + if not isinstance(seed, int): + raise ValueError( + f"Checkpoint {path} has no valid run seed. " + "Set run.reproducibility.seed to the original integer." + ) + return seed diff --git a/src/sequifier/config/ownership.py b/src/sequifier/config/ownership.py index 80859fe8..e55503a6 100644 --- a/src/sequifier/config/ownership.py +++ b/src/sequifier/config/ownership.py @@ -2,7 +2,7 @@ from typing import Literal -from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic import ConfigDict, Field, StrictInt, field_validator, model_validator from sequifier.config.components import ResumeConfig from sequifier.config.naming import ConfigModel, RunSeed @@ -56,11 +56,15 @@ class ExecutionSettings(Settings): max_concat_tensors: int = Field(default=12, gt=0) -class ReproducibilitySettings(Settings): - seed: RunSeed = 1010 +class TrainingReproducibilitySettings(Settings): + seed: StrictInt | Literal["random"] = 1010 enforce_determinism: bool = False +class ReproducibilitySettings(TrainingReproducibilitySettings): + seed: RunSeed = 1010 + + class RecoverySettings(Settings): resume: ResumeConfig = Field(default_factory=ResumeConfig) latest_checkpoint_interval_minutes: float | None = Field(default=None, gt=0) @@ -93,8 +97,8 @@ class RunSettings(Settings): project_root: str = "." model_name: str execution: ExecutionSettings = Field(default_factory=ExecutionSettings) - reproducibility: ReproducibilitySettings = Field( - default_factory=ReproducibilitySettings + reproducibility: TrainingReproducibilitySettings = Field( + default_factory=TrainingReproducibilitySettings ) recovery: RecoverySettings = Field(default_factory=RecoverySettings) logging: LoggingSettings = Field(default_factory=LoggingSettings) diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index d9a3d00e..a47a3cc3 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -3,6 +3,7 @@ from pathlib import Path from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata +from sequifier.config.naming import resolve_run_seed from sequifier.config.policy_train_config import ( SequenceSourceConfig, resolve_sequence_source, @@ -339,6 +340,13 @@ def resolve_training_run( supervised = [phase for phase in phases if phase.algorithm == "supervised"] has_policy = len(supervised) != len(phases) validate_runner_settings(config.run, supervised=bool(supervised), policy=has_policy) + seed = config.run.reproducibility.seed + if seed == "random": + from sequifier.artifacts.run_checkpoint import read_resume_seed + + saved_seed = read_resume_seed(config.run) + seed = resolve_run_seed(seed) if saved_seed is None else saved_seed + config.run.reproducibility.seed = seed needs_policy_contracts = has_policy or bool( config.policy_bindings or config.initialize_from ) @@ -499,9 +507,7 @@ def read_metadata(path): phases[index - 1] if index else None, phase, resolved=True ) - resolved_phases = resolve_phases( - phases, config.dataset_training, seed=config.run.reproducibility.seed - ) + resolved_phases = resolve_phases(phases, config.dataset_training, seed=seed) validate_training_contract(config, resolved_phases) datasets, dataset_interfaces, dataset_storage = resolve_supervised_datasets( @@ -644,7 +650,7 @@ def read_metadata(path): project_root=config.run.project_root, model_name=config.run.model_name, device=config.run.execution.device, - seed=config.run.reproducibility.seed, + seed=seed, runtime_settings=training, model_options=execution.options, model=execution.model, diff --git a/src/sequifier/training/coordinator.py b/src/sequifier/training/coordinator.py index 4cb5bf43..e2e0bfdd 100644 --- a/src/sequifier/training/coordinator.py +++ b/src/sequifier/training/coordinator.py @@ -235,6 +235,7 @@ def _run_learning_plan(config, prepared): raise ValueError("learning run checkpoint differs from the requested plan") elif resume.policy == "required": raise FileNotFoundError(checkpoint) + state["seed"] = config.seed revision = None if ( From 24ccd21a92d1c2c1af819df4800897acab6dadae Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 10:30:43 +0100 Subject: [PATCH 64/71] Fix rollout serialization --- src/sequifier/io/actions.py | 19 ++++++++++++++++++- src/sequifier/rl/runtime.py | 12 ++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/sequifier/io/actions.py b/src/sequifier/io/actions.py index e7927ef7..9ef5f95d 100644 --- a/src/sequifier/io/actions.py +++ b/src/sequifier/io/actions.py @@ -9,7 +9,24 @@ def _metadata_key(name, config, prefix): key = prefix + name - while key in config.actions: + if key not in config.actions: + return key + + # Give longer action names first claim on their keys so a shorter name + # cannot take a key reached by adding the same prefix to a longer name. + used = set(config.actions) + longer_real_names = ( + other + for other, field in config.actions.items() + if field.type == "real" and len(other) > len(name) + ) + for other in sorted(longer_real_names, key=len, reverse=True): + other_key = prefix + other + while other_key in used: + other_key = prefix + other_key + used.add(other_key) + + while key in used: key = prefix + key return key diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index e8d24c71..1da5b104 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -8,7 +8,7 @@ from sequifier.config.rollout_config import InputBinding from sequifier.contract_versions import ContractVersions from sequifier.io.actions import action_component_key, action_latent_key -from sequifier.io.values import numeric_scalar +from sequifier.io.values import numeric_scalar, portable_fields from sequifier.rl.collection import PolicyCursor from sequifier.rl.environment import ( OnlineEnvironmentAdapter, @@ -92,7 +92,8 @@ def collect_policy_rollout( ) recorder = TrajectoryRecorder(episode_id, config.max_episode_steps) previous_actions, log_probabilities = [], [] - for step_id in range(max_steps): + observation = dict(initial.observation) + for _ in range(max_steps): current = cursor.history.current decision = cursor.decide(deterministic=action_selection == "deterministic") result = cursor.step(decision) @@ -103,13 +104,15 @@ def collect_policy_rollout( assert cursor.desired_return is not None returns.append(cursor.desired_return) successor = ( - dict(result.observation) if result.observation is not None else None + portable_fields(result.observation) + if result.observation is not None + else None ) action_metadata = ( decision.action if isinstance(decision.action, dict) else {} ) transition = recorder.append( - observation=dict(current.observation), + observation=observation, predictions=decision.domain_action, action_latents={ name: action_metadata[action_latent_key(name, spec)] @@ -129,6 +132,7 @@ def collect_policy_rollout( ) if transition.terminated or transition.truncated: break + observation = successor return recorder.finish( format="sequifier.policy-trajectory", version=ContractVersions.JOINT_POLICY_TRAJECTORY, From 19deadabe908422ddcf21affee7c5f7d4371414d Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 10:33:38 +0100 Subject: [PATCH 65/71] fix grpo eval field --- src/sequifier/rl/run_state.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 8548e692..4cb3ebcb 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -488,7 +488,10 @@ def evaluate_online(self, adapter, *, initial=False, **progress): result = evaluate_policy( self.context.actor, phase.evaluation, forbidden_adapter=adapter ) - if phase.algorithm == "grpo": + if ( + phase.algorithm == "grpo" + and phase.evaluation.monitor.metric == "score_mean" + ): score = float( torch.tensor( result["score_mean"] * phase.reward_scale, dtype=torch.float32 @@ -496,14 +499,11 @@ def evaluate_online(self, adapter, *, initial=False, **progress): ) if not math.isfinite(score): raise ValueError("GRPO evaluation score exceeds finite FP32 range") - self.observe( - score, **{k: v for k, v in result.items() if k != "score_mean"} - ) result["score_mean"] = score - else: - self.observe(result["selection_value"], **result) - if initial: - progress["initial"] = True + result["selection_value"] = score + self.observe(result["selection_value"], **result) + if initial and phase.algorithm == "ppo": + progress["initial"] = True self.telemetry.append( "evaluations", {**result, "transaction": self.transaction, **progress} ) From 09b3525eba073f3a7e52f144efb39d0be36b4873 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 10:47:05 +0100 Subject: [PATCH 66/71] remove logical names --- documentation/configs/naming.md | 7 +- .../configs/import-policy-sequences.yaml | 5 +- .../configs/train-iql-from-table.yaml | 12 - .../example_inputs/configs/train-iql-toy.yaml | 7 - .../configs/train-multiphase-control.yaml | 9 - .../example_inputs/iql_toy_control.py | 4 +- .../example_inputs/policy_control_data.py | 8 +- documentation/policy-learning.md | 17 +- src/sequifier/config/policy_train_config.py | 32 +- src/sequifier/contract_versions.py | 2 +- src/sequifier/templates/train-dpo.yaml | 9 - src/sequifier/templates/train-iql.yaml | 7 - src/sequifier/trajectory/export.py | 78 ++--- src/sequifier/trajectory/sources.py | 13 +- src/sequifier/trajectory/tables.py | 278 ++++++++---------- 15 files changed, 188 insertions(+), 300 deletions(-) diff --git a/documentation/configs/naming.md b/documentation/configs/naming.md index b9dd688a..51dea8ac 100644 --- a/documentation/configs/naming.md +++ b/documentation/configs/naming.md @@ -129,9 +129,10 @@ recorded in the effective configuration. `sampler_seed` controls an independent search stream. Initialization and hash seeds stay scoped integer declarations. `enforce_determinism`, `action_selection`, and `seed` remain independent controls. -Keep `seed_window_index` and `seed_row_index`, model `previous_action_inputs` and -data `previous_action_columns`, scalar `actions` and structured `action_record`, -and model-owned versus fixed policy `scale`: these are meaningful distinctions. +Use `seed_window_index` for prepared windows and `seed_row_index` for table rows. +The model declares its lagged inputs in `previous_action_inputs`. Sequence tables +use fixed column names for [recorded actions and transitions](../policy-learning.md#offline-data). +Policy `scale` can use the model distribution or a fixed value. ## Search expressions diff --git a/documentation/example_inputs/configs/import-policy-sequences.yaml b/documentation/example_inputs/configs/import-policy-sequences.yaml index f9b14741..3d1b1017 100644 --- a/documentation/example_inputs/configs/import-policy-sequences.yaml +++ b/documentation/example_inputs/configs/import-policy-sequences.yaml @@ -4,9 +4,8 @@ source: - data/demonstrations.csv columns: observations: - position: position - velocity: velocity - static_inputs: {} + - position + - velocity output_path: data/imported-demonstrations episodes_per_shard: 64 policy_binding: diff --git a/documentation/example_inputs/configs/train-iql-from-table.yaml b/documentation/example_inputs/configs/train-iql-from-table.yaml index acd5c039..55829ba6 100644 --- a/documentation/example_inputs/configs/train-iql-from-table.yaml +++ b/documentation/example_inputs/configs/train-iql-from-table.yaml @@ -39,18 +39,6 @@ dataset_training: type: sequence paths: - data/offline-transitions.parquet - columns: - sequence_id: sequenceId - item_position: itemPosition - actions: - action: action - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: - position: next_position - velocity: next_velocity model_interface: actor split: validation_fraction: 0.2 diff --git a/documentation/example_inputs/configs/train-iql-toy.yaml b/documentation/example_inputs/configs/train-iql-toy.yaml index 483cb33d..47e5c4e6 100644 --- a/documentation/example_inputs/configs/train-iql-toy.yaml +++ b/documentation/example_inputs/configs/train-iql-toy.yaml @@ -38,13 +38,6 @@ dataset_training: type: sequence paths: - data/offline-transitions.csv - columns: - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: - position: next_position model_interface: actor split: validation_fraction: 0.2 diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index 11136817..51d05de3 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -41,15 +41,6 @@ dataset_training: type: sequence paths: - data/control-decisions.csv - columns: - episode_id: episodeId - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: - cue: next_cue - group: contextId model_interface: actor split: validation_fraction: 0.2 diff --git a/documentation/example_inputs/iql_toy_control.py b/documentation/example_inputs/iql_toy_control.py index 26679d24..3252a94b 100644 --- a/documentation/example_inputs/iql_toy_control.py +++ b/documentation/example_inputs/iql_toy_control.py @@ -85,7 +85,7 @@ def generate(path): "terminated", "truncated", "bootstrap_allowed", - "next_position", + "__next_position", ], ) writer.writeheader() @@ -112,7 +112,7 @@ def generate(path): "terminated": terminated, "truncated": truncated, "bootstrap_allowed": not terminated, - "next_position": float(successor), + "__next_position": float(successor), } ) position = successor diff --git a/documentation/example_inputs/policy_control_data.py b/documentation/example_inputs/policy_control_data.py index 0ac120ea..71830639 100644 --- a/documentation/example_inputs/policy_control_data.py +++ b/documentation/example_inputs/policy_control_data.py @@ -23,7 +23,7 @@ def generate(directory): fields = [ "sequenceId", "episodeId", - "contextId", + "__group", "itemPosition", "cue", "action", @@ -31,7 +31,7 @@ def generate(directory): "terminated", "truncated", "bootstrap_allowed", - "next_cue", + "__next_cue", ] writer = csv.DictWriter(handle, fieldnames=fields) writer.writeheader() @@ -52,7 +52,7 @@ def generate(directory): { "sequenceId": identity, "episodeId": identity, - "contextId": f"context-{context}", + "__group": f"context-{context}", "itemPosition": step, "cue": cue, "action": action, @@ -60,7 +60,7 @@ def generate(directory): "terminated": step + 1 == 8, "truncated": step + 1 == length and length != 8, "bootstrap_allowed": step + 1 != 8, - "next_cue": -cue, + "__next_cue": -cue, } ) cue = -cue diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 3ecff5b1..429d3633 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -45,11 +45,18 @@ See [continuation rules](configs/train.md#optimization-across-phases) for Sequence parts accept CSV/Parquet tables, JSON trajectories, checksummed sequence manifests and prepared histories. At row `t`, observations and lagged actions form inputs; the recorded current action is the label, and reward/successor describe -its resulting transition. `columns` maps observations, static inputs, actions, -previous actions, reward, successors, boundary flags, group and episode IDs. -Use `actions`/`previous_actions` for named scalar column mappings. Structured -columns use `action_record`/`previous_action_record`; record and scalar forms -cannot be combined. +its resulting transition. Tables use `sequenceId` and `itemPosition`. +Observations and static inputs use the same names in the table, metadata and model. +When importing without a model, declare those names in the +`source.columns.observations` and `source.columns.static_inputs` lists. + +Current actions use their policy field names or a structured `__executed_action` +column. Previous actions use `__previous_action_` scalar columns or a +structured `__previous_action` column. Optional transition columns are `reward`, +`terminated`, `truncated`, and `bootstrap_allowed`. When successors are present, +provide `__next_` for every observation. Use `episodeId` for a +stable episode ID and `__group` for a collection group. +Rename external table columns to these names before importing or training. Fragments require an explicit preceding action; file boundaries alone imply no termination. Forecast trajectories require `action_semantics: executed_predictions`. diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index e5a00026..7b6c91dd 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -9,34 +9,16 @@ class PolicyTableColumns(PolicyConfigBase): - """Explicit row timing: current observation/action, then resulting reward.""" + """Canonical feature names for tables without a model input schema.""" - sequence_id: NonemptyStr = "sequenceId" - item_position: NonemptyStr = "itemPosition" - observations: dict[str, NonemptyStr] = Field(default_factory=dict) - static_inputs: dict[str, NonemptyStr] = Field(default_factory=dict) - action_record: NonemptyStr | None = None - actions: dict[str, NonemptyStr] = Field(default_factory=dict) - previous_actions: dict[str, NonemptyStr] = Field(default_factory=dict) - reward: NonemptyStr | None = "reward" - terminated: NonemptyStr | None = "terminated" - truncated: NonemptyStr | None = "truncated" - previous_action_record: NonemptyStr | None = None - successors: dict[str, NonemptyStr] = Field(default_factory=dict) - bootstrap_allowed: NonemptyStr | None = None - group: NonemptyStr | None = None - episode_id: NonemptyStr | None = None + observations: list[NonemptyStr] = Field(default_factory=list) + static_inputs: list[NonemptyStr] = Field(default_factory=list) @model_validator(mode="after") - def validate_action_mappings(self): - for singular, plural in ( - ("action_record", "actions"), - ("previous_action_record", "previous_actions"), - ): - if getattr(self, singular) is not None and getattr(self, plural): - raise ValueError( - f"columns.{singular} and columns.{plural} cannot both be declared" - ) + def validate_distinct_columns(self): + names = self.observations + self.static_inputs + if len(names) != len(set(names)): + raise ValueError("observation and static columns must be distinct") return self diff --git a/src/sequifier/contract_versions.py b/src/sequifier/contract_versions.py index 4739e190..9e29aee2 100644 --- a/src/sequifier/contract_versions.py +++ b/src/sequifier/contract_versions.py @@ -39,7 +39,7 @@ class ContractVersions: COLLECTION_ARCHIVE = 122 PPO_COLLECTION = 123 GRPO_ARCHIVE = 124 - SEQUENCE_DATA_MANIFEST = 125 + SEQUENCE_DATA_MANIFEST = 145 PREFERENCE_PAIR_MANIFEST = 126 SCALAR_POLICY_TRAJECTORY = 138 TRAJECTORY_SOURCE_MANIFEST = 139 # Authored manifest of episode files. diff --git a/src/sequifier/templates/train-dpo.yaml b/src/sequifier/templates/train-dpo.yaml index 7bfce46a..789acc50 100644 --- a/src/sequifier/templates/train-dpo.yaml +++ b/src/sequifier/templates/train-dpo.yaml @@ -41,15 +41,6 @@ dataset_training: type: sequence paths: - data/control-decisions.csv - columns: - episode_id: episodeId - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: - cue: next_cue - group: contextId model_interface: actor split: validation_fraction: 0.2 diff --git a/src/sequifier/templates/train-iql.yaml b/src/sequifier/templates/train-iql.yaml index 8fb488df..68cae494 100644 --- a/src/sequifier/templates/train-iql.yaml +++ b/src/sequifier/templates/train-iql.yaml @@ -37,13 +37,6 @@ dataset_training: type: sequence paths: - PLEASE_FILL.csv - columns: - reward: reward - terminated: terminated - truncated: truncated - bootstrap_allowed: bootstrap_allowed - successors: - OBSERVATION_COLUMN: NEXT_OBSERVATION_COLUMN model_interface: actor split: validation_fraction: 0.2 diff --git a/src/sequifier/trajectory/export.py b/src/sequifier/trajectory/export.py index beeb4deb..21bf2c7c 100644 --- a/src/sequifier/trajectory/export.py +++ b/src/sequifier/trajectory/export.py @@ -6,12 +6,20 @@ import tempfile from pathlib import Path -from sequifier.config.policy_train_config import PolicyTableColumns from sequifier.contract_versions import ContractVersions from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.sequence import TransitionColumns from sequifier.trajectory.importers import source_episodes from sequifier.trajectory.sources import file_checksum, source_paths +from sequifier.trajectory.tables import ( + ACTION_RECORD, + EPISODE_ID_COLUMN, + GROUP_COLUMN, + PREVIOUS_ACTION_RECORD, + REWARD_COLUMN, + SUCCESSOR_PREFIX, + TRANSITION_FLAGS, +) def import_sequences(config) -> Path: @@ -27,21 +35,19 @@ def import_sequences(config) -> Path: "sequence import output_path must name a new dataset directory" ) destination.parent.mkdir(parents=True, exist_ok=True) - columns = PolicyTableColumns( - action_record="__executed_action", - reward="__reward", - terminated="__terminated", - truncated="__truncated", - previous_action_record="__previous_action", - bootstrap_allowed="__bootstrap", - group="__group", - episode_id="episodeId", - successors={name: "__next_" + name for name in policy.observation_columns}, - ) fields = policy.observation_columns + policy.static_columns + successor_columns = [SUCCESSOR_PREFIX + name for name in policy.observation_columns] reserved = { - value for value in columns.model_dump().values() if isinstance(value, str) - } | set(columns.successors.values()) + "sequenceId", + "itemPosition", + EPISODE_ID_COLUMN, + ACTION_RECORD, + PREVIOUS_ACTION_RECORD, + GROUP_COLUMN, + REWARD_COLUMN, + *TRANSITION_FLAGS, + *successor_columns, + } if set(fields) & reserved: raise ValueError( "policy feature names collide with sequence import metadata columns" @@ -70,17 +76,13 @@ def import_sequences(config) -> Path: if field.type == "real" ] ) - types = {name: pa.float64() for name in ({"__reward"})} - types.update( - {name: pa.bool_() for name in ("__terminated", "__truncated", "__bootstrap")} - ) - types.update( - { - "__executed_action": action_type, - "__previous_action": action_type, - "__group": pa.string(), - } - ) + types = { + REWARD_COLUMN: pa.float64(), + **{name: pa.bool_() for name in TRANSITION_FLAGS}, + ACTION_RECORD: action_type, + PREVIOUS_ACTION_RECORD: action_type, + GROUP_COLUMN: pa.string(), + } stage = Path(tempfile.mkdtemp(prefix=".sequifier-import-", dir=destination.parent)) tables, shards, seen, schema = [], [], set(), None @@ -116,25 +118,26 @@ def flush(): }, "sequenceId": [index] * length, "itemPosition": episode.item_positions, - "episodeId": [episode.episode_id] * length, - "__executed_action": episode.actions, - "__previous_action": (episode.previous_action, *episode.actions[:-1]), - "__group": [episode.group_id] * length, + EPISODE_ID_COLUMN: [episode.episode_id] * length, + ACTION_RECORD: episode.actions, + PREVIOUS_ACTION_RECORD: ( + episode.previous_action, + *episode.actions[:-1], + ), + GROUP_COLUMN: [episode.group_id] * length, + REWARD_COLUMN: transitions.rewards or [None] * length, **{ - column: getattr(transitions, field) or [None] * length - for field, column in { - "rewards": "__reward", - "terminated": "__terminated", - "truncated": "__truncated", - "bootstrap_allowed": "__bootstrap", - }.items() + field: getattr(transitions, field) or [None] * length + for field in TRANSITION_FLAGS }, } arrays = { name: pa.array(entries, type=types.get(name)) for name, entries in values.items() } - for name, column in columns.successors.items(): + for name, column in zip( + policy.observation_columns, successor_columns, strict=True + ): arrays[column] = pa.array( [ successor[name] if successor is not None else None @@ -151,7 +154,6 @@ def flush(): "version": ContractVersions.SEQUENCE_DATA_MANIFEST, "episodes": len(seen), "decisions": sum(shard["decisions"] for shard in shards), - "columns": columns.model_dump(), "policy_schema": policy.model_dump(mode="json"), "shards": shards, "provenance": { diff --git a/src/sequifier/trajectory/sources.py b/src/sequifier/trajectory/sources.py index 92fcc83d..8e39f074 100644 --- a/src/sequifier/trajectory/sources.py +++ b/src/sequifier/trajectory/sources.py @@ -6,7 +6,6 @@ from pathlib import Path from sequifier.config.policy_train_config import ( - PolicyTableColumns, SequenceSourceConfig, resolve_sequence_source, ) @@ -113,21 +112,11 @@ def resolve_source( verified = {path: _checksum(path, checksums) for path in dependencies} resolved_paths = tuple(path.resolve() for path in paths) if manifest is not None and manifest.get("format") == "sequifier.sequence-data": - recorded_columns = PolicyTableColumns.model_validate(manifest["columns"]) - if ( - source.columns != PolicyTableColumns() - and source.columns != recorded_columns - ): - raise ValueError( - "source mappings disagree with the committed sequence manifest" - ) if source.encoding != "raw" or source.episode_end != "unknown": raise ValueError( "committed sequence data already declares its encoding and boundaries" ) - source = source.model_copy( - update={"columns": recorded_columns, "format": "table"} - ) + source = source.model_copy(update={"format": "table"}) kind = source.format if kind == "auto": suffixes = {path.suffix.lower() for path in resolved_paths} diff --git a/src/sequifier/trajectory/tables.py b/src/sequifier/trajectory/tables.py index baf65417..a92ced15 100644 --- a/src/sequifier/trajectory/tables.py +++ b/src/sequifier/trajectory/tables.py @@ -20,6 +20,16 @@ from sequifier.io.values import domain_scalar, numeric_scalar from sequifier.trajectory.prepared import prepared_table +ACTION_RECORD = "__executed_action" +PREVIOUS_ACTION_RECORD = "__previous_action" +PREVIOUS_ACTION_PREFIX = "__previous_action_" +SUCCESSOR_PREFIX = "__next_" +BOOTSTRAP_COLUMN = "bootstrap_allowed" +GROUP_COLUMN = "__group" +EPISODE_ID_COLUMN = "episodeId" +REWARD_COLUMN = "reward" +TRANSITION_FLAGS = ("terminated", "truncated", BOOTSTRAP_COLUMN) + def scan_tables(paths: tuple[Path, ...]): scans = [] @@ -151,30 +161,21 @@ class TableEpisodeReference: image_store: ImageStore | None = None def load(self) -> SequenceEpisode: - columns = self.source.columns - data = ordered_sequence_rows( - self.table.load(self.sequence_id), columns.item_position - ) + data = ordered_sequence_rows(self.table.load(self.sequence_id)) decode = model_decoder( self.metadata if self.prepared or self.source.encoding == "model" else None, self.policy.actions, ) - preceding = None + preceding_row = None if self.prepared: decisions = data.filter(pl.col("__decision")) if decisions.is_empty(): raise ValueError("prepared episode contains no split-owned decisions") - first = decisions[columns.item_position][0] - previous = data.filter(pl.col(columns.item_position) == first - 1) + first = decisions["itemPosition"][0] + previous = data.filter(pl.col("itemPosition") == first - 1) if len(previous): - mapping = columns.actions or { - name: name for name in self.policy.action_columns - } - preceding = { - name: decode(previous[column][0], name) - for name, column in mapping.items() - } + preceding_row = previous.row(0, named=True) data = decisions static_data = None if self.static_table is not None: @@ -184,7 +185,7 @@ def load(self) -> SequenceEpisode: images, masks = self.image_store.gather( torch.tensor([int(self.sequence_id)]), - torch.tensor([data[columns.item_position].to_list()]), + torch.tensor([data["itemPosition"].to_list()]), torch.ones((1, len(data)), dtype=torch.bool), ) for name, value in {**images, **masks}.items(): @@ -202,9 +203,8 @@ def load(self) -> SequenceEpisode: return table_episode( data, self.policy, - columns, sequence_id=self.sequence_id, - preceding=preceding, + preceding_row=preceding_row, static_data=static_data, episode_end=self.source.episode_end, episode_namespace=self.source.episode_namespace, @@ -212,85 +212,19 @@ def load(self) -> SequenceEpisode: ) -def table_features(data, policy, columns, static_data): - observations, static_inputs = ( - {name: mapping.get(name, name) for name in names} - for names, mapping in ( - (policy.observation_columns, columns.observations), - (policy.static_columns, columns.static_inputs), - ) - ) - action_column = columns.action_record - action_columns = columns.actions or ( - {name: name for name in policy.action_columns} - if columns.action_record is None - else {name: action_column for name in policy.action_columns} - ) - inputs = set(observations.values()) - if static_data is None: - inputs.update(static_inputs.values()) - if ( - set(action_columns.values()) & inputs - or columns.previous_action_record in action_columns.values() - or set(columns.previous_actions.values()) & set(action_columns.values()) - ): - raise ValueError( - "current action labels require distinct input/previous-action columns" - ) - if ( - set(columns.observations) - observations.keys() - or set(columns.static_inputs) - static_inputs.keys() - ): - raise ValueError("table mappings must name declared policy features") - required = ( - inputs - | {columns.item_position} - | set(action_columns.values()) - | set(columns.previous_actions.values()) - | set(columns.successors.values()) - ) - required.update( - filter( - None, - ( - columns.previous_action_record, - columns.bootstrap_allowed, - columns.group, - columns.episode_id, - ), - ) - ) - if required - set(data.columns): - raise ValueError( - f"policy table is missing columns: {sorted(required - set(data.columns))}" - ) - - for record in (columns.action_record, columns.previous_action_record): - if ( - record is not None - and data.schema[record] != pl.Null - and not isinstance(data.schema[record], pl.Struct) - ): - raise ValueError( - f"{record}: action records require a structured column; use actions/previous_actions for scalar columns" - ) - return observations, static_inputs - - def table_episode( data, policy, - columns, *, sequence_id, - preceding=None, + preceding_row=None, static_data=None, episode_end="unknown", episode_namespace="", decode=domain_scalar, ) -> SequenceEpisode: - """Canonical table-to-episode validation shared by training and inference.""" - data = ordered_sequence_rows(data, columns.item_position) + """Read an episode using the policy's canonical feature names.""" + data = ordered_sequence_rows(data) if decode is domain_scalar: def identity_decode(value, column): @@ -300,34 +234,70 @@ def identity_decode(value, column): if data.is_empty(): raise ValueError("sequence source changed while loading") - action_column = columns.action_record - action_columns = columns.actions or ( - {name: name for name in policy.action_columns} - if columns.action_record is None - else {name: action_column for name in policy.action_columns} + + observations = policy.observation_columns + static_inputs = policy.static_columns + action_names = policy.action_columns + structured_actions = ACTION_RECORD in data.columns + structured_previous = PREVIOUS_ACTION_RECORD in data.columns + action_columns = {ACTION_RECORD} if structured_actions else set(action_names) + previous_columns = [PREVIOUS_ACTION_PREFIX + name for name in action_names] + scalar_previous = not structured_previous and any( + column in data.columns for column in previous_columns ) - observations, static_inputs = table_features(data, policy, columns, static_data) + if structured_previous: + previous_inputs = {PREVIOUS_ACTION_RECORD} + elif scalar_previous: + previous_inputs = set(previous_columns) + else: + previous_inputs = set() + successor_columns = [SUCCESSOR_PREFIX + name for name in observations] + has_successors = bool(set(successor_columns) & set(data.columns)) + inputs = set(observations) + if static_data is None: + inputs.update(static_inputs) + if action_columns & (inputs | previous_inputs): + raise ValueError( + "current action labels require distinct input/previous-action columns" + ) + if previous_inputs & inputs: + raise ValueError("previous actions require distinct input columns") + if has_successors and set(successor_columns) & ( + inputs | action_columns | previous_inputs + ): + raise ValueError("successors require distinct input and action columns") + required = inputs | {"itemPosition"} | action_columns | previous_inputs + if has_successors: + required.update(successor_columns) + if missing := required - set(data.columns): + raise ValueError(f"policy table is missing columns: {sorted(missing)}") + for record in (ACTION_RECORD, PREVIOUS_ACTION_RECORD): + if ( + record in data.columns + and data.schema[record] != pl.Null + and not isinstance(data.schema[record], pl.Struct) + ): + raise ValueError(f"{record}: action records require a structured column") def canonical_action(value): if value is None: - return value + return None return { name: entry for name, entry in value.items() if name in policy.actions or entry is not None } + def action_at(row): + if structured_actions: + return canonical_action(row[ACTION_RECORD]) + return {name: decode(row[name], name) for name in action_names} + def values(column, decoder=decode): - if policy.input_encoding == "model" and column in observations.values(): - return tuple(data[column].to_list()) return tuple(decoder(value, column) for value in data[column]) def optional_values(column, decoder=decode): - if ( - column is None - or column not in data.columns - or data[column].null_count() == len(data) - ): + if column not in data.columns or data[column].null_count() == len(data): return None return values(column, decoder) @@ -337,64 +307,52 @@ def constant(entries, label): raise ValueError(f"{label} must remain constant within an episode") return entries[0] - positions = values(columns.item_position, lambda value, _: value) - if set(action_columns) != set(policy.actions): - raise ValueError("table action mappings must cover every action field") - actions = tuple( - canonical_action(dict(row[columns.action_record])) - if columns.action_record is not None - and not columns.actions - and isinstance(row[columns.action_record], dict) - else { - name: decode(row[column], name) for name, column in action_columns.items() - } - for row in data.iter_rows(named=True) - ) - if columns.previous_actions: - if set(columns.previous_actions) != set(policy.actions): - raise ValueError("previous action mappings must cover every action field") + positions = tuple(data["itemPosition"]) + actions = tuple(action_at(row) for row in data.iter_rows(named=True)) + preceding = action_at(preceding_row) if preceding_row is not None else None + if scalar_previous: previous = tuple( - {name: row[column] for name, column in columns.previous_actions.items()} - for row in data.iter_rows(named=True) + dict(zip(action_names, entries, strict=True)) + for entries in data.select(previous_columns).iter_rows() ) + if previous[1:] != actions[:-1]: + raise ValueError( + "previous actions must lag recorded actions by one decision" + ) preceding = ( None if all(value is None for value in previous[0].values()) else previous[0] ) + if structured_previous: + previous = tuple(map(canonical_action, data[PREVIOUS_ACTION_RECORD])) if previous[1:] != actions[:-1]: - raise ValueError( - "previous actions must lag recorded actions by one decision" - ) - if columns.previous_action_record is not None: - preceding_actions = values( - columns.previous_action_record, - lambda value, _: None if value is None else (canonical_action(value)), - ) - if preceding_actions[1:] != actions[:-1]: raise ValueError( "previous-action column must lag executed actions by exactly one decision" ) - preceding = preceding_actions[0] + preceding = previous[0] if static_data is not None: if len(static_data) != 1: raise ValueError("static data requires exactly one row per sequenceId") + if missing := set(static_inputs) - set(static_data.columns): + raise ValueError(f"static table is missing columns: {sorted(missing)}") else: static_data = data + + def feature_value(value, name): + return value if policy.input_encoding == "model" else decode(value, name) + static = { name: constant( - ( - value if policy.input_encoding == "model" else decode(value, column) - for value in static_data[column] - ), + (feature_value(value, name) for value in static_data[name]), f"static input {name}", ) - for name, column in static_inputs.items() + for name in static_inputs } # Strict boolean and length checks belong to TransitionColumns/SequenceEpisode. transitions = { - field: optional_values(getattr(columns, field), lambda value, _: value) - for field in ("terminated", "truncated", "bootstrap_allowed") + field: optional_values(field, lambda value, _: value) + for field in TRANSITION_FLAGS } transitions = TransitionColumns(**transitions).model_dump(exclude_none=True) declared = episode_end @@ -404,36 +362,30 @@ def constant(entries, label): raise ValueError("episode_end conflicts with the recorded final boundary") transitions[declared] = (False,) * (len(data) - 1) + (True,) successors = None - if columns.successors: - if columns.successors.keys() != observations.keys(): - raise ValueError("successor mapping must cover every observation column") + if has_successors: successors = [] - for row in data.iter_rows(named=True): - present = [ - row[column] is not None for column in columns.successors.values() - ] + for entries in data.select(successor_columns).iter_rows(): + present = [value is not None for value in entries] if any(present) and not all(present): raise ValueError("a missing successor must mask all observation fields") successors.append( { - name: row[column] - if policy.input_encoding == "model" - else decode(row[column], observations[name]) - for name, column in columns.successors.items() + name: feature_value(value, name) + for name, value in zip(observations, entries, strict=True) } if all(present) else None ) - groups = optional_values(columns.group, domain_scalar) + groups = optional_values(GROUP_COLUMN, domain_scalar) identity = ( - constant(values(columns.episode_id, domain_scalar), "episode ID") - if columns.episode_id + constant(values(EPISODE_ID_COLUMN, domain_scalar), "episode ID") + if EPISODE_ID_COLUMN in data.columns else sequence_id ) return SequenceEpisode( episode_id=episode_namespace + str(identity), item_positions=positions, - observations={name: values(column) for name, column in observations.items()}, + observations={name: values(name, feature_value) for name in observations}, actions=actions, static_inputs=static, starts_episode=positions[0] == 0 and preceding is None, @@ -444,7 +396,7 @@ def constant(entries, label): transitions=TransitionColumns( **transitions, rewards=optional_values( - columns.reward, + REWARD_COLUMN, lambda value, column: numeric_scalar(decode(value, column), "reward"), ), successors=tuple(successors) if successors is not None else None, @@ -454,17 +406,17 @@ def constant(entries, label): def table_episodes(source, policy, paths, *, prepared=False): data = prepared_table(paths, source).lazy() if prepared else scan_tables(paths) - columns = source.columns - required = {columns.sequence_id, columns.item_position} - if required - set(data.collect_schema().names()): - raise ValueError( - "policy tables require sequenceId and itemPosition, or explicit column mappings" - ) - table = IndexedTable(data, columns.sequence_id) + if {"sequenceId", "itemPosition"} - set(data.collect_schema().names()): + raise ValueError("policy tables require sequenceId and itemPosition") + if set(source.columns.observations) - set(policy.observation_columns) or set( + source.columns.static_inputs + ) - set(policy.static_columns): + raise ValueError("table columns must name declared policy features") + table = IndexedTable(data, "sequenceId") static_table = None if source.static_data_path: static_table = IndexedTable( - scan_tables((Path(source.static_data_path),)), columns.sequence_id + scan_tables((Path(source.static_data_path),)), "sequenceId" ) if ( any(end - start != 1 for start, end in static_table.ranges.values()) @@ -493,10 +445,10 @@ def table_episodes(source, policy, paths, *, prepared=False): policy, identity, table, - metadata, - prepared, - static_table, - image_store, + metadata=metadata, + prepared=prepared, + static_table=static_table, + image_store=image_store, ) for identity in table.ranges ] From 95e3679f5e5218f71a0ebc13b748cc28cc0481ac Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 10:55:31 +0100 Subject: [PATCH 67/71] separate out contracts --- .github/workflows/contract-versions.yml | 2 +- AGENTS.md | 5 ++ documentation/configs/preprocess.md | 9 +- .../ddconfigs/preprocessing_input.json | 3 +- .../example_inputs/iql_toy_control.py | 3 +- documentation/policy-learning.md | 10 ++- documentation/rollout.md | 4 +- documentation/training/artifacts.md | 2 +- release-notes/unreleased-contract-versions.md | 24 +++--- src/sequifier/artifacts/model_config.py | 18 ++-- src/sequifier/config/infer_config.py | 8 +- src/sequifier/config/metadata.py | 83 ++++++++++--------- src/sequifier/config/train_config.py | 12 ++- src/sequifier/config/training_resolution.py | 21 ++++- src/sequifier/contract_versions.py | 33 ++++---- src/sequifier/export/preflight.py | 1 + src/sequifier/io/depth_preprocess.py | 27 +++--- src/sequifier/preprocess.py | 5 +- src/sequifier/rl/control.py | 3 +- src/sequifier/rl/environment.py | 10 ++- src/sequifier/rollout/contracts.py | 1 + src/sequifier/rollout/factories.py | 8 +- src/sequifier/trajectory/schema.py | 8 +- src/sequifier/trajectory/sources.py | 30 ++++--- tools/check_contract_versions.py | 75 ++++------------- 25 files changed, 205 insertions(+), 200 deletions(-) create mode 100644 AGENTS.md diff --git a/.github/workflows/contract-versions.yml b/.github/workflows/contract-versions.yml index 05412695..f19288cb 100644 --- a/.github/workflows/contract-versions.yml +++ b/.github/workflows/contract-versions.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Validate allocations and shipped examples on the merge result + - name: Validate contract versions and shipped examples on the merge result env: TARGET_SHA: ${{ github.event.merge_group.base_sha || github.event.pull_request.base.sha }} run: python tools/check_contract_versions.py --base "$TARGET_SHA" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..41d00963 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,5 @@ +# Contract versions + +Identify each contract independently of its version number. When a reader handles multiple contracts, select the contract using an explicit discriminator such as a format tag, mode, payload kind, or protocol name. A dedicated reader or field can establish the contract through its schema context. + +Validate the version only after identifying the contract. Different contracts may share the same version number. Version changes and CI checks must be scoped to each contract. diff --git a/documentation/configs/preprocess.md b/documentation/configs/preprocess.md index 6c1001be..adcd4525 100644 --- a/documentation/configs/preprocess.md +++ b/documentation/configs/preprocess.md @@ -511,9 +511,12 @@ stores shallow tensors as `[N,W]`, deep tensors as `[N,W,D]`, boolean masks unde positions needed to enforce split ownership. Optional signed Int64 curriculum values are stored under `metadata.sample_positions`; their source names are stored under `metadata.curriculum_columns`, and multiple preserved columns use -shape `[N,C]`. Public metadata records the model-facing -`tensor_payload_version` separately from this internal storage envelope. Readers -require the current envelope version. Re-run preprocessing to migrate them. +shape `[N,C]`. Public metadata declares a model-facing `tensor_payload_kind` +of `tabular` or `depth` and the corresponding `tensor_payload_version`. A model +interface retains the dataset kind even when it selects only shallow columns +from a depth dataset. PT readers validate the envelope version separately. +Depth preprocessing resume ledgers declare `mode: serial` or `mode: parallel` +and the version for that mode. Categorical padding is the existing unknown-token ID (zero); real padding is finite zero after normalization. Temporal padding has false depth masks. diff --git a/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json b/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json index 6f43198c..828121ad 100644 --- a/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json +++ b/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json @@ -63,6 +63,7 @@ }, "window_length": 11, "max_target_offset": 1, - "tensor_payload_version": 113, + "tensor_payload_kind": "tabular", + "tensor_payload_version": 114, "stored_window_layout_version": 115 } diff --git a/documentation/example_inputs/iql_toy_control.py b/documentation/example_inputs/iql_toy_control.py index 3252a94b..95c73132 100644 --- a/documentation/example_inputs/iql_toy_control.py +++ b/documentation/example_inputs/iql_toy_control.py @@ -13,7 +13,7 @@ from pathlib import Path from sequifier.config.policy_infer_config import PolicyRolloutConfig -from sequifier.contract_versions import ContractVersions +from sequifier.contract_versions import ONLINE_POLICY_PROTOCOL, ContractVersions from sequifier.rl.checkpoint import load_policy from sequifier.rl.environment import ( EnvironmentCapabilities, @@ -27,6 +27,7 @@ class ToyControlAdapter: + protocol = ONLINE_POLICY_PROTOCOL protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER implementation_id = f"sequifier.toy-control.v{ContractVersions.TOY_CONTROL}" capabilities = EnvironmentCapabilities(seeded_reset=True) diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index 429d3633..a2b49cc0 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -43,7 +43,10 @@ See [continuation rules](configs/train.md#optimization-across-phases) for ## Offline data Sequence parts accept CSV/Parquet tables, JSON trajectories, checksummed sequence -manifests and prepared histories. At row `t`, observations and lagged actions form +manifests and prepared histories. An authored manifest that lists trajectory +files declares `format: sequifier.trajectory-source`, the +`ContractVersions.TRAJECTORY_SOURCE_MANIFEST` version, and an `episodes` list +with a `path` per entry. At row `t`, observations and lagged actions form inputs; the recorded current action is the label, and reward/successor describe its resulting transition. Tables use `sequenceId` and `itemPosition`. Observations and static inputs use the same names in the table, metadata and model. @@ -109,8 +112,11 @@ a still-present changed reference is rejected. ## Online adapters -PPO, GRPO, evaluation and policy rollout use `ContractVersions.ONLINE_POLICY_ADAPTER`: +PPO, GRPO, evaluation and policy rollout use `environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. +Adapters declare `protocol = ONLINE_POLICY_PROTOCOL` and +`protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER`, both imported from +`sequifier.contract_versions`. Factories receive `config` as keyword arguments. Implementation identity and units must match the declaration. `OnlineEnvironmentAdapter`, `EnvironmentCapabilities`, `OnlineResetResult` and `OnlineStepResult` are exported by `sequifier.rl_api`. diff --git a/documentation/rollout.md b/documentation/rollout.md index 986e9d27..4b673f7b 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -26,7 +26,9 @@ one prepared window in dataset/chunk order, following normal stride/window rules before first-subsequence filtering. Each temporal input needs exactly one producer; static inputs cannot have bindings. Extra/missing environment fields fail validation. -Forecast adapters declare `protocol_version = ContractVersions.FORECAST_ROLLOUT_ADAPTER` and implement `reset(ResetRequest)`, +Forecast adapters declare `protocol = FORECAST_ROLLOUT_PROTOCOL` and +`protocol_version = ContractVersions.FORECAST_ROLLOUT_ADAPTER`. Import both from +`sequifier.contract_versions`. They implement `reset(ResetRequest)`, `step(StepRequest) -> StepResult`, and `close()`, exported by `sequifier.rollout_api`. Factories receive `environment.config` as keyword arguments. Reset initializes from the supplied history without executing a transition; step receives copied current diff --git a/documentation/training/artifacts.md b/documentation/training/artifacts.md index a1ce4f31..46e664cc 100644 --- a/documentation/training/artifacts.md +++ b/documentation/training/artifacts.md @@ -2,7 +2,7 @@ Supervised learning, policy imitation, IQL, PPO, DPO, GRPO and mixed plans use the same public output, checkpoint, report and scalar metric contracts. Each reader requires the current version for its contract. See -[ContractVersions](../../src/sequifier/contract_versions.py) for the IDs. +[ContractVersions](../../src/sequifier/contract_versions.py) for the current versions. ## Configuration and layout diff --git a/release-notes/unreleased-contract-versions.md b/release-notes/unreleased-contract-versions.md index a9ba1883..143e0f0a 100644 --- a/release-notes/unreleased-contract-versions.md +++ b/release-notes/unreleased-contract-versions.md @@ -1,7 +1,7 @@ # Consolidated Sequifier contract versions -[ContractVersions](../src/sequifier/contract_versions.py) is the source of current -IDs for Sequifier artifacts, metadata, adapter protocols, and implementation +[ContractVersions](../src/sequifier/contract_versions.py) defines current +versions for Sequifier artifacts, metadata, adapter protocols, and implementation identifiers. Checkpoint and collection archive envelopes have their own IDs, as do the payloads they contain. @@ -15,19 +15,17 @@ do the payloads they contain. - Re-export sequence data and regenerate trajectories and collection archives. Update authored preference-pair and trajectory-source manifests to declare the current version and point to current source artifacts. -- Update external forecast adapters to declare - `ContractVersions.FORECAST_ROLLOUT_ADAPTER` and external online policy - adapters to declare `ContractVersions.ONLINE_POLICY_ADAPTER`. Use the - current implementation IDs in configurations that select the built-in or toy - control. +- Forecast adapters declare `protocol = FORECAST_ROLLOUT_PROTOCOL` and + `protocol_version = ContractVersions.FORECAST_ROLLOUT_ADAPTER`. Online policy + adapters declare `protocol = ONLINE_POLICY_PROTOCOL` and + `protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER`. Use the current + implementation IDs in configurations that select the built-in or toy control. - Initialization and policy collection use new seed namespaces. The same inputs are deterministic within the new version and can produce different seeds from earlier releases. Versioned identities change fingerprints and export preflight cache keys. The ONNX operator set remains 18. -To change a contract, allocate an ID above all active and retired IDs on the -target branch, retire the previous ID in the same change, and update the affected -boundary checks. For a pure rename, record the old and new names in -`tools/check_contract_versions.py`. Bump an envelope only when its own contract -changes. Require the Contract versions check on an up-to-date merge result, -using the merge queue when enabled. +Each contract has its own version sequence. Compare versions only after +identifying the contract. Increase the version for a changed contract and bump +an envelope only when its own contract changes. The Contract versions check +validates version changes and shipped examples against the merge target. diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index 667e6be4..a443fae2 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -4,6 +4,7 @@ import json +from sequifier.config.metadata import validate_tensor_payload_contract from sequifier.config.ownership import ExportSettings from sequifier.config.policy_config import ResolvedPolicyBinding from sequifier.config.run_capabilities import PublicationPlan @@ -113,20 +114,11 @@ def validate_interface_storage_versions(values: dict, depth_layouts=None) -> Non storage.get("version") if isinstance(storage, dict) else None, ContractVersions.STORED_WINDOW_LAYOUT, ) - # The interface stores only the depth columns it consumes. A shallow projection - # can still refer to a dataset that uses the depth metadata version. - expected = ( - ContractVersions.DEPTH_TENSOR_METADATA - if (values.get("depth_layouts") if depth_layouts is None else depth_layouts) - else ( - ContractVersions.DATASET_TENSOR_METADATA, - ContractVersions.DEPTH_TENSOR_METADATA, - ) - ) - require_contract_version( - "model dataset tensor metadata", + validate_tensor_payload_contract( + values.get("tensor_payload_kind"), values.get("tensor_payload_version"), - expected, + values.get("depth_layouts") if depth_layouts is None else depth_layouts, + projected=True, ) diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index e9f02685..a4336902 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -158,6 +158,7 @@ def _execution_metadata(interface: Any, storage: Any) -> DatasetMetadata: return DatasetMetadata.model_validate( dict( depth_layouts=interface.depth_layouts, + tensor_payload_kind=storage.tensor_payload_kind, tensor_payload_version=storage.tensor_payload_version, split_paths=storage.split_paths, column_data_types=dict(interface.column_data_types), @@ -325,11 +326,8 @@ def _assert_folder_metadata_matches( ) ): mismatches.append("prediction_alignment") - if ( - folder_metadata.tensor_payload_version - != selected_metadata.tensor_payload_version - ): - mismatches.append("tensor_payload_version") + if folder_metadata.tensor_payload_kind != selected_metadata.tensor_payload_kind: + mismatches.append("tensor_payload_kind") if folder_metadata.depth_layouts.compatibility_signature( relevant_columns ) != selected_metadata.depth_layouts.compatibility_signature(relevant_columns): diff --git a/src/sequifier/config/metadata.py b/src/sequifier/config/metadata.py index b6696969..a0a5d0e2 100644 --- a/src/sequifier/config/metadata.py +++ b/src/sequifier/config/metadata.py @@ -4,7 +4,7 @@ import copy import json -from typing import Any +from typing import Any, Literal from pydantic import ConfigDict, Field, ValidationInfo, field_validator, model_validator @@ -33,27 +33,51 @@ "image_sources", "time_alignment_column", "preconditioning_image", + "tensor_payload_kind", "tensor_payload_version", "prediction_aligned_splits", "split_index", } -def dataset_tensor_version(depth_layouts) -> int: - return ( - ContractVersions.DEPTH_TENSOR_METADATA - if depth_layouts - else ContractVersions.DATASET_TENSOR_METADATA +TensorPayloadKind = Literal["tabular", "depth"] + +_TENSOR_PAYLOAD_VERSIONS: dict[TensorPayloadKind, int] = { + "tabular": ContractVersions.DATASET_TENSOR_METADATA, + "depth": ContractVersions.DEPTH_TENSOR_METADATA, +} + + +def dataset_tensor_kind(depth_layouts) -> TensorPayloadKind: + return "depth" if depth_layouts else "tabular" + + +def dataset_tensor_version(kind: TensorPayloadKind) -> int: + return _TENSOR_PAYLOAD_VERSIONS[kind] + + +def validate_tensor_payload_contract(kind, version, depth_layouts, *, projected=False): + expected_kind = dataset_tensor_kind(depth_layouts) + # A shallow model interface may come from a dataset that has depth columns. + allowed = ( + ("tabular", "depth") if projected and not depth_layouts else (expected_kind,) + ) + if kind not in allowed: + raise ValueError( + f"tensor_payload_kind must be one of {allowed!r}, got {kind!r}" + ) + require_contract_version( + f"{kind} dataset tensor metadata", version, dataset_tensor_version(kind) ) def validate_persisted_metadata_versions(values: dict) -> None: if not isinstance(values, dict): raise ValueError("Dataset metadata must contain a mapping") - require_contract_version( - "dataset tensor metadata", + validate_tensor_payload_contract( + values.get("tensor_payload_kind"), values.get("tensor_payload_version"), - dataset_tensor_version(values.get("depth_layouts")), + values.get("depth_layouts"), ) require_contract_version( "stored-window layout", @@ -78,28 +102,20 @@ def normalize_inline_metadata(cls, values: Any) -> Any: raise ValueError( "Preprocessing metadata uses obsolete placement settings; re-run preprocessing" ) - if isinstance(values, dict) and "tensor_payload_version" not in values: + if isinstance(values, dict): + kind = dataset_tensor_kind(values.get("depth_layouts")) values = { + "tensor_payload_kind": kind, + "tensor_payload_version": dataset_tensor_version(kind), **values, - "tensor_payload_version": dataset_tensor_version( - values.get("depth_layouts") - ), } return values depth_layouts: DepthLayoutRegistryModel = Field( default_factory=DepthLayoutRegistryModel ) - tensor_payload_version: int = Field( - default=ContractVersions.DATASET_TENSOR_METADATA, - strict=True, - json_schema_extra={ - "enum": [ - ContractVersions.DATASET_TENSOR_METADATA, - ContractVersions.DEPTH_TENSOR_METADATA, - ] - }, - ) + tensor_payload_kind: TensorPayloadKind + tensor_payload_version: int = Field(strict=True) split_paths: list[str] = Field(default_factory=list) split_index: int | None = Field(default=None, ge=0) column_data_types: dict[str, str] = Field( @@ -136,19 +152,11 @@ def normalize_inline_metadata(cls, values: Any) -> Any: @model_validator(mode="after") def validate_layout_metadata(self, info: ValidationInfo): - expected: int | tuple[int, ...] = dataset_tensor_version(self.depth_layouts) - if not self.depth_layouts and (info.context or {}).get( - "projected_depth_layouts" - ): - # A model interface can project only shallow columns from a depth dataset. - expected = ( - ContractVersions.DATASET_TENSOR_METADATA, - ContractVersions.DEPTH_TENSOR_METADATA, - ) - require_contract_version( - "dataset tensor metadata", + validate_tensor_payload_contract( + self.tensor_payload_kind, self.tensor_payload_version, - expected, + self.depth_layouts, + projected=(info.context or {}).get("projected_depth_layouts", False), ) require_contract_version( "stored-window layout", @@ -266,6 +274,7 @@ def extract_inline_metadata( elif "target_offset" not in authored and isinstance(window_view, ModelWindowView): authored["target_offset"] = window_view.target_offset + tensor_kind = dataset_tensor_kind(authored.get("depth_layouts")) metadata_values: dict[str, Any] = { "split_paths": [ path @@ -276,9 +285,9 @@ def extract_inline_metadata( if path is not None ], "depth_layouts": authored.get("depth_layouts", {}), + "tensor_payload_kind": authored.get("tensor_payload_kind", tensor_kind), "tensor_payload_version": authored.get( - "tensor_payload_version", - dataset_tensor_version(authored.get("depth_layouts")), + "tensor_payload_version", dataset_tensor_version(tensor_kind) ), "column_data_types": authored.get("column_data_types", {}), "static_columns": authored.get("static_columns", []), diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index c6d1fcd2..eeb06d98 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -46,6 +46,8 @@ from sequifier.config.learning_config import SupervisedLearningConfig from sequifier.config.metadata import ( DatasetMetadata, + TensorPayloadKind, + dataset_tensor_kind, dataset_tensor_version, load_dataset_metadata, ) @@ -655,7 +657,8 @@ class ResolvedDatasetPart(ConfigModel): class ResolvedInterfaceStorage(ConfigModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - tensor_payload_version: int = ContractVersions.DATASET_TENSOR_METADATA + tensor_payload_kind: TensorPayloadKind + tensor_payload_version: int = Field(strict=True) split_paths: list[str] = Field(default_factory=list) prediction_aligned_splits: list[int] = Field(default_factory=list) prediction_alignment_steps: int = 1 @@ -670,6 +673,7 @@ def from_metadata( cls, metadata: DatasetMetadata, layout: StoredWindowLayout | None = None ) -> ResolvedInterfaceStorage: return cls( + tensor_payload_kind=metadata.tensor_payload_kind, tensor_payload_version=metadata.tensor_payload_version, split_paths=metadata.split_paths, prediction_aligned_splits=metadata.prediction_aligned_splits, @@ -1717,6 +1721,7 @@ def resolve_sequifier_config( "depth_layouts", "image_sources", "preconditioning_image", + "tensor_payload_kind", "tensor_payload_version", "metadata_by_part", "column_data_types", @@ -1763,12 +1768,13 @@ def _inline_metadata( if global_spec.context_length is None: raise ValueError("inline metadata requires context_length or window_length") window_length = global_spec.context_length + max(1, global_spec.target_offset) + tensor_kind = dataset_tensor_kind(values.get("depth_layouts")) return DatasetMetadata.model_validate( { "depth_layouts": values.get("depth_layouts", {}), + "tensor_payload_kind": values.get("tensor_payload_kind", tensor_kind), "tensor_payload_version": values.get( - "tensor_payload_version", - dataset_tensor_version(values.get("depth_layouts")), + "tensor_payload_version", dataset_tensor_version(tensor_kind) ), "split_paths": values.get("split_paths", []), "column_data_types": values.get("column_data_types", {}), diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index a47a3cc3..5ebcacea 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -2,7 +2,12 @@ from pathlib import Path -from sequifier.config.metadata import DatasetMetadata, load_dataset_metadata +from sequifier.config.metadata import ( + DatasetMetadata, + dataset_tensor_kind, + dataset_tensor_version, + load_dataset_metadata, +) from sequifier.config.naming import resolve_run_seed from sequifier.config.policy_train_config import ( SequenceSourceConfig, @@ -222,6 +227,11 @@ def _history_initialized_metadata(config, name, binding, source, storage): values.pop(column, None) if column in original: values[column] = original[column] + tensor_kind = ( + storage.tensor_payload_kind + if storage + else dataset_tensor_kind(source.depth_layouts) + ) return metadata.model_copy( update={ "static_columns": static.columns if static else [], @@ -230,9 +240,12 @@ def _history_initialized_metadata(config, name, binding, source, storage): "preconditioning_image": source.preconditioning_image, "special_token_ids": source.special_token_ids, "depth_layouts": source.depth_layouts, - "tensor_payload_version": storage.tensor_payload_version - if storage - else metadata.tensor_payload_version, + "tensor_payload_kind": tensor_kind, + "tensor_payload_version": ( + storage.tensor_payload_version + if storage + else dataset_tensor_version(tensor_kind) + ), "normalize_real_columns": source.normalize_real_columns, } ) diff --git a/src/sequifier/contract_versions.py b/src/sequifier/contract_versions.py index 9e29aee2..25aa4686 100644 --- a/src/sequifier/contract_versions.py +++ b/src/sequifier/contract_versions.py @@ -1,11 +1,7 @@ -"""Sequifier's global allocation of persisted contract and protocol versions.""" +"""Current versions scoped to individual Sequifier contracts.""" class ContractVersions: - HISTORICAL_MIN = 0 - HISTORICAL_MAX = 99 - RETIRED = (101, 102, 104) - # Checkpoint envelope and runner-owned payloads. RUN_CHECKPOINT = 100 SUPERVISED_RECOVERY = 140 @@ -25,13 +21,13 @@ class ContractVersions: # Dataset storage and preprocessing resume records. PT_TENSOR_BATCH = 112 - DATASET_TENSOR_METADATA = 113 - DEPTH_TENSOR_METADATA = 114 + DATASET_TENSOR_METADATA = 114 + DEPTH_TENSOR_METADATA = 115 STORED_WINDOW_LAYOUT = 115 PREPROCESSING_MANIFEST = 116 VIDEO_FRAME_INDEX = 117 - SERIAL_DEPTH_LEDGER = 118 - PARALLEL_DEPTH_LEDGER = 119 + SERIAL_DEPTH_LEDGER = 119 + PARALLEL_DEPTH_LEDGER = 120 # Rollout records, algorithm archives, and offline source manifests. FORECAST_TRAJECTORY = 120 @@ -42,10 +38,10 @@ class ContractVersions: SEQUENCE_DATA_MANIFEST = 145 PREFERENCE_PAIR_MANIFEST = 126 SCALAR_POLICY_TRAJECTORY = 138 - TRAJECTORY_SOURCE_MANIFEST = 139 # Authored manifest of episode files. + TRAJECTORY_SOURCE_MANIFEST = 140 # Authored manifest of episode files. - FORECAST_ROLLOUT_ADAPTER = 127 - ONLINE_POLICY_ADAPTER = 128 + FORECAST_ROLLOUT_ADAPTER = 128 + ONLINE_POLICY_ADAPTER = 129 MODEL_EXECUTION_SCHEMA = 129 EXPORT_PREFLIGHT = 130 @@ -61,15 +57,16 @@ class ContractVersions: PHASE_COMMIT = 143 PHASE_BOUNDARY = 144 - # External ONNX operator set, outside the Sequifier allocation. + # External ONNX operator set. ONNX_OPSET = 18 -def require_contract_version( - contract: str, actual, expected: int | tuple[int, ...] -) -> None: - allowed = (expected,) if isinstance(expected, int) else expected - if not isinstance(actual, int) or isinstance(actual, bool) or actual not in allowed: +FORECAST_ROLLOUT_PROTOCOL = "sequifier.forecast-rollout-adapter" +ONLINE_POLICY_PROTOCOL = "sequifier.online-policy-adapter" + + +def require_contract_version(contract: str, actual, expected: int) -> None: + if not isinstance(actual, int) or isinstance(actual, bool) or actual != expected: raise ValueError( f"Unsupported {contract} version {actual!r}, expected {expected}" ) diff --git a/src/sequifier/export/preflight.py b/src/sequifier/export/preflight.py index 13f092f0..0dca9a99 100644 --- a/src/sequifier/export/preflight.py +++ b/src/sequifier/export/preflight.py @@ -42,6 +42,7 @@ def graph_only(value): "selected_columns_statistics", "split_paths", "storage_layout", + "tensor_payload_kind", "tensor_payload_version", } } diff --git a/src/sequifier/io/depth_preprocess.py b/src/sequifier/io/depth_preprocess.py index c53cbbf2..70f38d6e 100644 --- a/src/sequifier/io/depth_preprocess.py +++ b/src/sequifier/io/depth_preprocess.py @@ -1218,34 +1218,33 @@ def fit_jobs(): ledger_path = scratch_root / "depth-shards.json" source_identity = [list(entry) for entry in source_snapshot] parallel_render = parallel_groups is not None + ledger_versions = { + "serial": ContractVersions.SERIAL_DEPTH_LEDGER, + "parallel": ContractVersions.PARALLEL_DEPTH_LEDGER, + } if owner.continue_preprocessing and ledger_path.exists(): ledger = json.loads(ledger_path.read_text()) + mode = ledger.get("mode") if isinstance(ledger, dict) else None + if not isinstance(mode, str) or mode not in ledger_versions: + raise ValueError("Depth resume ledger requires serial or parallel mode") require_contract_version( - "depth ledger", - ledger.get("version"), - ( - ContractVersions.SERIAL_DEPTH_LEDGER, - ContractVersions.PARALLEL_DEPTH_LEDGER, - ), + "depth ledger", ledger.get("version"), ledger_versions[mode] ) if ledger.get("sources") != source_identity: raise ValueError( "Depth resume source files differ from the completed shards" ) - was_parallel = ledger["version"] == ContractVersions.PARALLEL_DEPTH_LEDGER - if was_parallel and not parallel_render: + if mode == "parallel" and not parallel_render: raise ValueError( "Depth parallel resume requires the original Parquet row groups " "and num_workers > 1" ) - parallel_render = was_parallel + parallel_render = mode == "parallel" else: + mode = "parallel" if parallel_render else "serial" ledger = { - "version": ( - ContractVersions.PARALLEL_DEPTH_LEDGER - if parallel_render - else ContractVersions.SERIAL_DEPTH_LEDGER - ), + "mode": mode, + "version": ledger_versions[mode], "sources": source_identity, "shards": {}, } diff --git a/src/sequifier/preprocess.py b/src/sequifier/preprocess.py index 0704aa97..5ebf15bf 100644 --- a/src/sequifier/preprocess.py +++ b/src/sequifier/preprocess.py @@ -23,6 +23,7 @@ from sequifier.config.depth_layout import DepthLayoutRegistryModel from sequifier.config.image_config import ImageSourceConfig, PreconditioningImageConfig from sequifier.config.metadata import ( + dataset_tensor_kind, dataset_tensor_version, validate_persisted_metadata_versions, ) @@ -2886,9 +2887,11 @@ def _cleanup(self, write_format: str) -> None: @beartype def _layout_metadata(self) -> dict[str, Any]: + tensor_kind = dataset_tensor_kind(self.depth_layouts) return { "depth_layouts": self.depth_layouts.model_dump(mode="json"), - "tensor_payload_version": dataset_tensor_version(self.depth_layouts), + "tensor_payload_kind": tensor_kind, + "tensor_payload_version": dataset_tensor_version(tensor_kind), "window_length": self.storage_layout.window_length, "window_lengths": self.window_lengths, "prediction_alignment_steps": self.prediction_alignment_steps, diff --git a/src/sequifier/rl/control.py b/src/sequifier/rl/control.py index fc992326..0f85501c 100644 --- a/src/sequifier/rl/control.py +++ b/src/sequifier/rl/control.py @@ -7,7 +7,7 @@ import random -from sequifier.contract_versions import ContractVersions +from sequifier.contract_versions import ONLINE_POLICY_PROTOCOL, ContractVersions from sequifier.rl.environment import ( EnvironmentCapabilities, OnlineResetResult, @@ -16,6 +16,7 @@ class CategoricalControlAdapter: + protocol = ONLINE_POLICY_PROTOCOL protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER implementation_id = ( f"sequifier.categorical-control.v{ContractVersions.CATEGORICAL_CONTROL}" diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py index 61c97849..df1698bb 100644 --- a/src/sequifier/rl/environment.py +++ b/src/sequifier/rl/environment.py @@ -7,7 +7,11 @@ import numpy as np -from sequifier.contract_versions import ContractVersions, require_contract_version +from sequifier.contract_versions import ( + ONLINE_POLICY_PROTOCOL, + ContractVersions, + require_contract_version, +) from sequifier.io.values import ( DomainValue, numeric_scalar, @@ -67,6 +71,7 @@ def __post_init__(self): class OnlineEnvironmentAdapter(Protocol): + protocol: str protocol_version: int implementation_id: str capabilities: EnvironmentCapabilities @@ -97,6 +102,8 @@ def load_online_adapter(spec): def validate_online_adapter(adapter, spec): + if getattr(adapter, "protocol", None) != ONLINE_POLICY_PROTOCOL: + raise ValueError(f"adapter.protocol must be {ONLINE_POLICY_PROTOCOL!r}") require_contract_version( "online policy environment adapter protocol", getattr(adapter, "protocol_version", None), @@ -225,6 +232,7 @@ def adapter_identity(adapter): return { "module": type(adapter).__module__, "class": type(adapter).__qualname__, + "protocol": adapter.protocol, "protocol_version": adapter.protocol_version, "implementation_id": adapter.implementation_id, "capabilities": asdict(adapter.capabilities), diff --git a/src/sequifier/rollout/contracts.py b/src/sequifier/rollout/contracts.py index 14f7ae3a..f4d8e077 100644 --- a/src/sequifier/rollout/contracts.py +++ b/src/sequifier/rollout/contracts.py @@ -35,6 +35,7 @@ class StepResult: class EnvironmentAdapter(Protocol): + protocol: str protocol_version: int def reset(self, request: ResetRequest) -> None: diff --git a/src/sequifier/rollout/factories.py b/src/sequifier/rollout/factories.py index 8bf7d9c5..b7d84bc6 100644 --- a/src/sequifier/rollout/factories.py +++ b/src/sequifier/rollout/factories.py @@ -4,11 +4,17 @@ from typing import cast from sequifier.config.rollout_config import AdapterConfig -from sequifier.contract_versions import ContractVersions, require_contract_version +from sequifier.contract_versions import ( + FORECAST_ROLLOUT_PROTOCOL, + ContractVersions, + require_contract_version, +) from sequifier.rollout.contracts import EnvironmentAdapter def validate_adapter(adapter): + if getattr(adapter, "protocol", None) != FORECAST_ROLLOUT_PROTOCOL: + raise ValueError(f"adapter.protocol must be {FORECAST_ROLLOUT_PROTOCOL!r}") protocol_version = getattr(adapter, "protocol_version", None) require_contract_version( "forecast rollout adapter protocol", diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index 8235ef28..cb64c452 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -77,10 +77,8 @@ class Trajectory(Record): "sequifier.numeric-forecast-trajectory", "sequifier.scalar-policy-trajectory", "sequifier.policy-trajectory", - ] = "sequifier.numeric-forecast-trajectory" - version: int = Field( - json_schema_extra={"enum": sorted(_TRAJECTORY_VERSIONS.values())} - ) + ] + version: int = Field(strict=True) episode_id: str = Field(min_length=1) seed: SeedHistory bindings: list[InputBinding] @@ -94,7 +92,7 @@ class Trajectory(Record): @classmethod def validate_version(cls, values): if isinstance(values, dict): - kind = values.get("format", "sequifier.numeric-forecast-trajectory") + kind = values.get("format") if isinstance(kind, str) and kind in _TRAJECTORY_VERSIONS: require_contract_version( kind, values.get("version"), _TRAJECTORY_VERSIONS[kind] diff --git a/src/sequifier/trajectory/sources.py b/src/sequifier/trajectory/sources.py index 8e39f074..0f6cdd7b 100644 --- a/src/sequifier/trajectory/sources.py +++ b/src/sequifier/trajectory/sources.py @@ -44,19 +44,21 @@ def _source_files(source, checksums): manifest = json.loads(content.decode("utf-8")) if not isinstance(manifest, dict): raise ValueError("Source manifest must be a mapping") - sequence_manifest = manifest.get("format") == "sequifier.sequence-data" - require_contract_version( - "sequence data manifest" - if sequence_manifest - else "trajectory source manifest", - manifest.get("version"), - ContractVersions.SEQUENCE_DATA_MANIFEST - if sequence_manifest - else ContractVersions.TRAJECTORY_SOURCE_MANIFEST, - ) - entries = manifest.get("shards" if sequence_manifest else "episodes") + manifest_format = manifest.get("format") + if manifest_format == "sequifier.sequence-data": + version = ContractVersions.SEQUENCE_DATA_MANIFEST + entries_field = "shards" + elif manifest_format == "sequifier.trajectory-source": + version = ContractVersions.TRAJECTORY_SOURCE_MANIFEST + entries_field = "episodes" + else: + raise ValueError(f"Unsupported source manifest format {manifest_format!r}") + require_contract_version(manifest_format, manifest.get("version"), version) + entries = manifest.get(entries_field) if not isinstance(entries, list) or not entries: - raise ValueError("source manifest requires a nonempty episodes list") + raise ValueError( + f"source manifest requires a nonempty {entries_field} list" + ) paths = [] for entry in entries: if ( @@ -66,7 +68,9 @@ def _source_files(source, checksums): ): raise ValueError("each manifest entry requires a nonempty path") path = manifest_path.parent / entry["path"] - if sequence_manifest and _checksum(path, checksums) != entry.get("sha256"): + if manifest_format == "sequifier.sequence-data" and _checksum( + path, checksums + ) != entry.get("sha256"): raise ValueError(f"sequence shard checksum disagrees with {path}") if ( "episode_id" in entry diff --git a/tools/check_contract_versions.py b/tools/check_contract_versions.py index 8e829704..1ad0504a 100644 --- a/tools/check_contract_versions.py +++ b/tools/check_contract_versions.py @@ -1,4 +1,4 @@ -"""Check global contract allocations against the target commit of a merge.""" +"""Check contract versions against the merge target.""" import argparse import ast @@ -8,10 +8,6 @@ from pathlib import Path REGISTRY = "src/sequifier/contract_versions.py" -NON_CONTRACTS = {"HISTORICAL_MIN", "HISTORICAL_MAX", "ONNX_OPSET", "RETIRED"} - -# Put explicit old-name to new-name mappings here when preserving a contract ID. -RENAMES: dict[str, str] = {} def registry(source): @@ -37,62 +33,17 @@ def registry(source): return values -def validate_allocations(current, base, renames=RENAMES): - if current.get("HISTORICAL_MIN") != 0 or current.get("HISTORICAL_MAX") != 99: - raise ValueError("The historical contract reservation must remain 0–99") - active = { - name: value for name, value in current.items() if name not in NON_CONTRACTS - } - retired = current["RETIRED"] - if not isinstance(retired, tuple) or any( - not isinstance(value, int) or isinstance(value, bool) for value in retired - ): - raise ValueError("RETIRED must be a tuple of integer IDs") - if not active or any( - not isinstance(value, int) - or isinstance(value, bool) - or value <= current["HISTORICAL_MAX"] - for value in (*active.values(), *retired) - ): - raise ValueError( - "Active and retired IDs must be integers above the historical range" - ) - if len(set(active.values())) != len(active): - raise ValueError("Duplicate active contract IDs") - if len(set(retired)) != len(retired) or set(retired) & set(active.values()): - raise ValueError("Duplicate or reused retired contract IDs") - - old_active = { - name: value for name, value in base.items() if name not in NON_CONTRACTS - } - old_retired = set(base.get("RETIRED", ())) - if not old_retired <= set(retired): - raise ValueError("Retirement records may only be added") - high_watermark = max(current["HISTORICAL_MAX"], *old_active.values(), *old_retired) - for value in set(retired) - old_retired - set(old_active.values()): - if value <= high_watermark: - raise ValueError( - f"Retired ID {value} was never allocated on the target branch" - ) - preserved: set[str] = set() - for name, old_id in old_active.items(): - replacement = renames.get(name) - if active.get(name) == old_id: - preserved.add(name) - elif ( - replacement is not None - and name not in active - and replacement not in old_active - and active.get(replacement) == old_id +def validate_versions(current, base): + for name, value in current.items(): + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise ValueError(f"{name} must be a positive integer") + previous = base.get(name) + if ( + isinstance(previous, int) + and not isinstance(previous, bool) + and value < previous ): - preserved.add(replacement) - elif old_id not in retired: - raise ValueError(f"{name} must retire ID {old_id}") - for name, value in active.items(): - if name not in preserved and value <= high_watermark: - raise ValueError( - f"{name} allocates ID {value} at or below target high-water mark {high_watermark}" - ) + raise ValueError(f"{name} cannot decrease from {previous} to {value}") VERSION_FIELDS = { @@ -245,6 +196,8 @@ def validate_boundaries(root, values): "documentation/example_inputs/configs/ddconfigs/preprocessing_input.json" ) metadata = json.loads((root / metadata_path).read_text()) + if metadata.get("tensor_payload_kind") != "tabular": + errors.append(f"{metadata_path}: tensor_payload_kind must be tabular") for field, name in ( ("tensor_payload_version", "DATASET_TENSOR_METADATA"), ("stored_window_layout_version", "STORED_WINDOW_LAYOUT"), @@ -283,7 +236,7 @@ def main(): else {} ) current = registry((root / REGISTRY).read_text()) - validate_allocations(current, base) + validate_versions(current, base) validate_boundaries(root, current) From 4720cb137ac0fd0eb6f2c46ebf1ce26d429d87d6 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 11:10:56 +0100 Subject: [PATCH 68/71] reset contracts --- .github/workflows/contract-versions.yml | 6 +- AGENTS.md | 2 + documentation/configs/infer.md | 2 +- documentation/configs/preprocess.md | 12 ++- documentation/configs/train.md | 5 +- .../ddconfigs/preprocessing_input.json | 4 +- .../configs/rollout-ppo-toy.yaml | 1 - .../configs/train-multiphase-control.yaml | 4 - .../example_inputs/iql_toy_control.py | 15 ++-- .../example_inputs/policy_control_data.py | 3 - documentation/policy-learning.md | 22 +++-- documentation/rollout.md | 10 +-- documentation/supervised-policy-learning.md | 4 +- documentation/training/artifacts.md | 16 ++-- .../training/runtime-architecture.md | 7 +- release-notes/unreleased-contract-versions.md | 23 ++--- src/sequifier/artifacts/loading.py | 1 - src/sequifier/config/metadata.py | 27 +++--- src/sequifier/config/policy_plan.py | 6 -- src/sequifier/config/rollout_config.py | 5 -- src/sequifier/config/train_config.py | 21 ++--- src/sequifier/contract_versions.py | 83 +++++++++---------- src/sequifier/io/pt_payload.py | 2 +- src/sequifier/rl/control.py | 7 +- src/sequifier/rl/environment.py | 24 ++++-- src/sequifier/rl/runtime.py | 4 +- src/sequifier/rl_api.py | 2 + src/sequifier/rollout/contracts.py | 8 ++ src/sequifier/rollout/coordinator.py | 2 - src/sequifier/rollout_api.py | 2 + src/sequifier/templates/train-grpo.yaml | 2 - src/sequifier/templates/train-ppo.yaml | 2 - src/sequifier/training/metrics.py | 8 +- src/sequifier/trajectory/preferences.py | 19 +---- src/sequifier/trajectory/schema.py | 20 ----- src/sequifier/trajectory/sources.py | 12 ++- src/sequifier/trajectory/store.py | 20 ++++- tools/check_contract_versions.py | 62 ++++---------- 38 files changed, 191 insertions(+), 284 deletions(-) diff --git a/.github/workflows/contract-versions.yml b/.github/workflows/contract-versions.yml index f19288cb..22e32087 100644 --- a/.github/workflows/contract-versions.yml +++ b/.github/workflows/contract-versions.yml @@ -12,12 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: actions/setup-python@v5 with: python-version: '3.12' - name: Validate contract versions and shipped examples on the merge result - env: - TARGET_SHA: ${{ github.event.merge_group.base_sha || github.event.pull_request.base.sha }} - run: python tools/check_contract_versions.py --base "$TARGET_SHA" + run: python tools/check_contract_versions.py diff --git a/AGENTS.md b/AGENTS.md index 41d00963..a1f342c4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,3 +3,5 @@ Identify each contract independently of its version number. When a reader handles multiple contracts, select the contract using an explicit discriminator such as a format tag, mode, payload kind, or protocol name. A dedicated reader or field can establish the contract through its schema context. Validate the version only after identifying the contract. Different contracts may share the same version number. Version changes and CI checks must be scoped to each contract. + +Contract versions are internal and must never be user-facing. Keep version values out of user-authored configs and manifests, shipped examples, user documentation, and CLI output. This applies to standalone fields and to versions embedded in identifiers or other strings. Sequifier supplies and validates versions internally, including in generated artifact metadata. diff --git a/documentation/configs/infer.md b/documentation/configs/infer.md index cfab48e1..4ab1e8c3 100644 --- a/documentation/configs/infer.md +++ b/documentation/configs/infer.md @@ -316,7 +316,7 @@ Forecast rejects policy-only output/return/action/history options; embeddings al reject generative sampling, decoding, probabilities, autoregression and rollout. Policy rollout declares `environment: {factory: module:factory, config: {}, -implementation_id: ..., units: {}}`, required `max_steps`, `seed_row_index` and +units: {}}`, required `max_steps`, `seed_row_index` and `trajectory_path`, plus optional `max_episode_steps`. It rejects forecast `bindings`. Forecast rollout uses the forecast rollout adapter protocol, requires bindings and selects `seed_window_index`. Neither accepts `rollout.type` or `seed_index`. diff --git a/documentation/configs/preprocess.md b/documentation/configs/preprocess.md index adcd4525..a9a7302b 100644 --- a/documentation/configs/preprocess.md +++ b/documentation/configs/preprocess.md @@ -505,18 +505,16 @@ always allowed. With gaps enabled, physical slots remain unchanged. Outer item positions must be continuous within each selected sequence. An item in this raw format must have at least one child; null child rows do not encode emptiness. -All PT files use the current `ContractVersions.PT_TENSOR_BATCH` version of the `sequifier_tensor_batch` envelope. The payload +Sequifier writes and validates the PT tensor-batch envelope. The payload stores shallow tensors as `[N,W]`, deep tensors as `[N,W,D]`, boolean masks under `metadata.depth_valid_masks.`, and the absolute window and split-boundary positions needed to enforce split ownership. Optional signed Int64 curriculum values are stored under `metadata.sample_positions`; their source names are stored under `metadata.curriculum_columns`, and multiple preserved columns use -shape `[N,C]`. Public metadata declares a model-facing `tensor_payload_kind` -of `tabular` or `depth` and the corresponding `tensor_payload_version`. A model -interface retains the dataset kind even when it selects only shallow columns -from a depth dataset. PT readers validate the envelope version separately. -Depth preprocessing resume ledgers declare `mode: serial` or `mode: parallel` -and the version for that mode. +shape `[N,C]`. Generated metadata records whether the dataset is tabular +or includes depth. A model interface retains the dataset kind even when it selects +only shallow columns from a depth dataset. Depth preprocessing resume ledgers +record whether preprocessing ran in serial or parallel. Categorical padding is the existing unknown-token ID (zero); real padding is finite zero after normalization. Temporal padding has false depth masks. diff --git a/documentation/configs/train.md b/documentation/configs/train.md index 80295355..269632d0 100644 --- a/documentation/configs/train.md +++ b/documentation/configs/train.md @@ -1293,9 +1293,8 @@ now take effect; exact resume of those older runs is rejected. `initialization_seed` accepts integers from zero through `2**63-1`, excluding booleans. An explicit seed isolates constructor and custom-initializer randomness -from the run stream. Descendants derive seeds using SHA-256 over `ContractVersions.INITIALIZATION_SEED_DERIVATION`, -namespace seed, relative branch path, and phase; explicit child seeds start a new -namespace. Omitted seeds preserve the existing RNG stream and initialization +from the run stream. Descendants derive seeds from the namespace seed, relative +branch path, and phase. Explicit child seeds start a new namespace. Omitted seeds preserve the existing RNG stream and initialization traversal. Initialization seeds do not control runtime dropout. Dropout continues using the run RNG and checkpointed per-rank state. diff --git a/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json b/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json index 828121ad..1c5f892e 100644 --- a/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json +++ b/documentation/example_inputs/configs/ddconfigs/preprocessing_input.json @@ -64,6 +64,6 @@ "window_length": 11, "max_target_offset": 1, "tensor_payload_kind": "tabular", - "tensor_payload_version": 114, - "stored_window_layout_version": 115 + "tensor_payload_version": 100, + "stored_window_layout_version": 100 } diff --git a/documentation/example_inputs/configs/rollout-ppo-toy.yaml b/documentation/example_inputs/configs/rollout-ppo-toy.yaml index ef98214a..04fae968 100644 --- a/documentation/example_inputs/configs/rollout-ppo-toy.yaml +++ b/documentation/example_inputs/configs/rollout-ppo-toy.yaml @@ -10,7 +10,6 @@ rollout: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 action_selection: deterministic task: policy model_paths: diff --git a/documentation/example_inputs/configs/train-multiphase-control.yaml b/documentation/example_inputs/configs/train-multiphase-control.yaml index 51d05de3..e3f82d3b 100644 --- a/documentation/example_inputs/configs/train-multiphase-control.yaml +++ b/documentation/example_inputs/configs/train-multiphase-control.yaml @@ -112,13 +112,11 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 evaluation: environment: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 seeds: - 17 - 29 @@ -145,7 +143,6 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 total_env_steps: 8192 rollout_env_steps: 256 optimization_epochs: 4 @@ -158,7 +155,6 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 seeds: - 17 - 29 diff --git a/documentation/example_inputs/iql_toy_control.py b/documentation/example_inputs/iql_toy_control.py index 95c73132..dcd838cc 100644 --- a/documentation/example_inputs/iql_toy_control.py +++ b/documentation/example_inputs/iql_toy_control.py @@ -13,23 +13,21 @@ from pathlib import Path from sequifier.config.policy_infer_config import PolicyRolloutConfig -from sequifier.contract_versions import ONLINE_POLICY_PROTOCOL, ContractVersions -from sequifier.rl.checkpoint import load_policy -from sequifier.rl.environment import ( +from sequifier.rl_api import ( EnvironmentCapabilities, + OnlineEnvironmentAdapterBase, OnlineResetResult, OnlineStepResult, + collect_policy_rollout, + load_policy, ) -from sequifier.rl.runtime import collect_policy_rollout from sequifier.trajectory.schema import SeedHistory HORIZON = 12 -class ToyControlAdapter: - protocol = ONLINE_POLICY_PROTOCOL - protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER - implementation_id = f"sequifier.toy-control.v{ContractVersions.TOY_CONTROL}" +class ToyControlAdapter(OnlineEnvironmentAdapterBase): + implementation_id = "sequifier.toy-control" capabilities = EnvironmentCapabilities(seeded_reset=True) def reset(self, seed): @@ -126,7 +124,6 @@ def evaluate(paths): { "environment": { "factory": "iql_toy_control:ToyControlAdapter", - "implementation_id": ToyControlAdapter.implementation_id, "units": {}, }, "max_steps": HORIZON, diff --git a/documentation/example_inputs/policy_control_data.py b/documentation/example_inputs/policy_control_data.py index 71830639..39200565 100644 --- a/documentation/example_inputs/policy_control_data.py +++ b/documentation/example_inputs/policy_control_data.py @@ -10,8 +10,6 @@ import sys from pathlib import Path -from sequifier.contract_versions import ContractVersions - def generate(directory): directory = Path(directory) @@ -81,7 +79,6 @@ def generate(directory): json.dumps( { "format": "sequifier.preference-pairs", - "version": ContractVersions.PREFERENCE_PAIR_MANIFEST, "pairs": pairs, }, indent=2, diff --git a/documentation/policy-learning.md b/documentation/policy-learning.md index a2b49cc0..603c79be 100644 --- a/documentation/policy-learning.md +++ b/documentation/policy-learning.md @@ -44,9 +44,8 @@ See [continuation rules](configs/train.md#optimization-across-phases) for Sequence parts accept CSV/Parquet tables, JSON trajectories, checksummed sequence manifests and prepared histories. An authored manifest that lists trajectory -files declares `format: sequifier.trajectory-source`, the -`ContractVersions.TRAJECTORY_SOURCE_MANIFEST` version, and an `episodes` list -with a `path` per entry. At row `t`, observations and lagged actions form +files declares `format: sequifier.trajectory-source` and an `episodes` +list with a `path` per entry. At row `t`, observations and lagged actions form inputs; the recorded current action is the label, and reward/successor describe its resulting transition. Tables use `sequenceId` and `itemPosition`. Observations and static inputs use the same names in the table, metadata and model. @@ -91,8 +90,8 @@ critic diagnostics are separate. Critics are fresh per phase and are not exporte ## DPO preferences -A preference manifest declares `format: sequifier.preference-pairs`, `version: 126` (`ContractVersions.PREFERENCE_PAIR_MANIFEST`) -and `pairs`. Each pair has `pair_id`, `context_id`, optional `group_id`, and +A preference manifest declares `format: sequifier.preference-pairs` and +`pairs`. Each pair has `pair_id`, `context_id`, optional `group_id`, and `chosen`/`rejected` spans `{source, episode_id, start, end}`. Bounds are zero-based, start-inclusive/end-exclusive within loaded episode rows. Initial executable histories must match. Duplicate IDs, overlapping/identical alternatives and @@ -113,13 +112,12 @@ a still-present changed reference is rejected. ## Online adapters PPO, GRPO, evaluation and policy rollout use -`environment: {factory: module:factory, implementation_id: ..., config: {}, units: {}}`. -Adapters declare `protocol = ONLINE_POLICY_PROTOCOL` and -`protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER`, both imported from -`sequifier.contract_versions`. -Factories receive `config` as keyword arguments. Implementation identity and units -must match the declaration. `OnlineEnvironmentAdapter`, `EnvironmentCapabilities`, -`OnlineResetResult` and `OnlineStepResult` are exported by `sequifier.rl_api`. +`environment: {factory: module:factory, config: {}, units: {}}`. +Factories receive `config` as keyword arguments. Adapter-reported units must match +the configuration. Custom adapters subclass `OnlineEnvironmentAdapterBase` from +`sequifier.rl_api` and supply a nonempty `implementation_id` on the adapter. +Sequifier checks that identity when resuming a run. The same module exports +`EnvironmentCapabilities`, `OnlineResetResult`, and `OnlineStepResult`. Adapters implement `reset(seed)`, `step(named_domain_actions)` and `close()`; policy rollout also requires `reset_from_observation(initial, seed)`. diff --git a/documentation/rollout.md b/documentation/rollout.md index 4b673f7b..aadf4834 100644 --- a/documentation/rollout.md +++ b/documentation/rollout.md @@ -26,10 +26,10 @@ one prepared window in dataset/chunk order, following normal stride/window rules before first-subsequence filtering. Each temporal input needs exactly one producer; static inputs cannot have bindings. Extra/missing environment fields fail validation. -Forecast adapters declare `protocol = FORECAST_ROLLOUT_PROTOCOL` and -`protocol_version = ContractVersions.FORECAST_ROLLOUT_ADAPTER`. Import both from -`sequifier.contract_versions`. They implement `reset(ResetRequest)`, -`step(StepRequest) -> StepResult`, and `close()`, exported by `sequifier.rollout_api`. +Forecast adapters subclass `ForecastRolloutAdapterBase` from +`sequifier.rollout_api`. They implement `reset(ResetRequest)`, +`step(StepRequest) -> StepResult`, and `close()`. The same module exports the +request and result types. Factories receive `environment.config` as keyword arguments. Reset initializes from the supplied history without executing a transition; step receives copied current observations, forecasts and static values. It returns external inputs, a finite reward @@ -43,7 +43,7 @@ together. Unsupported shapes, overflow and non-finite transformed inputs fail. ## Policy rollout Policy rollout uses the [online policy adapter](policy-learning.md#online-adapters), -matching `implementation_id`/`units` and seeded `reset_from_observation`. It selects +matching `units` and seeded `reset_from_observation`. It selects raw CSV/Parquet rows with `seed_row_index`, rejects forecast bindings, and executes named policy actions. A seed supplies exactly one current observation; model history, codecs and normalization come from its artifact. See diff --git a/documentation/supervised-policy-learning.md b/documentation/supervised-policy-learning.md index 711c8b43..c85ab916 100644 --- a/documentation/supervised-policy-learning.md +++ b/documentation/supervised-policy-learning.md @@ -88,5 +88,5 @@ and `GaussianNLLLoss` opt into the shared distribution implementation; learned s belong to the model artifact. `task: forecast` selects point prediction from a policy-bound artifact; `task: policy` selects policy execution. See [inference routing](configs/infer.md#policy-model-interfaces). -Model artifacts, joint trajectories, and PPO collection records each use their -current version from `ContractVersions`. ONNX remains a prediction export. +Sequifier handles format validation when it loads model artifacts, trajectories, +and PPO collection records. ONNX remains a prediction export. diff --git a/documentation/training/artifacts.md b/documentation/training/artifacts.md index 46e664cc..809cc75e 100644 --- a/documentation/training/artifacts.md +++ b/documentation/training/artifacts.md @@ -1,8 +1,8 @@ # Training artifacts Supervised learning, policy imitation, IQL, PPO, DPO, GRPO and mixed plans use the -same public output, checkpoint, report and scalar metric contracts. Each reader requires the current version for its contract. See -[ContractVersions](../../src/sequifier/contract_versions.py) for the current versions. +same output, checkpoint, report and scalar metric formats. Sequifier checks +generated artifact formats when it loads them. ## Configuration and layout @@ -35,7 +35,7 @@ publication, including generated phase destinations and symlink escapes. ## Portable models and recovery -Portable models use `artifact_type: sequifier_model`, `format_version: 105`. +Portable models use the `sequifier_model` artifact type. Their execution contract embeds interfaces, codecs, normalization, policy bindings and units. Policy execution reads units from the selected interface; provenance is not a second source of execution configuration. @@ -44,7 +44,6 @@ All newly written run checkpoints have the same envelope: ```text artifact_type: sequifier_run_checkpoint -format_version: 100 model: recovery_type: supervised | policy | learning recovery: @@ -61,14 +60,13 @@ checkpoint contains in-phase progress. ## Reports and discovery -All final reports use `artifact_type: sequifier_run_manifest`, -`format_version: 108`, at the configured report path and at `manifest.json` beside +Sequifier writes a run manifest at the configured report path and at +`manifest.json` beside the rolling checkpoint. Common fields identify the logical run, current session, status, completion reason, final model, checkpoint, exports and completed phases. Additional fields contain runner-specific provenance. -Each phase report uses `artifact_type: sequifier_phase_report`, -`format_version: 107`. Its common fields include phase index, name, algorithm, +Each phase report includes phase index, name, algorithm, model and checkpoint paths, monitor/direction, handoff, selected metric and transaction, selected weight hash, clocks, metrics, provenance and stop scope. Unavailable measurements are null. Algorithm-specific metrics keep their names @@ -79,7 +77,7 @@ and units; a reward monitor is not represented as a loss. Every native runner configures the model-scoped logger under `logs/`. Structured scalar observations use the supervised writer's `*-training.csv`, `*-training-full.csv`, `*-validation.csv` and `*-validation-full.csv` tables. -All tables carry `schema_version: 109`. Full tables include phase, algorithm and +Full tables include phase, algorithm and iteration identity, run/session IDs, UTC timestamps and progress. Named scalar observations use `metric`, `target`, `value` and `is_primary`; policy observations use `target: __total__`. Nested diagnostic records remain in phase reports. diff --git a/documentation/training/runtime-architecture.md b/documentation/training/runtime-architecture.md index 7e7a5a0a..a2c7eb01 100644 --- a/documentation/training/runtime-architecture.md +++ b/documentation/training/runtime-architecture.md @@ -58,10 +58,9 @@ precedes deep feature arithmetic in eager, compiled, and exported forwards. Unique layers in all temporal and depth stacks are compiled once with their shared aliases restored. Warm-up preserves the run RNG, including on failure. -Portable model artifacts, tensor payloads and stored-window layouts each use -their own current version from `ContractVersions`. -Run checkpoints record initialization-policy and seed-derivation versions in -addition to resolved branch policies and the existing optimizer/scaler/RNG state. +Sequifier validates portable model artifacts, tensor payloads, and stored-window +layouts independently. Run checkpoints retain the initialization contract, resolved +branch policies, and optimizer, scaler, and RNG state. Resume comparisons use the current layout and payload fields. ## Export preflight and publication diff --git a/release-notes/unreleased-contract-versions.md b/release-notes/unreleased-contract-versions.md index 143e0f0a..d01f722b 100644 --- a/release-notes/unreleased-contract-versions.md +++ b/release-notes/unreleased-contract-versions.md @@ -1,9 +1,7 @@ # Consolidated Sequifier contract versions -[ContractVersions](../src/sequifier/contract_versions.py) defines current -versions for Sequifier artifacts, metadata, adapter protocols, and implementation -identifiers. Checkpoint and collection archive envelopes have their own IDs, as -do the payloads they contain. +Sequifier manages compatibility for generated artifacts, metadata, and adapter +protocols internally. Rollout configs select the adapter by factory. - Re-run preprocessing from raw data to rebuild dataset metadata, PT batches, stored-window metadata, preprocessing manifests, depth ledgers, and video @@ -13,19 +11,10 @@ do the payloads they contain. load from that release. Produce fresh portable models, backbone repository revisions, reports, metrics, and ONNX exports for the current release. - Re-export sequence data and regenerate trajectories and collection archives. - Update authored preference-pair and trajectory-source manifests to declare - the current version and point to current source artifacts. -- Forecast adapters declare `protocol = FORECAST_ROLLOUT_PROTOCOL` and - `protocol_version = ContractVersions.FORECAST_ROLLOUT_ADAPTER`. Online policy - adapters declare `protocol = ONLINE_POLICY_PROTOCOL` and - `protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER`. Use the current - implementation IDs in configurations that select the built-in or toy control. + Authored preference-pair and trajectory-source manifests use a format tag. +- Custom adapters can inherit protocol metadata from + `ForecastRolloutAdapterBase` or `OnlineEnvironmentAdapterBase`. - Initialization and policy collection use new seed namespaces. The same inputs are deterministic within the new version and can produce different seeds from earlier releases. Versioned identities change fingerprints and export - preflight cache keys. The ONNX operator set remains 18. - -Each contract has its own version sequence. Compare versions only after -identifying the contract. Increase the version for a changed contract and bump -an envelope only when its own contract changes. The Contract versions check -validates version changes and shipped examples against the merge target. + preflight cache keys. diff --git a/src/sequifier/artifacts/loading.py b/src/sequifier/artifacts/loading.py index 60ae665e..3ef7dcfc 100644 --- a/src/sequifier/artifacts/loading.py +++ b/src/sequifier/artifacts/loading.py @@ -67,7 +67,6 @@ def load_model_for_analysis( artifact_metadata={ "path": str(artifact_path), "artifact_type": artifact_type, - "format_version": artifact.format_version, "metadata": artifact.metadata, "selected_interface": interface_name, }, diff --git a/src/sequifier/config/metadata.py b/src/sequifier/config/metadata.py index a0a5d0e2..e81d7d87 100644 --- a/src/sequifier/config/metadata.py +++ b/src/sequifier/config/metadata.py @@ -257,12 +257,23 @@ def load_dataset_metadata(path: str) -> DatasetMetadata: return DatasetMetadata.model_validate(values) +def validate_authored_metadata(values: dict) -> None: + layout = values.get("storage_layout") + if ( + "tensor_payload_version" in values + or "stored_window_layout_version" in values + or (isinstance(layout, dict) and "version" in layout) + ): + raise ValueError("Inline metadata cannot declare internal storage versions") + + @beartype def extract_inline_metadata( values: dict[str, Any], ) -> tuple[dict[str, Any], dict[str, Any] | None]: """Separate inline metadata facts from the authored request.""" + validate_authored_metadata(values) authored = copy.deepcopy(values) window_view = authored.get("window_view") if "context_length" not in authored and isinstance(window_view, dict): @@ -274,7 +285,6 @@ def extract_inline_metadata( elif "target_offset" not in authored and isinstance(window_view, ModelWindowView): authored["target_offset"] = window_view.target_offset - tensor_kind = dataset_tensor_kind(authored.get("depth_layouts")) metadata_values: dict[str, Any] = { "split_paths": [ path @@ -285,9 +295,10 @@ def extract_inline_metadata( if path is not None ], "depth_layouts": authored.get("depth_layouts", {}), - "tensor_payload_kind": authored.get("tensor_payload_kind", tensor_kind), - "tensor_payload_version": authored.get( - "tensor_payload_version", dataset_tensor_version(tensor_kind) + **( + {"tensor_payload_kind": authored["tensor_payload_kind"]} + if "tensor_payload_kind" in authored + else {} ), "column_data_types": authored.get("column_data_types", {}), "static_columns": authored.get("static_columns", []), @@ -318,7 +329,6 @@ def extract_inline_metadata( { "window_length": storage_layout.window_length, "max_target_offset": storage_layout.max_target_offset, - "stored_window_layout_version": storage_layout.version, } ) elif isinstance(storage_layout, dict): @@ -326,9 +336,6 @@ def extract_inline_metadata( { "window_length": storage_layout.get("window_length"), "max_target_offset": storage_layout.get("max_target_offset", 1), - "stored_window_layout_version": storage_layout.get( - "version", ContractVersions.STORED_WINDOW_LAYOUT - ), } ) else: @@ -336,10 +343,6 @@ def extract_inline_metadata( { "window_length": authored.get("window_length"), "max_target_offset": authored.get("max_target_offset", 1), - "stored_window_layout_version": authored.get( - "stored_window_layout_version", - ContractVersions.STORED_WINDOW_LAYOUT, - ), } ) diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index eb56463f..bdc68727 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -24,7 +24,6 @@ ) from sequifier.config.policy_optimization import OptimizationConfig from sequifier.config.sources import TrainingSourceSpecModel -from sequifier.contract_versions import ContractVersions class OfflineEvaluationMonitor(ConfigModel): @@ -158,11 +157,6 @@ class OnlineEnvironmentConfig(ConfigModel): factory: str config: dict = Field(default_factory=dict) - @property - def protocol_version(self): - return ContractVersions.ONLINE_POLICY_ADAPTER - - implementation_id: str = Field(min_length=1) # Empty means unitless; adapters must report this exact declaration. units: dict[str, str] = Field(default_factory=dict) diff --git a/src/sequifier/config/rollout_config.py b/src/sequifier/config/rollout_config.py index 0603ea55..5f658094 100644 --- a/src/sequifier/config/rollout_config.py +++ b/src/sequifier/config/rollout_config.py @@ -5,7 +5,6 @@ from pydantic import ConfigDict, Field, field_validator, model_validator from sequifier.config.naming import ConfigModel -from sequifier.contract_versions import ContractVersions class InputBinding(ConfigModel): @@ -22,10 +21,6 @@ class AdapterConfig(ConfigModel): factory: str config: dict[str, Any] = Field(default_factory=dict) - @property - def protocol_version(self): - return ContractVersions.FORECAST_ROLLOUT_ADAPTER - @field_validator("factory") @classmethod def validate_factory(cls, value: str) -> str: diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index eeb06d98..629d467b 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -47,9 +47,8 @@ from sequifier.config.metadata import ( DatasetMetadata, TensorPayloadKind, - dataset_tensor_kind, - dataset_tensor_version, load_dataset_metadata, + validate_authored_metadata, ) from sequifier.config.naming import ConfigModel from sequifier.config.normalization_config import FrozenNormalizationConfig @@ -76,7 +75,6 @@ from sequifier.config.sources import validate_source as _validate_source # Reusable schemas are independent of the canonical root configuration. -from sequifier.contract_versions import ContractVersions from sequifier.helpers import ( ModelWindowView, StoredWindowLayout, @@ -1722,7 +1720,6 @@ def resolve_sequifier_config( "image_sources", "preconditioning_image", "tensor_payload_kind", - "tensor_payload_version", "metadata_by_part", "column_data_types", "static_columns", @@ -1735,7 +1732,6 @@ def resolve_sequifier_config( "window_length", "window_lengths", "max_target_offset", - "stored_window_layout_version", "storage_layout", "split_paths", "prediction_aligned_splits", @@ -1749,32 +1745,28 @@ def resolve_sequifier_config( def _inline_metadata( values: dict[str, Any], global_spec: TrainingRuntimeSettings ) -> DatasetMetadata: + validate_authored_metadata(values) layout = values.get("storage_layout") if isinstance(layout, StoredWindowLayout): window_length = layout.window_length max_target_offset = layout.max_target_offset - layout_version = layout.version elif isinstance(layout, dict): window_length = layout.get("window_length") max_target_offset = layout.get("max_target_offset", 1) - layout_version = layout.get("version", ContractVersions.STORED_WINDOW_LAYOUT) else: window_length = values.get("window_length") max_target_offset = values.get("max_target_offset", 1) - layout_version = values.get( - "stored_window_layout_version", ContractVersions.STORED_WINDOW_LAYOUT - ) if window_length is None: if global_spec.context_length is None: raise ValueError("inline metadata requires context_length or window_length") window_length = global_spec.context_length + max(1, global_spec.target_offset) - tensor_kind = dataset_tensor_kind(values.get("depth_layouts")) return DatasetMetadata.model_validate( { "depth_layouts": values.get("depth_layouts", {}), - "tensor_payload_kind": values.get("tensor_payload_kind", tensor_kind), - "tensor_payload_version": values.get( - "tensor_payload_version", dataset_tensor_version(tensor_kind) + **( + {"tensor_payload_kind": values["tensor_payload_kind"]} + if "tensor_payload_kind" in values + else {} ), "split_paths": values.get("split_paths", []), "column_data_types": values.get("column_data_types", {}), @@ -1798,7 +1790,6 @@ def _inline_metadata( "target_offset": values.get("target_offset"), "window_length": window_length, "max_target_offset": max_target_offset, - "stored_window_layout_version": layout_version, } ) diff --git a/src/sequifier/contract_versions.py b/src/sequifier/contract_versions.py index 25aa4686..c377d51a 100644 --- a/src/sequifier/contract_versions.py +++ b/src/sequifier/contract_versions.py @@ -4,58 +4,55 @@ class ContractVersions: # Checkpoint envelope and runner-owned payloads. RUN_CHECKPOINT = 100 - SUPERVISED_RECOVERY = 140 - POLICY_RECOVERY = 141 - POLICY_PHASE_STATE = 103 - LEARNING_RUN_STATE = 142 + SUPERVISED_RECOVERY = 100 + POLICY_RECOVERY = 100 + POLICY_PHASE_STATE = 100 + LEARNING_RUN_STATE = 100 # Portable artifacts and training telemetry. - MODEL_ARTIFACT = 105 - BACKBONE_MANIFEST = 106 - PHASE_REPORT = 107 - RUN_MANIFEST = 108 - METRICS_SCHEMA = 109 + MODEL_ARTIFACT = 100 + BACKBONE_MANIFEST = 100 + PHASE_REPORT = 100 + RUN_MANIFEST = 100 + METRICS_SCHEMA = 100 - INITIALIZATION_POLICY = 110 - INITIALIZATION_SEED_DERIVATION = 111 + INITIALIZATION_POLICY = 100 + INITIALIZATION_SEED_DERIVATION = 100 # Dataset storage and preprocessing resume records. - PT_TENSOR_BATCH = 112 - DATASET_TENSOR_METADATA = 114 - DEPTH_TENSOR_METADATA = 115 - STORED_WINDOW_LAYOUT = 115 - PREPROCESSING_MANIFEST = 116 - VIDEO_FRAME_INDEX = 117 - SERIAL_DEPTH_LEDGER = 119 - PARALLEL_DEPTH_LEDGER = 120 + PT_TENSOR_BATCH = 100 + DATASET_TENSOR_METADATA = 100 + DEPTH_TENSOR_METADATA = 100 + STORED_WINDOW_LAYOUT = 100 + PREPROCESSING_MANIFEST = 100 + VIDEO_FRAME_INDEX = 100 + SERIAL_DEPTH_LEDGER = 100 + PARALLEL_DEPTH_LEDGER = 100 # Rollout records, algorithm archives, and offline source manifests. - FORECAST_TRAJECTORY = 120 - JOINT_POLICY_TRAJECTORY = 121 - COLLECTION_ARCHIVE = 122 - PPO_COLLECTION = 123 - GRPO_ARCHIVE = 124 - SEQUENCE_DATA_MANIFEST = 145 - PREFERENCE_PAIR_MANIFEST = 126 - SCALAR_POLICY_TRAJECTORY = 138 - TRAJECTORY_SOURCE_MANIFEST = 140 # Authored manifest of episode files. + FORECAST_TRAJECTORY = 100 + JOINT_POLICY_TRAJECTORY = 100 + COLLECTION_ARCHIVE = 100 + PPO_COLLECTION = 100 + GRPO_ARCHIVE = 100 + SEQUENCE_DATA_MANIFEST = 100 + SCALAR_POLICY_TRAJECTORY = 100 - FORECAST_ROLLOUT_ADAPTER = 128 - ONLINE_POLICY_ADAPTER = 129 - MODEL_EXECUTION_SCHEMA = 129 - EXPORT_PREFLIGHT = 130 + FORECAST_ROLLOUT_ADAPTER = 100 + ONLINE_POLICY_ADAPTER = 100 + MODEL_EXECUTION_SCHEMA = 100 + EXPORT_PREFLIGHT = 100 # Algorithm and implementation identities used in fingerprints and seeds. - HASH_CODEC = 131 - RESERVED_CODE_SPECIAL_TOKENS = 132 - INJECTIVE_SPECIAL_TOKENS = 133 - POLICY_COLLECTION_SEED = 134 - EXPORT_ATTENTION_LOWERING = 135 - CATEGORICAL_CONTROL = 136 - TOY_CONTROL = 137 + HASH_CODEC = 100 + RESERVED_CODE_SPECIAL_TOKENS = 100 + INJECTIVE_SPECIAL_TOKENS = 100 + POLICY_COLLECTION_SEED = 100 + EXPORT_ATTENTION_LOWERING = 100 + CATEGORICAL_CONTROL = 100 - PHASE_COMMIT = 143 - PHASE_BOUNDARY = 144 + PHASE_COMMIT = 100 + PHASE_BOUNDARY = 100 # External ONNX operator set. ONNX_OPSET = 18 @@ -67,6 +64,4 @@ class ContractVersions: def require_contract_version(contract: str, actual, expected: int) -> None: if not isinstance(actual, int) or isinstance(actual, bool) or actual != expected: - raise ValueError( - f"Unsupported {contract} version {actual!r}, expected {expected}" - ) + raise ValueError(f"Unsupported {contract} version") diff --git a/src/sequifier/io/pt_payload.py b/src/sequifier/io/pt_payload.py index 69bd18e6..cd8658ee 100644 --- a/src/sequifier/io/pt_payload.py +++ b/src/sequifier/io/pt_payload.py @@ -251,7 +251,7 @@ def load_pt_payload(path, *, layouts=None, n_classes=None) -> StoredTensorBatch: or not isinstance(metadata, dict) or set(metadata) != expected_metadata ): - raise ValueError(f"Inconsistent version {version} tensor payload schema") + raise ValueError("Inconsistent tensor payload schema") batch = StoredTensorBatch( **{key: value[key] for key in expected - {"format", "version", "metadata"}}, depth_valid_masks=metadata["depth_valid_masks"], diff --git a/src/sequifier/rl/control.py b/src/sequifier/rl/control.py index 0f85501c..1494cd96 100644 --- a/src/sequifier/rl/control.py +++ b/src/sequifier/rl/control.py @@ -7,17 +7,16 @@ import random -from sequifier.contract_versions import ONLINE_POLICY_PROTOCOL, ContractVersions +from sequifier.contract_versions import ContractVersions from sequifier.rl.environment import ( EnvironmentCapabilities, + OnlineEnvironmentAdapterBase, OnlineResetResult, OnlineStepResult, ) -class CategoricalControlAdapter: - protocol = ONLINE_POLICY_PROTOCOL - protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER +class CategoricalControlAdapter(OnlineEnvironmentAdapterBase): implementation_id = ( f"sequifier.categorical-control.v{ContractVersions.CATEGORICAL_CONTROL}" ) diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py index df1698bb..eb134613 100644 --- a/src/sequifier/rl/environment.py +++ b/src/sequifier/rl/environment.py @@ -12,6 +12,7 @@ ContractVersions, require_contract_version, ) +from sequifier.fingerprints import fingerprint from sequifier.io.values import ( DomainValue, numeric_scalar, @@ -70,6 +71,13 @@ def __post_init__(self): ) +class OnlineEnvironmentAdapterBase: + """Supply the protocol metadata for an online policy adapter.""" + + protocol = ONLINE_POLICY_PROTOCOL + protocol_version = ContractVersions.ONLINE_POLICY_ADAPTER + + class OnlineEnvironmentAdapter(Protocol): protocol: str protocol_version: int @@ -92,7 +100,7 @@ def load_online_adapter(spec): module, attribute = spec.factory.split(":") adapter = getattr(import_module(module), attribute)(**spec.config) try: - validate_online_adapter(adapter, spec) + validate_online_adapter(adapter) return adapter except BaseException: close = getattr(adapter, "close", None) @@ -101,7 +109,7 @@ def load_online_adapter(spec): raise -def validate_online_adapter(adapter, spec): +def validate_online_adapter(adapter): if getattr(adapter, "protocol", None) != ONLINE_POLICY_PROTOCOL: raise ValueError(f"adapter.protocol must be {ONLINE_POLICY_PROTOCOL!r}") require_contract_version( @@ -109,10 +117,9 @@ def validate_online_adapter(adapter, spec): getattr(adapter, "protocol_version", None), ContractVersions.ONLINE_POLICY_ADAPTER, ) - if getattr(adapter, "implementation_id", None) != spec.implementation_id: - raise ValueError( - "environment.implementation_id differs from adapter implementation" - ) + implementation_id = getattr(adapter, "implementation_id", None) + if not isinstance(implementation_id, str) or not implementation_id.strip(): + raise ValueError("online adapter requires a nonempty implementation_id") if not isinstance(getattr(adapter, "capabilities", None), EnvironmentCapabilities): raise ValueError("online adapter requires explicit EnvironmentCapabilities") for method in ("reset", "step", "close"): @@ -233,7 +240,8 @@ def adapter_identity(adapter): "module": type(adapter).__module__, "class": type(adapter).__qualname__, "protocol": adapter.protocol, - "protocol_version": adapter.protocol_version, - "implementation_id": adapter.implementation_id, + "implementation_fingerprint": fingerprint( + [adapter.protocol_version, adapter.implementation_id] + ), "capabilities": asdict(adapter.capabilities), } diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index 1da5b104..4c476e57 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -6,7 +6,6 @@ from sequifier.config.policy_infer_config import PolicyRolloutConfig from sequifier.config.rollout_config import InputBinding -from sequifier.contract_versions import ContractVersions from sequifier.io.actions import action_component_key, action_latent_key from sequifier.io.values import numeric_scalar, portable_fields from sequifier.rl.collection import PolicyCursor @@ -36,7 +35,7 @@ def collect_policy_rollout( """Execute a supplied initial observation, preserving actual episode boundaries.""" was_training, failed = policy.training, False try: - validate_online_adapter(adapter, config.environment) + validate_online_adapter(adapter) require_seeded_reset(adapter) if (not isinstance(max_steps, int)) or max_steps < 1: raise ValueError("policy collection budget must be a positive integer") @@ -135,7 +134,6 @@ def collect_policy_rollout( observation = successor return recorder.finish( format="sequifier.policy-trajectory", - version=ContractVersions.JOINT_POLICY_TRAJECTORY, seed=seed, bindings=[ InputBinding(input=name, source="environment", key=name) diff --git a/src/sequifier/rl_api.py b/src/sequifier/rl_api.py index 4987c824..6cf4b8f9 100644 --- a/src/sequifier/rl_api.py +++ b/src/sequifier/rl_api.py @@ -47,6 +47,7 @@ from sequifier.rl.environment import ( EnvironmentCapabilities, OnlineEnvironmentAdapter, + OnlineEnvironmentAdapterBase, OnlineResetResult, OnlineStepResult, ) @@ -87,6 +88,7 @@ "GRPOEvaluationConfig", "IQLPhaseConfig", "OnlineEnvironmentAdapter", + "OnlineEnvironmentAdapterBase", "OnlineEnvironmentConfig", "OnlineEvaluationConfig", "OnlineResetResult", diff --git a/src/sequifier/rollout/contracts.py b/src/sequifier/rollout/contracts.py index f4d8e077..90ed4e62 100644 --- a/src/sequifier/rollout/contracts.py +++ b/src/sequifier/rollout/contracts.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import Mapping, Protocol +from sequifier.contract_versions import FORECAST_ROLLOUT_PROTOCOL, ContractVersions from sequifier.io.values import DomainValue @@ -34,6 +35,13 @@ class StepResult: truncated: bool = False +class ForecastRolloutAdapterBase: + """Supply the protocol metadata for a numeric forecast adapter.""" + + protocol = FORECAST_ROLLOUT_PROTOCOL + protocol_version = ContractVersions.FORECAST_ROLLOUT_ADAPTER + + class EnvironmentAdapter(Protocol): protocol: str protocol_version: int diff --git a/src/sequifier/rollout/coordinator.py b/src/sequifier/rollout/coordinator.py index 817c6250..2c5ba3ad 100644 --- a/src/sequifier/rollout/coordinator.py +++ b/src/sequifier/rollout/coordinator.py @@ -5,7 +5,6 @@ from uuid import uuid4 from sequifier.config.rollout_config import RolloutConfig -from sequifier.contract_versions import ContractVersions from sequifier.io.values import DomainValue from sequifier.rollout.contracts import ( EnvironmentAdapter, @@ -137,7 +136,6 @@ def collect_rollout( break return recorder.finish( format="sequifier.numeric-forecast-trajectory", - version=ContractVersions.FORECAST_TRAJECTORY, seed=initial_history, bindings=config.bindings, prediction_columns=prediction_columns, diff --git a/src/sequifier/rollout_api.py b/src/sequifier/rollout_api.py index 0c2ec045..1e1798b9 100644 --- a/src/sequifier/rollout_api.py +++ b/src/sequifier/rollout_api.py @@ -3,6 +3,7 @@ from sequifier.config.rollout_config import AdapterConfig, InputBinding, RolloutConfig from sequifier.rollout.contracts import ( EnvironmentAdapter, + ForecastRolloutAdapterBase, ResetRequest, StepRequest, StepResult, @@ -21,6 +22,7 @@ "InputBinding", "RolloutConfig", "EnvironmentAdapter", + "ForecastRolloutAdapterBase", "ResetRequest", "StepRequest", "StepResult", diff --git a/src/sequifier/templates/train-grpo.yaml b/src/sequifier/templates/train-grpo.yaml index e5772faf..dc3d5151 100644 --- a/src/sequifier/templates/train-grpo.yaml +++ b/src/sequifier/templates/train-grpo.yaml @@ -46,13 +46,11 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 evaluation: environment: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 seeds: - 17 - 29 diff --git a/src/sequifier/templates/train-ppo.yaml b/src/sequifier/templates/train-ppo.yaml index d95f8c3a..5116e405 100644 --- a/src/sequifier/templates/train-ppo.yaml +++ b/src/sequifier/templates/train-ppo.yaml @@ -42,7 +42,6 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 total_env_steps: 8192 rollout_env_steps: 256 optimization_epochs: 4 @@ -55,7 +54,6 @@ training_plan: factory: sequifier.rl.control:CategoricalControlAdapter config: horizon: 8 - implementation_id: sequifier.categorical-control.v136 seeds: - 17 - 29 diff --git a/src/sequifier/training/metrics.py b/src/sequifier/training/metrics.py index a67aa257..5641b43f 100644 --- a/src/sequifier/training/metrics.py +++ b/src/sequifier/training/metrics.py @@ -126,18 +126,14 @@ def _format_number(number: float) -> str: def validate_metrics_csv_header(fields, path): if fields is None or "schema_version" not in fields: raise ValueError( - f"Unsupported metrics schema in {path}: missing schema_version, " - f"expected {ContractVersions.METRICS_SCHEMA}" + f"Unsupported metrics schema in {path}: missing schema_version" ) def validate_metrics_csv_row(row, path): value = row.get("schema_version") if value != str(ContractVersions.METRICS_SCHEMA): - raise ValueError( - f"Unsupported metrics schema version {value!r} in {path}; " - f"expected {ContractVersions.METRICS_SCHEMA}" - ) + raise ValueError(f"Unsupported metrics schema in {path}") class _CsvAppender: diff --git a/src/sequifier/trajectory/preferences.py b/src/sequifier/trajectory/preferences.py index d30b84f7..0490284d 100644 --- a/src/sequifier/trajectory/preferences.py +++ b/src/sequifier/trajectory/preferences.py @@ -1,4 +1,4 @@ -"""Versioned reward-free preference manifests over contiguous decision spans.""" +"""Authored reward-free preference manifests over contiguous decision spans.""" import json from pathlib import Path @@ -6,8 +6,6 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator -from sequifier.contract_versions import ContractVersions, require_contract_version - class PreferenceSpan(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) @@ -37,23 +35,8 @@ class PreferencePair(BaseModel): class PreferenceManifest(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) format: Literal["sequifier.preference-pairs"] - version: int = Field( - strict=True, - json_schema_extra={"const": ContractVersions.PREFERENCE_PAIR_MANIFEST}, - ) pairs: list[PreferencePair] = Field(min_length=1) - @model_validator(mode="before") - @classmethod - def validate_version(cls, values): - if isinstance(values, dict): - require_contract_version( - "preference-pair manifest", - values.get("version"), - ContractVersions.PREFERENCE_PAIR_MANIFEST, - ) - return values - @model_validator(mode="after") def unique_ids(self): ids = [pair.pair_id for pair in self.pairs] diff --git a/src/sequifier/trajectory/schema.py b/src/sequifier/trajectory/schema.py index cb64c452..75515a84 100644 --- a/src/sequifier/trajectory/schema.py +++ b/src/sequifier/trajectory/schema.py @@ -16,7 +16,6 @@ ) from sequifier.config.rollout_config import InputBinding -from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.io.values import DomainValue from sequifier.trajectory.returns import remaining_return @@ -65,20 +64,12 @@ class Transition(Record): bootstrap_allowed: bool -_TRAJECTORY_VERSIONS = { - "sequifier.numeric-forecast-trajectory": ContractVersions.FORECAST_TRAJECTORY, - "sequifier.scalar-policy-trajectory": ContractVersions.SCALAR_POLICY_TRAJECTORY, - "sequifier.policy-trajectory": ContractVersions.JOINT_POLICY_TRAJECTORY, -} - - class Trajectory(Record): format: Literal[ "sequifier.numeric-forecast-trajectory", "sequifier.scalar-policy-trajectory", "sequifier.policy-trajectory", ] - version: int = Field(strict=True) episode_id: str = Field(min_length=1) seed: SeedHistory bindings: list[InputBinding] @@ -88,17 +79,6 @@ class Trajectory(Record): total_return: FiniteFloat metadata: dict[str, Any] = Field(default_factory=dict) - @model_validator(mode="before") - @classmethod - def validate_version(cls, values): - if isinstance(values, dict): - kind = values.get("format") - if isinstance(kind, str) and kind in _TRAJECTORY_VERSIONS: - require_contract_version( - kind, values.get("version"), _TRAJECTORY_VERSIONS[kind] - ) - return values - @model_validator(mode="after") def validate_temporal_contract(self): # Metadata must round-trip through the same portable JSON representation. diff --git a/src/sequifier/trajectory/sources.py b/src/sequifier/trajectory/sources.py index 0f6cdd7b..7a3da2e0 100644 --- a/src/sequifier/trajectory/sources.py +++ b/src/sequifier/trajectory/sources.py @@ -46,14 +46,20 @@ def _source_files(source, checksums): raise ValueError("Source manifest must be a mapping") manifest_format = manifest.get("format") if manifest_format == "sequifier.sequence-data": - version = ContractVersions.SEQUENCE_DATA_MANIFEST + require_contract_version( + manifest_format, + manifest.get("version"), + ContractVersions.SEQUENCE_DATA_MANIFEST, + ) entries_field = "shards" elif manifest_format == "sequifier.trajectory-source": - version = ContractVersions.TRAJECTORY_SOURCE_MANIFEST + if "version" in manifest: + raise ValueError( + "Authored trajectory-source manifests must omit version" + ) entries_field = "episodes" else: raise ValueError(f"Unsupported source manifest format {manifest_format!r}") - require_contract_version(manifest_format, manifest.get("version"), version) entries = manifest.get(entries_field) if not isinstance(entries, list) or not entries: raise ValueError( diff --git a/src/sequifier/trajectory/store.py b/src/sequifier/trajectory/store.py index d13dc6e1..1ff0423f 100644 --- a/src/sequifier/trajectory/store.py +++ b/src/sequifier/trajectory/store.py @@ -3,13 +3,29 @@ import json from pathlib import Path +from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.io.atomic import write_json_atomic from sequifier.trajectory.schema import Trajectory, validate_trajectory +_TRAJECTORY_VERSIONS = { + "sequifier.numeric-forecast-trajectory": ContractVersions.FORECAST_TRAJECTORY, + "sequifier.scalar-policy-trajectory": ContractVersions.SCALAR_POLICY_TRAJECTORY, + "sequifier.policy-trajectory": ContractVersions.JOINT_POLICY_TRAJECTORY, +} + def save_trajectory(trajectory: Trajectory, path: str | Path) -> None: - write_json_atomic(path, validate_trajectory(trajectory).model_dump(mode="json")) + values = validate_trajectory(trajectory).model_dump(mode="json") + values["version"] = _TRAJECTORY_VERSIONS[values["format"]] + write_json_atomic(path, values) def load_trajectory(path: str | Path) -> Trajectory: - return validate_trajectory(json.loads(Path(path).read_text(encoding="utf-8"))) + values = json.loads(Path(path).read_text(encoding="utf-8")) + kind = values.get("format") if isinstance(values, dict) else None + if not isinstance(kind, str) or kind not in _TRAJECTORY_VERSIONS: + raise ValueError("Unsupported trajectory format") + require_contract_version( + kind, values.pop("version", None), _TRAJECTORY_VERSIONS[kind] + ) + return validate_trajectory(values) diff --git a/tools/check_contract_versions.py b/tools/check_contract_versions.py index 1ad0504a..f2cef2a1 100644 --- a/tools/check_contract_versions.py +++ b/tools/check_contract_versions.py @@ -1,10 +1,8 @@ -"""Check contract versions against the merge target.""" +"""Check declared contract versions and shipped examples.""" -import argparse import ast import json import re -import subprocess from pathlib import Path REGISTRY = "src/sequifier/contract_versions.py" @@ -33,17 +31,10 @@ def registry(source): return values -def validate_versions(current, base): - for name, value in current.items(): +def validate_versions(values): + for name, value in values.items(): if not isinstance(value, int) or isinstance(value, bool) or value < 1: raise ValueError(f"{name} must be a positive integer") - previous = base.get(name) - if ( - isinstance(previous, int) - and not isinstance(previous, bool) - and value < previous - ): - raise ValueError(f"{name} cannot decrease from {previous} to {value}") VERSION_FIELDS = { @@ -176,22 +167,19 @@ def validate_boundaries(root, values): for line in hardcoded_versions(tree) ) - identities = { - "sequifier.categorical-control": values["CATEGORICAL_CONTROL"], - "sequifier.toy-control": values["TOY_CONTROL"], - } - for directory in ("src/sequifier/templates", "documentation/example_inputs"): + versioned_identity = re.compile(r"sequifier[.-][\w.-]+\.v[0-9]+\b") + for directory in ( + "src/sequifier/templates", + "documentation/example_inputs/configs", + ): for path in (root / directory).rglob("*"): - if not path.is_file() or path.suffix not in (".yaml", ".yml", ".json"): - continue - content = path.read_text() - for prefix, version in identities.items(): - expected = f"{prefix}.v{version}" - for match in re.finditer(rf"{re.escape(prefix)}\.v[\w.-]+", content): - if match[0] != expected: - errors.append( - f"{path.relative_to(root)}: found {match[0]}, expected {expected}" - ) + if path.suffix in {".yaml", ".yml"} and versioned_identity.search( + path.read_text() + ): + errors.append( + f"{path.relative_to(root)}: use a version-free adapter config" + ) + metadata_path = ( "documentation/example_inputs/configs/ddconfigs/preprocessing_input.json" ) @@ -216,27 +204,9 @@ def validate_boundaries(root, values): def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--base", required=True, help="Target commit included in HEAD") - args = parser.parse_args() root = Path(__file__).resolve().parents[1] - subprocess.run( - ["git", "merge-base", "--is-ancestor", args.base, "HEAD"], cwd=root, check=True - ) - base_paths = subprocess.check_output( - ["git", "ls-tree", "--name-only", args.base, REGISTRY], text=True, cwd=root - ) - base = ( - registry( - subprocess.check_output( - ["git", "show", f"{args.base}:{REGISTRY}"], text=True, cwd=root - ) - ) - if base_paths - else {} - ) current = registry((root / REGISTRY).read_text()) - validate_versions(current, base) + validate_versions(current) validate_boundaries(root, current) From 790625c38435b3e5bc1de28b9299d512e4899cc5 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 13:15:56 +0100 Subject: [PATCH 69/71] clean up bugs --- AGENTS.md | 7 ------- src/sequifier/config/command_ownership.py | 7 ++++++- src/sequifier/config/policy_train_config.py | 11 +++++++++++ src/sequifier/config/training_resolution.py | 5 +---- src/sequifier/io/episode_features.py | 6 +++--- src/sequifier/rl/prepared_inputs.py | 6 +++++- src/sequifier/training/runtime.py | 6 +++++- src/sequifier/trajectory/prepared.py | 10 ++++++++-- 8 files changed, 39 insertions(+), 19 deletions(-) delete mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index a1f342c4..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,7 +0,0 @@ -# Contract versions - -Identify each contract independently of its version number. When a reader handles multiple contracts, select the contract using an explicit discriminator such as a format tag, mode, payload kind, or protocol name. A dedicated reader or field can establish the contract through its schema context. - -Validate the version only after identifying the contract. Different contracts may share the same version number. Version changes and CI checks must be scoped to each contract. - -Contract versions are internal and must never be user-facing. Keep version values out of user-authored configs and manifests, shipped examples, user documentation, and CLI output. This applies to standalone fields and to versions embedded in identifiers or other strings. Sequifier supplies and validates versions internally, including in generated artifact metadata. diff --git a/src/sequifier/config/command_ownership.py b/src/sequifier/config/command_ownership.py index f798d4c2..2df2032f 100644 --- a/src/sequifier/config/command_ownership.py +++ b/src/sequifier/config/command_ownership.py @@ -63,4 +63,9 @@ def apply_run_overrides(values, overrides): run.setdefault("reproducibility", {})["enforce_determinism"] = overrides.pop( "enforce_determinism" ) - return values, {key: value for key, value in overrides.items() if value is not None} + # The CLI uses an explicit None to clear either column selection. + return values, { + key: value + for key, value in overrides.items() + if value is not None or key in {"selected_columns", "input_columns"} + } diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 7b6c91dd..4030fa94 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -24,6 +24,7 @@ def validate_distinct_columns(self): class SequenceSourceConfig(PolicyConfigBase): _project_root: str = PrivateAttr(default=".") + _resolved_split_index: int | None = PrivateAttr(default=None) units: dict[str, str] | None = None paths: list[NonemptyStr] = Field(default_factory=list) manifest_path: NonemptyStr | None = None @@ -43,6 +44,15 @@ def project_root(self) -> str: """Run-owned media root, excluded from the authored and serialized schema.""" return self._project_root + @property + def selected_split_index(self) -> int | None: + """Keep the split selection available after expanding it into paths.""" + return ( + self.split_index + if self.split_index is not None + else self._resolved_split_index + ) + @model_validator(mode="after") def validate_location(self): metadata_split = ( @@ -105,4 +115,5 @@ def resolve_sequence_source( ) resolved = SequenceSourceConfig.model_validate(values) resolved._project_root = str(root) + resolved._resolved_split_index = source.selected_split_index return resolved diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index 5ebcacea..da573ec2 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -122,12 +122,9 @@ def _resolve_phase_sources(config, phase, available): def phase_interface(config, phase): - refs = list(getattr(phase, "sources", [])) - if phase.algorithm in {"policy_imitation", "iql"}: - refs.extend(phase.evaluation.sources or []) names = { config.dataset_training[ref.source.partition(".")[0]].model_interface - for ref in refs + for ref in phase_sources(phase) } explicit = getattr(phase, "model_interface", None) if explicit is not None: diff --git a/src/sequifier/io/episode_features.py b/src/sequifier/io/episode_features.py index 6d4b47a1..c20c1555 100644 --- a/src/sequifier/io/episode_features.py +++ b/src/sequifier/io/episode_features.py @@ -102,14 +102,14 @@ def validate_features(self) -> None: self._features_validated = True def validate_normalization(self, statistics): - """Check FP32 transforms episode-by-episode without constructing windows.""" + """Check FP32 transforms for decisions in the selected episode prefixes.""" if self._validated_statistics == statistics: return from sequifier.io.policy_window import PolicyWindowView view = PolicyWindowView(self.config) - for episode in self.episodes: - for step in range(len(episode.actions)): + for episode, length in zip(self.episodes, self.lengths): + for step in range(length): view.episode_inputs(episode, step, None) self._validated_statistics = {k: dict(v) for k, v in statistics.items()} return diff --git a/src/sequifier/rl/prepared_inputs.py b/src/sequifier/rl/prepared_inputs.py index e18eedc2..93d3ec8d 100644 --- a/src/sequifier/rl/prepared_inputs.py +++ b/src/sequifier/rl/prepared_inputs.py @@ -193,7 +193,10 @@ def phase_checksums(self, config, phase, inputs): def resolve(self, config): declarations = { - name: source.model_dump(mode="json") + name: { + **source.model_dump(mode="json"), + "split_index": source.selected_split_index, + } for name, source in config.sources.items() } key = fingerprint({"sources": declarations, "preferences": config.preferences}) @@ -247,6 +250,7 @@ def _prepare_phase(self, config, phase, policy, inputs, *, fitting=False): "sources": { name: { "source": source.source.model_dump(mode="json"), + "split_index": source.source.selected_split_index, "paths": [str(path) for path in source.paths], "kind": source.kind, "checksums": { diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index e528392a..cb72a4c2 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -259,8 +259,12 @@ def load_loader_state_dict( str(key): value for key, value in state.get("iterator_start_states", {}).items() } + # Mid-epoch recovery replays training batches. Validation runs to completion + # before a checkpoint and starts its next iterator from the saved generator. self._restore_from_iterator_start = ( - set(self.iterator_start_states) if resume_from_iterator_start else set() + {"training"} & self.iterator_start_states.keys() + if resume_from_iterator_start + else set() ) diff --git a/src/sequifier/trajectory/prepared.py b/src/sequifier/trajectory/prepared.py index a7624045..13391369 100644 --- a/src/sequifier/trajectory/prepared.py +++ b/src/sequifier/trajectory/prepared.py @@ -18,11 +18,12 @@ def prepared_table( """ if source.metadata_config_path is None: raise ValueError("prepared histories require metadata_config_path") + from sequifier.config.depth_layout import depth_mask_metadata_key from sequifier.helpers import PANDAS_TO_TORCH_TYPES, read_data from sequifier.io.pt_payload import load_pt_payload, tabular_tensor_batch metadata = load_dataset_metadata(source.metadata_config_path) - from sequifier.config.depth_layout import depth_mask_metadata_key + split_index = source.selected_split_index rows = {} for path in paths: @@ -31,6 +32,11 @@ def prepared_table( path, layouts=metadata.depth_layouts, n_classes=metadata.n_classes ) else: + window_length = ( + metadata.width_for_split(split_index) + if split_index is not None + else metadata.window_length + ) data = read_data(str(path), path.suffix.lower().lstrip(".")) batch = tabular_tensor_batch( data, @@ -39,7 +45,7 @@ def prepared_table( for column, dtype in metadata.column_data_types.items() }, sorted(set(data["inputCol"].to_list())), - metadata.window_length, + window_length, ) sequences, ids, starts, pads = ( batch.sequences, From a4835167e5349ec3f663596840106df02b342bed Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 13:58:53 +0100 Subject: [PATCH 70/71] cleanup --- documentation/training/artifacts.md | 2 +- src/sequifier/config/inference_common.py | 10 +- src/sequifier/config/policy_train_config.py | 20 +++- src/sequifier/hyperparameter_search.py | 5 + src/sequifier/infer.py | 100 ++++++++++++++-- src/sequifier/io/output_paths.py | 20 ++-- src/sequifier/rl/inference.py | 9 ++ src/sequifier/rl/ppo_collector.py | 1 + src/sequifier/trajectory/export.py | 120 ++++++++++++++++---- 9 files changed, 241 insertions(+), 46 deletions(-) diff --git a/documentation/training/artifacts.md b/documentation/training/artifacts.md index 809cc75e..a571025c 100644 --- a/documentation/training/artifacts.md +++ b/documentation/training/artifacts.md @@ -112,7 +112,7 @@ files keep their canonical target-specific destinations. Forecast and policy rollout use the shared validated trajectory record. Training-time PPO and GRPO archives instead use the common -`sequifier.collection-archive` envelope (version 122) with `algorithm` and +`sequifier.collection-archive` envelope with `algorithm` and `collection` fields. The collection payload contains algorithm-specific replay and diagnostic information. `read_collection_archive` reads the envelope; `load_collection_record` additionally validates PPO collections. These archives diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index 0ceada0c..aa128486 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -66,11 +66,15 @@ def validate_output_destination( self, destination: str | Path, *, label: str ) -> None: inputs = list(getattr(self, "model_paths", ())) - if (data_path := getattr(self, "data_path", None)) is not None: - inputs.append(data_path) + for field in ("data_path", "metadata_config_path", "static_features_path"): + if path := getattr(self, field, None): + inputs.append(path) + inputs = [normalize_path(path, self.project_root) for path in inputs] + if training_config_path := getattr(self, "training_config_path", None): + inputs.append(training_config_path) validate_inference_destination( destination, - (normalize_path(path, self.project_root) for path in inputs), + inputs, label=label, ) diff --git a/src/sequifier/config/policy_train_config.py b/src/sequifier/config/policy_train_config.py index 4030fa94..2ab1759e 100644 --- a/src/sequifier/config/policy_train_config.py +++ b/src/sequifier/config/policy_train_config.py @@ -1,5 +1,6 @@ """Authored and resolved trajectory training contracts for the train command.""" +import json from pathlib import Path from typing import Literal @@ -82,7 +83,10 @@ def resolve_sequence_source( if source.metadata_config_path is not None and ( source.static_data_path is None or source.split_index is not None ): - from sequifier.config.metadata import load_dataset_metadata + from sequifier.config.metadata import ( + load_dataset_metadata, + validate_persisted_metadata_versions, + ) if metadata is None: metadata = load_dataset_metadata(values["metadata_config_path"]) @@ -105,11 +109,17 @@ def resolve_sequence_source( "source split_index is outside the prepared metadata splits" ) path = (root / metadata.split_paths[source.split_index]).resolve() - pattern = "*.pt" if any(path.glob("*.pt")) else "*.parquet" + paths = [str(path)] + if path.is_dir(): + manifest_path = path / "metadata.json" + folder_metadata = json.loads(manifest_path.read_text()) + validate_persisted_metadata_versions(folder_metadata) + paths = [ + str(path / shard["path"]) + for shard in folder_metadata["batch_files"] + ] values.update( - paths=sorted(str(item) for item in path.glob(pattern)) - if path.is_dir() - else [str(path)], + paths=paths, format="prepared", split_index=None, ) diff --git a/src/sequifier/hyperparameter_search.py b/src/sequifier/hyperparameter_search.py index 6f876cc9..ea79fe15 100644 --- a/src/sequifier/hyperparameter_search.py +++ b/src/sequifier/hyperparameter_search.py @@ -110,6 +110,7 @@ def objective( monitor = final_phase.evaluation.monitor if final_phase.evaluation else None monitor_metric = monitor.metric if monitor is not None else "loss" monitor_direction = monitor.direction if monitor is not None else "min" + monitor_part = monitor.source.partition(".")[2] if monitor is not None else "" expected_direction = (config.evaluation_metric_directions or ["min"])[0] if not config.evaluation_script_path and expected_direction != monitor_direction: raise ValueError( @@ -191,6 +192,10 @@ def consume_metrics( or evaluation_id in consumed_evaluation_ids or data.get("metric") != monitor_metric or (data.get("phase") and data["phase"] != final_phase.name) + or ( + monitor is not None + and (data.get("part") or "") != monitor_part + ) or data.get("target") != "__total__" or not data.get("value") or not data.get("global_step") diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index aa1c3a8b..1d43dbca 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -28,6 +28,7 @@ validate_stored_window_width, write_data, ) +from sequifier.io.output_paths import require_file_destinations, unique_paths from sequifier.io.preprocessing import denormalize_values from sequifier.objectives import get_objective_class from sequifier.special_tokens import ( @@ -618,6 +619,7 @@ def infer_worker( is_folder_input = os.path.isdir( normalize_path(config.data_path, config.project_root) ) + start_pct, end_pct = percentage_limits or (0.0, 100.0) dataset = None if not is_folder_input: # Standalone Single-File Path Execution @@ -671,8 +673,6 @@ def infer_worker( raise ValueError( "percentage_limits must be provided for folder-based read formats" ) - start_pct, end_pct = percentage_limits - # Direct folders to their respective lazy loaders based on file format if config.read_format == "pt": dataset = load_pt_dataset(config.data_path, start_pct, end_pct) @@ -726,6 +726,22 @@ def infer_worker( logger.info(f"Inferring for {model_id}") if config.task == "forecast": + if getattr(config, "output_path", None) is not None and getattr( + config, "output_probabilities", False + ): + part_count = 1 + if is_folder_input: + total = len( + list(Path(config.data_path).glob(f"*.{config.read_format}")) + ) + part_count = len( + range(total)[ + int(total * start_pct / 100) : int(total * end_pct / 100) + ] + ) + validate_probability_destinations( + config, model_id, inferer.target_column_types, part_count + ) infer_generative(config, inferer, model_id, dataset, column_data_types) if config.task == "embedding": infer_embedding(config, inferer, model_id, dataset, column_data_types) @@ -843,18 +859,15 @@ def _autoregressive_seed_dataframe( @beartype -def inference_output_path( +def _inference_destination( project_root: str, write_format: str, artifact_type: str, model_id: str, data_id: int, - target_column: Optional[str] = None, - output_path: Optional[str] = None, - *, - config: InferenceExecutionConfig | None = None, -) -> str: - """Prepare an inference output path, checking inputs before creating directories.""" + target_column: str | None = None, + output_path: str | None = None, +) -> tuple[Path, str]: if output_path is not None and artifact_type in {"predictions", "embeddings"}: destination = Path(project_root) / output_path if data_id: @@ -868,12 +881,81 @@ def inference_output_path( output_dir /= target_column destination = output_dir / f"part-{data_id:03d}.{write_format}" label = f"{artifact_type} output" + return destination, label + + +def inference_output_path( + project_root: str, + write_format: str, + artifact_type: str, + model_id: str, + data_id: int, + target_column: str | None = None, + output_path: str | None = None, + *, + config: InferenceExecutionConfig | None = None, +) -> str: + """Prepare an inference output path, checking inputs before creating directories.""" + destination, label = _inference_destination( + project_root, + write_format, + artifact_type, + model_id, + data_id, + target_column, + output_path, + ) if config is not None: config.validate_output_destination(destination, label=label) destination.parent.mkdir(parents=True, exist_ok=True) return str(destination) +def validate_probability_destinations(config, model_id, target_types, part_count): + """Reserve all prediction and probability files before writing either.""" + output_path = config.output_path + if config.task == "policy" and output_path is not None: + output_path = str( + Path(normalize_path(output_path, config.project_root)).absolute() + ) + paths = [] + for index in range(part_count): + paths.append( + _inference_destination( + config.project_root, + config.write_format, + "predictions", + model_id, + index, + output_path=output_path, + )[0] + ) + paths.extend( + _inference_destination( + config.project_root, + config.write_format, + "probabilities", + model_id, + index, + target, + )[0] + for target, kind in target_types.items() + if kind == "categorical" + ) + paths = unique_paths(paths, "prediction and probability outputs must be distinct") + require_file_destinations(paths) + existing_files = set() + for path in paths: + try: + stat = path.stat() + except FileNotFoundError: + continue + identity = (stat.st_dev, stat.st_ino) + if identity in existing_files: + raise ValueError("prediction and probability outputs must be distinct") + existing_files.add(identity) + + @beartype def infer_embedding( config: "ResolvedInferenceConfig", diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py index 648a61b4..7955888c 100644 --- a/src/sequifier/io/output_paths.py +++ b/src/sequifier/io/output_paths.py @@ -1,5 +1,6 @@ """Shared path geometry and destination checks.""" +import re from pathlib import Path @@ -109,22 +110,29 @@ def validate_training_destinations(config, checkpoint, protected, *, phase_offse require_file_destinations(files) require_directory_destinations(directories) # A generated snapshot may be selected as the rolling checkpoint on resume. - snapshot_dirs = { + snapshot_phases = { ( PhaseArtifacts.for_phase( config.phase_directory, index, phase.name ).model.parent / "checkpoints" - ).resolve() + ).resolve(): phase for index, phase in enumerate(phases, phase_offset) } - resume = getattr(getattr(config, "runtime_settings", None), "resume", None) + resume = getattr(config, "resume", None) or getattr( + getattr(config, "runtime_settings", None), "resume", None + ) + snapshot_phase = snapshot_phases.get(checkpoint.parent) + snapshot_name = snapshot_phase is not None and ( + checkpoint.name.startswith(f"{config.model_name}-") + if snapshot_phase.algorithm == "supervised" + else re.fullmatch(r"transaction-\d{8,}\.pt", checkpoint.name) is not None + ) snapshot_resume = ( resume is not None and resume.policy != "never" and checkpoint.is_file() - and checkpoint.parent in snapshot_dirs - and checkpoint.name.startswith(f"{config.model_name}-") + and snapshot_name and checkpoint.suffix == ".pt" and manifest != Path(config.report_path).resolve() ) @@ -143,8 +151,6 @@ def validate_training_destinations(config, checkpoint, protected, *, phase_offse [*files, *directories], protected, label="training output" ) if any(phase.algorithm == "supervised" for phase in phases): - import re - export_directory = (Path(config.project_root) / "models").resolve() names = [config.model_name] + [ f"{config.model_name}-{name}" for name in config.dataset_training diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index 88a70d92..f9a1472b 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -134,6 +134,15 @@ def policy_windows(data, spec, config): @torch.no_grad() def predict_policy(policy, data, config): spec = policy.config + if getattr(config, "output_path", None) is not None and config.output_probabilities: + from sequifier.infer import validate_probability_destinations + + validate_probability_destinations( + config, + Path(config.model_paths[0]).stem, + {name: field.type for name, field in spec.actions.items()}, + 1, + ) row_key = policy_row_key(data, spec) windows = policy_windows(data, spec, config) generator = action_generator(policy.device, config.seed) diff --git a/src/sequifier/rl/ppo_collector.py b/src/sequifier/rl/ppo_collector.py index ef943b34..2f298d52 100644 --- a/src/sequifier/rl/ppo_collector.py +++ b/src/sequifier/rl/ppo_collector.py @@ -145,6 +145,7 @@ def load_state_dict(self, state, *, exact): self.cursor.generator.set_state(state["generator"]) if exact: self.cursor.history.load_state_dict(state["history"]) + self.cursor.desired_return = self.cursor.history.current.desired_return self.cursor.refresh() if ( self.reset_count != reset_count diff --git a/src/sequifier/trajectory/export.py b/src/sequifier/trajectory/export.py index 21bf2c7c..5775f6be 100644 --- a/src/sequifier/trajectory/export.py +++ b/src/sequifier/trajectory/export.py @@ -3,6 +3,7 @@ import json import os import shutil +import sys import tempfile from pathlib import Path @@ -21,6 +22,71 @@ TRANSITION_FLAGS, ) +MAX_FLOAT64_INTEGER = int(sys.float_info.max) + + +def _is_integer(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _float64_value(value, dtype, column, pa): + if value is None: + return value + if pa.types.is_float64(dtype) and _is_integer(value): + if abs(value) > MAX_FLOAT64_INTEGER: + raise ValueError(f"{column}: integer exceeds the finite float64 range") + converted = float(value) + if converted != value: + raise ValueError( + f"{column}: integer cannot be represented exactly as float64" + ) + return converted + if pa.types.is_list(dtype) or pa.types.is_large_list(dtype): + return [_float64_value(item, dtype.value_type, column, pa) for item in value] + if pa.types.is_struct(dtype): + return { + name: _float64_value(item, dtype.field(name).type, f"{column}.{name}", pa) + for name, item in value.items() + } + return value + + +def _checked_array(entries, dtype, column, pa): + dtype = dtype if dtype is not None else pa.infer_type(entries) + return pa.array( + [_float64_value(value, dtype, column, pa) for value in entries], type=dtype + ) + + +def _cast_table(table, schema, pa): + arrays = [] + for field in schema: + column = table[field.name] + arrays.append( + column + if column.type == field.type + else _checked_array(column.to_pylist(), field.type, field.name, pa) + ) + return pa.Table.from_arrays(arrays, schema=schema) + + +def _action_type(field, pa): + if field.type == "real": + return pa.float64() + if not field.values: + raise ValueError("categorical sequence actions require declared values") + if all(isinstance(label, str) for label in field.values): + return pa.string() + if any(isinstance(label, str) for label in field.values): + raise ValueError( + "categorical sequence actions cannot mix string and numeric labels" + ) + if all(_is_integer(label) for label in field.values): + if min(field.values) >= 0 and max(field.values) >= 2**63: + return pa.uint64() + return pa.int64() + return pa.float64() + def import_sequences(config) -> Path: """Commit canonical columnar episodes as checksummed raw Parquet shards.""" @@ -57,12 +123,7 @@ def import_sequences(config) -> Path: raise ValueError("sequence import requires at least one episode") action_type = pa.struct( [ - pa.field( - name, - pa.string() - if field.type == "categorical" and isinstance(field.values[0], str) - else pa.float64(), - ) + pa.field(name, _action_type(field, pa)) for name, field in policy.actions.items() ] + [ @@ -89,14 +150,24 @@ def import_sequences(config) -> Path: def flush(): nonlocal schema if tables: - table = pa.concat_tables(tables, promote_options="permissive") - schema = schema or table.schema + shard_schema = pa.unify_schemas( + [table.schema for table in tables], promote_options="permissive" + ) + table = pa.concat_tables( + [_cast_table(table, shard_schema, pa) for table in tables] + ) + schema = ( + shard_schema + if schema is None + else pa.unify_schemas( + [schema, shard_schema], promote_options="permissive" + ) + ) name = f"part-{len(shards):05d}.parquet" - pq.write_table(table.cast(schema), stage / name) + pq.write_table(table, stage / name) shards.append( { "path": name, - "sha256": file_checksum(stage / name), "decisions": table.num_rows, } ) @@ -131,24 +202,31 @@ def flush(): for field in TRANSITION_FLAGS }, } - arrays = { - name: pa.array(entries, type=types.get(name)) - for name, entries in values.items() - } + episode_types = dict(types) for name, column in zip( policy.observation_columns, successor_columns, strict=True ): - arrays[column] = pa.array( - [ - successor[name] if successor is not None else None - for successor in (transitions.successors or [None] * length) - ], - type=arrays[name].type, - ) + values[column] = [ + successor[name] if successor is not None else None + for successor in (transitions.successors or [None] * length) + ] + dtype = pa.infer_type([*values[name], *values[column]]) + episode_types[name] = episode_types[column] = dtype + arrays = { + name: _checked_array(entries, episode_types.get(name), name, pa) + for name, entries in values.items() + } tables.append(pa.table(arrays)) if (index + 1) % config.episodes_per_shard == 0: flush() flush() + # Promote earlier shards only when a later episode widened their schema. + for shard in shards: + path = stage / shard["path"] + if pq.read_schema(path) != schema: + table = _cast_table(pq.read_table(path), schema, pa) + pq.write_table(table, path) + shard["sha256"] = file_checksum(path) manifest = { "format": "sequifier.sequence-data", "version": ContractVersions.SEQUENCE_DATA_MANIFEST, From 4a94301fcc20f795ca22de4b9488efa1d2bb7e59 Mon Sep 17 00:00:00 2001 From: Leon Luithlen Date: Thu, 19 Nov 2026 16:26:02 +0100 Subject: [PATCH 71/71] remove metaprogramming elements --- .../artifacts/backbone_repository.py | 10 +- src/sequifier/artifacts/dependencies.py | 2 +- src/sequifier/artifacts/model_artifact.py | 7 +- src/sequifier/artifacts/model_config.py | 6 +- .../canonical_hyperparameter_search_config.py | 5 +- src/sequifier/config/components.py | 23 +-- src/sequifier/config/depth_layout.py | 11 +- src/sequifier/config/identifiers.py | 15 ++ src/sequifier/config/infer_config.py | 30 +++- src/sequifier/config/inference_common.py | 30 ++-- src/sequifier/config/initialization_config.py | 19 ++- src/sequifier/config/learning_config.py | 8 +- src/sequifier/config/parameter_selection.py | 24 ++- src/sequifier/config/phase_controls.py | 34 +++- src/sequifier/config/policy_infer_config.py | 6 +- src/sequifier/config/policy_plan.py | 64 ++++++-- src/sequifier/config/sources.py | 11 +- src/sequifier/config/train_config.py | 49 ++++-- src/sequifier/config/training_resolution.py | 99 ++++++------ src/sequifier/evaluation/service.py | 30 ++-- src/sequifier/export/onnx.py | 2 +- src/sequifier/helpers.py | 15 +- src/sequifier/hyperparameter_search.py | 16 +- src/sequifier/infer.py | 51 ++---- src/sequifier/integration/callbacks.py | 100 ++++++++---- src/sequifier/integration/factories.py | 14 ++ src/sequifier/io/depth_preprocess.py | 20 +-- src/sequifier/io/episode_features.py | 11 +- src/sequifier/io/iql_dataset.py | 9 +- src/sequifier/io/memory_budget.py | 10 +- src/sequifier/io/output_paths.py | 37 +++-- src/sequifier/io/policy_materialization.py | 9 +- src/sequifier/io/preference_dataset.py | 6 +- src/sequifier/io/pt_payload.py | 53 +++--- src/sequifier/io/sample_order.py | 30 ++-- src/sequifier/io/sequence.py | 7 +- .../io/sequifier_dataset_from_file.py | 5 +- .../io/sequifier_dataset_from_folder_pt.py | 4 +- .../sequifier_dataset_from_folder_pt_lazy.py | 4 +- src/sequifier/io/static_features.py | 10 +- src/sequifier/io/tensors.py | 4 +- src/sequifier/io/values.py | 8 +- src/sequifier/model/decoders.py | 25 +-- src/sequifier/model/depth_ingestion.py | 3 - src/sequifier/model/execution_schema.py | 8 +- src/sequifier/model/factory.py | 126 ++++++++------- src/sequifier/model/hash_codec.py | 2 +- src/sequifier/model/history_transfer.py | 25 +-- src/sequifier/model/ingestion_compiler.py | 16 +- src/sequifier/model/ingestions.py | 35 ++-- src/sequifier/model/initialization.py | 153 +++++------------- src/sequifier/model/layers.py | 4 +- src/sequifier/model/module_roles.py | 92 +++++++++++ src/sequifier/model/network.py | 70 +++++--- src/sequifier/model/parameter_catalog.py | 17 +- src/sequifier/model/parameter_groups.py | 107 +++--------- src/sequifier/model/static_conditioning.py | 8 +- src/sequifier/model/tracing.py | 11 +- src/sequifier/objectives.py | 2 +- src/sequifier/preprocess.py | 22 ++- src/sequifier/rl/checkpoint.py | 2 +- src/sequifier/rl/environment.py | 71 +++++--- src/sequifier/rl/inference.py | 2 +- src/sequifier/rl/joint_distribution.py | 2 +- src/sequifier/rl/learner.py | 6 +- src/sequifier/rl/offline_data.py | 18 ++- src/sequifier/rl/policy.py | 10 +- src/sequifier/rl/ppo_evaluation.py | 7 +- src/sequifier/rl/preparation.py | 11 +- src/sequifier/rl/prepared_inputs.py | 22 ++- src/sequifier/rl/references.py | 14 +- src/sequifier/rl/run_checkpoint.py | 2 +- src/sequifier/rl/run_state.py | 30 ++-- src/sequifier/rl/runtime.py | 2 +- src/sequifier/rl/simulator_workers.py | 11 +- src/sequifier/rl/window_storage.py | 2 +- src/sequifier/rollout/factories.py | 28 +++- src/sequifier/train.py | 15 ++ .../training/distributed_strategy.py | 10 +- src/sequifier/training/loss.py | 12 +- src/sequifier/training/metrics.py | 6 +- src/sequifier/training/optimization.py | 4 +- src/sequifier/training/runtime.py | 9 +- src/sequifier/trajectory/export.py | 7 +- src/sequifier/trajectory/importers.py | 15 +- src/sequifier/visualize_training.py | 4 +- tools/check_contract_versions.py | 2 +- tools/stream_folder_via_scp.py | 8 +- 88 files changed, 1131 insertions(+), 835 deletions(-) create mode 100644 src/sequifier/integration/factories.py create mode 100644 src/sequifier/model/module_roles.py diff --git a/src/sequifier/artifacts/backbone_repository.py b/src/sequifier/artifacts/backbone_repository.py index 7781804f..ae1ebbe5 100644 --- a/src/sequifier/artifacts/backbone_repository.py +++ b/src/sequifier/artifacts/backbone_repository.py @@ -8,6 +8,7 @@ from typing import Any import torch +from pydantic import BaseModel from torch import Tensor, nn from sequifier.contract_versions import ContractVersions, require_contract_version @@ -26,17 +27,14 @@ torch.bfloat16, torch.float32, torch.float64, + torch.float8_e4m3fn, + torch.float8_e5m2, } -SUPPORTED_DTYPES.update( - dtype - for name in ("float8_e4m3fn", "float8_e5m2") - if (dtype := getattr(torch, name, None)) is not None -) @beartype def canonical_architecture(architecture: Any) -> dict[str, Any]: - if hasattr(architecture, "model_dump"): + if isinstance(architecture, BaseModel): return architecture.model_dump(mode="json") if not isinstance(architecture, dict): raise TypeError("Backbone architecture must be a mapping or Pydantic model.") diff --git a/src/sequifier/artifacts/dependencies.py b/src/sequifier/artifacts/dependencies.py index e3382831..a7006752 100644 --- a/src/sequifier/artifacts/dependencies.py +++ b/src/sequifier/artifacts/dependencies.py @@ -163,7 +163,7 @@ def policy_dependencies( manifest.add(path) for index, phase in enumerate(config.phases): manifest.include(prepared.phase_checksums(config, phase, inputs)) - reference = getattr(phase, "reference", None) + reference = phase.reference_config if reference and reference.path: path = Path(config.project_root) / reference.path checksum = ( diff --git a/src/sequifier/artifacts/model_artifact.py b/src/sequifier/artifacts/model_artifact.py index 3efeb27b..2daea4ce 100644 --- a/src/sequifier/artifacts/model_artifact.py +++ b/src/sequifier/artifacts/model_artifact.py @@ -104,10 +104,11 @@ def build_model_artifact( for site in network.trace_catalog_for(interface_name) ) resolved_provenance = dict(provenance or {}) - phases = getattr(config, "training_plan", None) - if phases: + from sequifier.config.train_config import ResolvedSequifierConfig + + if isinstance(config, ResolvedSequifierConfig) and config.training_plan: resolved_provenance["training_plan"] = [ - phase.model_dump(mode="json") for phase in phases + phase.model_dump(mode="json") for phase in config.training_plan ] artifact = ModelArtifact( format_version=ContractVersions.MODEL_ARTIFACT, diff --git a/src/sequifier/artifacts/model_config.py b/src/sequifier/artifacts/model_config.py index a443fae2..4edb70d9 100644 --- a/src/sequifier/artifacts/model_config.py +++ b/src/sequifier/artifacts/model_config.py @@ -301,7 +301,9 @@ def resolved_training_config( def extend_prediction_distributions(execution, declared_model): """Allow explicit new scale parameters while preserving an initialized model.""" exclude = {"interfaces": {"__all__": {"prediction_distributions"}}} - if hasattr(declared_model, "architecture_model"): + from sequifier.config.train_config import OwnedModelSpec + + if isinstance(declared_model, OwnedModelSpec): declared_model = declared_model.architecture_model() declared = declared_model.model_dump(exclude=exclude) existing = execution.model.model_dump(exclude=exclude) @@ -346,7 +348,7 @@ def load_attached_model_weights(network, artifact): missing = expected.keys() - source.keys() allowed = set() for name, route in network.interfaces.items(): - scales = getattr(route, "prediction_scales", None) + scales = route.prediction_scales if not isinstance(scales, PredictionScales): continue original = artifact.model_config.values["interfaces"][name].get( diff --git a/src/sequifier/config/canonical_hyperparameter_search_config.py b/src/sequifier/config/canonical_hyperparameter_search_config.py index fb0b2e70..03722615 100644 --- a/src/sequifier/config/canonical_hyperparameter_search_config.py +++ b/src/sequifier/config/canonical_hyperparameter_search_config.py @@ -854,7 +854,10 @@ def sample_trial(self, trial: Any, run_index: int) -> SequifierConfig: try: return _validate_training_candidate(values) except (ValidationError, ValueError) as error: - parameters = getattr(trial, "params", {}) + try: + parameters = trial.params + except AttributeError: + parameters = {} raise ValueError( "Sampled hyperparameters produce an invalid canonical training " f"config for parameters {parameters!r}:\n{error}" diff --git a/src/sequifier/config/components.py b/src/sequifier/config/components.py index 1a863a48..36caa981 100644 --- a/src/sequifier/config/components.py +++ b/src/sequifier/config/components.py @@ -15,6 +15,7 @@ model_validator, ) +from sequifier.config.identifiers import validate_module_dict_key from sequifier.config.initialization_config import ModelInitializationConfig from sequifier.config.naming import ConfigModel from sequifier.config.probabilities import ProbabilityDistribution @@ -226,7 +227,7 @@ class GroupedIngestionConfig(IngestionComponentBase): def validate_groups(self): grouped_columns = [] for group_name, columns in self.groups.items(): - _validate_module_dict_key( + validate_module_dict_key( group_name, f"grouped ingestion group {group_name!r}" ) if not columns: @@ -432,18 +433,6 @@ def _validate_column_list_unique(columns: list[str], field_name: str) -> None: raise ValueError(f"{field_name} cannot contain duplicate columns") -@beartype -def _validate_module_dict_key(key: str, usage: str) -> None: - if key == "": - raise ValueError(f"{usage} cannot be empty") - if "." in key: - raise ValueError(f"{usage} cannot contain '.'") - from torch import nn - - if hasattr(nn.ModuleDict(), key): - raise ValueError(f"{usage} collides with a ModuleDict attribute") - - @beartype def _validate_feature_embedding_dims( feature_embedding_dims: Optional[dict[str, int]], field_name: str @@ -745,7 +734,7 @@ class CompositeDecoderComponentConfig(DecoderComponentBase): @beartype def validate_branch_names(cls, branches): for branch_name in branches: - _validate_module_dict_key( + validate_module_dict_key( branch_name, f"Target decoding branch {branch_name!r}" ) return branches @@ -882,7 +871,9 @@ def default_global_position_scope(cls, values): position_type = ( position_encoding.get("type", "learned") if isinstance(position_encoding, dict) - else getattr(position_encoding, "type", "learned") + else position_encoding.type + if isinstance(position_encoding, BackbonePositionEncodingConfig) + else "learned" ) if ( position_type in {"range", "range_concat", "sinusoidal"} @@ -969,7 +960,7 @@ class CompositeIngestionConfig(IngestionComponentBase): @beartype def validate_branch_names(cls, branches): for branch_name in branches: - _validate_module_dict_key( + validate_module_dict_key( branch_name, f"Composite ingestion branch {branch_name!r}" ) return branches diff --git a/src/sequifier/config/depth_layout.py b/src/sequifier/config/depth_layout.py index 0040070b..f3bf9b71 100644 --- a/src/sequifier/config/depth_layout.py +++ b/src/sequifier/config/depth_layout.py @@ -3,8 +3,8 @@ from typing import Literal from pydantic import ConfigDict, Field, RootModel, model_validator -from torch import nn +from sequifier.config.identifiers import validate_module_dict_key from sequifier.config.naming import ConfigModel @@ -43,10 +43,13 @@ class DepthLayoutRegistryModel(RootModel[dict[str, DepthLayoutModel]]): @model_validator(mode="after") def validate_registry(self): occupied = set() - storage = nn.ModuleDict() for name, layout in self.root.items(): - if not name or "." in name or hasattr(storage, name): - raise ValueError(f"Invalid depth layout module key: {name!r}") + try: + validate_module_dict_key(name, "Depth layout name") + except ValueError as error: + raise ValueError( + f"Invalid depth layout module key: {name!r}" + ) from error if occupied.intersection(layout.columns): raise ValueError("A feature cannot belong to multiple depth layouts") occupied.update(layout.columns) diff --git a/src/sequifier/config/identifiers.py b/src/sequifier/config/identifiers.py index 6de312de..5050d9b4 100644 --- a/src/sequifier/config/identifiers.py +++ b/src/sequifier/config/identifiers.py @@ -10,3 +10,18 @@ def identifier(value: str, usage: str) -> str: f"{usage} {value!r} must be a valid identifier and cannot contain '.'." ) return value + + +@beartype +def validate_module_dict_key(key: str, usage: str) -> None: + if not key: + raise ValueError(f"{usage} cannot be empty") + if "." in key: + raise ValueError(f"{usage} cannot contain '.'") + + from torch import nn + + try: + nn.ModuleDict().add_module(key, None) + except KeyError as error: + raise ValueError(f"{usage} collides with a ModuleDict attribute") from error diff --git a/src/sequifier/config/infer_config.py b/src/sequifier/config/infer_config.py index a4336902..f062ea32 100644 --- a/src/sequifier/config/infer_config.py +++ b/src/sequifier/config/infer_config.py @@ -210,6 +210,8 @@ def _assert_metadata_matches_source( ) -> None: if _image_contract(loaded_metadata) != _image_contract(selected_metadata): raise ValueError(f"Inference image source contract does not match {source}") + loaded_fields = dict(loaded_metadata) + selected_fields = dict(selected_metadata) relevant = list(loaded_metadata.column_data_types) if loaded_metadata.depth_layouts.compatibility_signature( relevant @@ -221,16 +223,16 @@ def _assert_metadata_matches_source( "id_maps", "selected_columns_statistics", ): - expected = getattr(loaded_metadata, field) - found = getattr(selected_metadata, field) + expected = loaded_fields[field] + found = selected_fields[field] found = {key: found.get(key) for key in expected} if _comparable_value(field, found) != _comparable_value(field, expected): raise ValueError( f"Inference metadata field {field!r} does not match {source}." ) for field in ("special_token_ids", "normalize_real_columns"): - expected = getattr(loaded_metadata, field) - found = getattr(selected_metadata, field) + expected = loaded_fields[field] + found = selected_fields[field] if found != expected: raise ValueError( f"Inference metadata field {field!r} does not match {source}: " @@ -368,9 +370,14 @@ def _assert_folder_metadata_matches( if folder_classes != selected_classes: mismatches.append("n_classes") - for field in ("id_maps", "selected_columns_statistics"): - selected_values = getattr(selected_metadata, field) - folder_values = getattr(folder_metadata, field) + for field, selected_values, folder_values in ( + ("id_maps", selected_metadata.id_maps, folder_metadata.id_maps), + ( + "selected_columns_statistics", + selected_metadata.selected_columns_statistics, + folder_metadata.selected_columns_statistics, + ), + ): if { column: selected_values[column] for column in relevant_columns @@ -887,7 +894,6 @@ class _InferenceConfigBase( model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") - metadata_config_path: Optional[str] = None model_paths: list[str] = Field(min_length=1) task: Literal["forecast", "embedding"] training_objective: _ObjectiveT @@ -920,6 +926,14 @@ class _InferenceConfigBase( autoregressive_generation: bool = Field(default=False) generation_steps: Optional[int] = Field(default=None) + def input_paths(self) -> list[str]: + inputs = super().input_paths() + if self.static_features_path: + inputs.append(normalize_path(self.static_features_path, self.project_root)) + if self.training_config_path: + inputs.append(self.training_config_path) + return inputs + @model_validator(mode="after") @beartype def validate_route_selection(self): diff --git a/src/sequifier/config/inference_common.py b/src/sequifier/config/inference_common.py index aa128486..baa82948 100644 --- a/src/sequifier/config/inference_common.py +++ b/src/sequifier/config/inference_common.py @@ -3,7 +3,7 @@ import json from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import TYPE_CHECKING, Any, Literal from pydantic import ConfigDict, Field, field_validator, model_validator @@ -21,6 +21,13 @@ class InferenceExecutionConfig(ConfigModel): read_format: str = "parquet" write_format: str = "csv" output_path: str | None = None + metadata_config_path: str | None = None + + if TYPE_CHECKING: + model_paths: list[str] + data_path: Any + task: Any + rollout: Any @property def project_root(self): @@ -45,9 +52,9 @@ def inference_batch_size(self): @model_validator(mode="after") def validate_resources(self): self.run.validate_for("infer") - if self.output_path is not None and len(getattr(self, "model_paths", ())) != 1: + if self.output_path is not None and len(self.model_paths) != 1: raise ValueError("output_path requires exactly one model path") - rollout = getattr(self, "rollout", None) + rollout = self.rollout if rollout is not None: self.validate_output_destination( normalize_path(rollout.trajectory_path, self.project_root), @@ -56,7 +63,7 @@ def validate_resources(self): elif self.output_path is not None: destination = ( normalize_path(self.output_path, self.project_root) - if getattr(self, "task", None) == "policy" + if self.task == "policy" else Path(self.project_root) / self.output_path ) self.validate_output_destination(destination, label="output_path") @@ -65,19 +72,20 @@ def validate_resources(self): def validate_output_destination( self, destination: str | Path, *, label: str ) -> None: - inputs = list(getattr(self, "model_paths", ())) - for field in ("data_path", "metadata_config_path", "static_features_path"): - if path := getattr(self, field, None): - inputs.append(path) - inputs = [normalize_path(path, self.project_root) for path in inputs] - if training_config_path := getattr(self, "training_config_path", None): - inputs.append(training_config_path) + inputs = self.input_paths() validate_inference_destination( destination, inputs, label=label, ) + def input_paths(self) -> list[str]: + inputs = list(self.model_paths) + inputs.extend( + path for path in (self.data_path, self.metadata_config_path) if path + ) + return [normalize_path(path, self.project_root) for path in inputs] + class InferenceRouteConfig(ConfigModel): """Validate artifact and interface selection before route-specific options.""" diff --git a/src/sequifier/config/initialization_config.py b/src/sequifier/config/initialization_config.py index 77971d29..7d428fd1 100644 --- a/src/sequifier/config/initialization_config.py +++ b/src/sequifier/config/initialization_config.py @@ -127,6 +127,14 @@ class LayerGroupInitialization(ConfigModel): weight: InitializationMethodConfig | None = None bias: InitializationMethodConfig | None = None + def parameter_kinds(self) -> tuple[ParameterKind, ...]: + kinds: list[ParameterKind] = [] + if self.weight is not None: + kinds.append("weight") + if self.bias is not None: + kinds.append("bias") + return tuple(kinds) + @model_validator(mode="after") @beartype def validate_not_empty(self): @@ -149,8 +157,7 @@ def configured_targets(self) -> set[InitializationTarget]: return { (group, parameter_kind) for group, initialization in self.root.items() - for parameter_kind in ("weight", "bias") - if getattr(initialization, parameter_kind) is not None + for parameter_kind in initialization.parameter_kinds() } @@ -176,6 +183,14 @@ class LayerGroupInitializationSampling(ConfigModel): weight: InitializationMethodSamplingConfig | None = None bias: InitializationMethodSamplingConfig | None = None + def parameter_kinds(self) -> tuple[ParameterKind, ...]: + kinds: list[ParameterKind] = [] + if self.weight is not None: + kinds.append("weight") + if self.bias is not None: + kinds.append("bias") + return tuple(kinds) + @model_validator(mode="after") @beartype def validate_not_empty(self): diff --git a/src/sequifier/config/learning_config.py b/src/sequifier/config/learning_config.py index 2ab89ba1..e5b6f0a6 100644 --- a/src/sequifier/config/learning_config.py +++ b/src/sequifier/config/learning_config.py @@ -21,8 +21,10 @@ def validate_targets(self, targets, usage): targets = set(targets) if set(self.criterion) != targets: raise ValueError(f"{usage}: criterion keys must equal interface targets") - for label in ("class_weights", "loss_weights"): - weights = getattr(self, label) + for label, weights in ( + ("class_weights", self.class_weights), + ("loss_weights", self.loss_weights), + ): if weights is not None and weights.keys() - targets: raise ValueError(f"{usage}: {label} references unknown targets") if self.loss_weights is not None and all( @@ -35,7 +37,7 @@ def validate_targets(self, targets, usage): @model_validator(mode="after") def losses(self): for name in self.criterion.values(): - if name != "AccurateCrossEntropyLoss" and not hasattr(torch.nn, name): + if name != "AccurateCrossEntropyLoss" and name not in vars(torch.nn): raise ValueError(f"unknown criterion {name!r}") for weights in (self.loss_weights, self.class_weights): for value in (weights or {}).values(): diff --git a/src/sequifier/config/parameter_selection.py b/src/sequifier/config/parameter_selection.py index 7c37fbba..8aa8b42e 100644 --- a/src/sequifier/config/parameter_selection.py +++ b/src/sequifier/config/parameter_selection.py @@ -22,22 +22,20 @@ class ParameterSelector(ConfigModel): @model_validator(mode="after") def conditions(self): - if all(getattr(self, key) in (None, []) for key in type(self).model_fields): + if all(value in (None, []) for _, value in self): raise ValueError("a parameter selector requires at least one condition") - for key in ( - "semantic_groups", - "branch_path", - "exclude_modules", - "exclude_semantic_groups", + for key, value in ( + ("semantic_groups", self.semantic_groups), + ("branch_path", self.branch_path), + ("exclude_modules", self.exclude_modules), + ("exclude_semantic_groups", self.exclude_semantic_groups), ): - value = getattr(self, key) if value is not None and ( (not value and key in {"semantic_groups", "branch_path"}) or any(not x for x in value) ): raise ValueError(f"{key} requires nonempty entries") - for key in ("module", "interface"): - value = getattr(self, key) + for key, value in (("module", self.module), ("interface", self.interface)): if value is not None and ( not value or any(not x for x in value.split(".")) ): @@ -122,11 +120,11 @@ def validate_semantic_patterns(self, descriptors): ] positive = self.model_copy(update={"exclude_semantic_groups": []}) included = [descriptor for descriptor in scoped if positive.matches(descriptor)] - for field, candidates in ( - ("semantic_groups", scoped), - ("exclude_semantic_groups", included), + for field, patterns, candidates in ( + ("semantic_groups", self.semantic_groups, scoped), + ("exclude_semantic_groups", self.exclude_semantic_groups, included), ): - for pattern in getattr(self, field) or []: + for pattern in patterns or []: if not any( fnmatchcase(group, pattern) for descriptor in candidates diff --git a/src/sequifier/config/phase_controls.py b/src/sequifier/config/phase_controls.py index 805983e3..c249bb05 100644 --- a/src/sequifier/config/phase_controls.py +++ b/src/sequifier/config/phase_controls.py @@ -6,6 +6,9 @@ from sequifier.config.naming import ConfigModel +if TYPE_CHECKING: + from sequifier.config.policy_plan import OnlineEnvironmentConfig, ReferenceConfig + class Cadence(ConfigModel): model_config = ConfigDict(extra="forbid") @@ -61,6 +64,26 @@ class PhaseControls(ConfigModel): model_selection: ModelSelection = Field(default_factory=ModelSelection) stopping: StoppingConfig | None = None + @property + def reference_config(self) -> "ReferenceConfig | None": + return None + + @property + def rollout_archive_directory(self) -> str | None: + return None + + @property + def training_environment(self) -> "OnlineEnvironmentConfig | None": + return None + + @property + def evaluation_environment(self) -> "OnlineEnvironmentConfig | None": + return None + + @property + def uses_actor_history(self) -> bool: + return False + @property def handoff(self): return self.model_selection.handoff @@ -93,14 +116,19 @@ def validate_lifecycle(self): ("checkpoints", self.checkpoints.cadence), ("evaluation", self.evaluation.cadence if self.evaluation else None), ): - if cadence is not None and getattr(cadence, clock) is None: + if ( + cadence is not None + and (cadence.epochs if clock == "epochs" else cadence.iterations) + is None + ): raise ValueError(f"{self.algorithm} {label}.cadence requires {clock}") if self.evaluation is None and (self.stopping or self.handoff == "best"): raise ValueError("stopping and best handoff require evaluation.monitor") if ( self.stopping - and getattr(self.evaluation, "monitor", None) is None - and len(getattr(self.evaluation, "sources", None) or []) != 1 + and self.evaluation is not None + and self.evaluation.monitor is None + and len(self.evaluation.sources or []) != 1 ): raise ValueError("stopping requires evaluation.monitor") if self.algorithm != "supervised": diff --git a/src/sequifier/config/policy_infer_config.py b/src/sequifier/config/policy_infer_config.py index da207b7e..f8045f06 100644 --- a/src/sequifier/config/policy_infer_config.py +++ b/src/sequifier/config/policy_infer_config.py @@ -38,7 +38,6 @@ class PolicyInferenceConfig(InferenceExecutionConfig): item_position_column: str = "itemPosition" rollout: PolicyRolloutConfig | None = None # Optional assertions against the portable artifact contract. - metadata_config_path: str | None = None input_columns: list[str] | None = None static_columns: list[str] | None = None target_columns: list[str] | None = None @@ -146,11 +145,12 @@ def policy_inference_config( ): raise ValueError("action labels must have unique probability column names") contract = _execution_values(execution.options, interface) + requested_fields = dict(request) for key, actual in contract.items(): if ( key in request.model_fields_set - and getattr(request, key) is not None - and getattr(request, key) != actual + and requested_fields[key] is not None + and requested_fields[key] != actual ): raise ValueError(f"{key} differs from policy artifact") if request.metadata_config_path: diff --git a/src/sequifier/config/policy_plan.py b/src/sequifier/config/policy_plan.py index bdc68727..421c7f49 100644 --- a/src/sequifier/config/policy_plan.py +++ b/src/sequifier/config/policy_plan.py @@ -74,6 +74,9 @@ class PhaseConfig(PhaseControls): def optimizer_step_budget(self, training_size=None): raise NotImplementedError("built-in phase must declare its optimizer budget") + def iteration_budget(self, optimizer_step_budget: int | None) -> int: + return optimizer_step_budget or 1 + def optimization_configs(self): unknown = self.optimization.keys() - set(self.optimization_roles) if unknown: @@ -85,10 +88,7 @@ def optimization_configs(self): @model_validator(mode="after") def validate_optimization(self): - if ( - getattr(self, "history_initialization", None) == "phase_start_actor" - and self.optimizer_state == "continue" - ): + if self.uses_actor_history and self.optimizer_state == "continue": raise ValueError("history_initialization requires optimizer_state: reset") self.trainability.validate_roles(self.optimization_roles) for config in self.optimization_configs().values(): @@ -148,6 +148,13 @@ class IQLPhaseConfig(OfflinePhaseConfig): max_advantage_weight: float = Field(default=100.0, ge=1, allow_inf_nan=False) target_update_rate: float = Field(default=0.005, gt=0, le=1, allow_inf_nan=False) + @property + def uses_actor_history(self) -> bool: + return self.history_initialization == "phase_start_actor" + + def iteration_budget(self, optimizer_step_budget: int | None) -> int: + return self.iterations + def optimizer_step_budget(self, training_size=None): return self.iterations @@ -206,12 +213,28 @@ def serial_evaluation(self): monitor: OnlineEvaluationMonitor = Field(default_factory=OnlineEvaluationMonitor) -class PPOPhaseConfig(PhaseConfig): +class OnlinePhaseConfig(PhaseConfig): + environment: OnlineEnvironmentConfig + archive_directory: str | None = None + + @property + def rollout_archive_directory(self) -> str | None: + return self.archive_directory + + @property + def training_environment(self) -> OnlineEnvironmentConfig: + return self.environment + + @property + def evaluation_environment(self) -> OnlineEnvironmentConfig: + return self.evaluation.environment + + +class PPOPhaseConfig(OnlinePhaseConfig): optimization_roles: ClassVar[tuple[str, ...]] = ("actor", "value") algorithm: Literal["ppo"] history_initialization: Literal["random", "phase_start_actor"] = "random" name: str = Field(default="online_ppo", min_length=1) - environment: OnlineEnvironmentConfig evaluation: OnlineEvaluationConfig seed_policy: Literal["derived_per_reset"] = "derived_per_reset" total_env_steps: int = Field(gt=0) @@ -233,6 +256,14 @@ class PPOPhaseConfig(PhaseConfig): archive_scores: bool = False max_buffer_bytes: int = Field(default=536870912, gt=0) + @property + def uses_actor_history(self) -> bool: + return self.history_initialization == "phase_start_actor" + + def iteration_budget(self, optimizer_step_budget: int | None) -> int: + total, per_iteration = self.total_env_steps, self.rollout_env_steps + return (total + per_iteration - 1) // per_iteration + def optimizer_step_budget(self, training_size=None): full, remainder = divmod(self.total_env_steps, self.rollout_env_steps) size = self.batch_size * self.accumulation_batches @@ -274,6 +305,13 @@ class DPOPhaseConfig(BatchedPhaseConfig): scoring_chunk_size: int = Field(default=64, gt=0) reference: ReferenceConfig = Field(default_factory=ReferenceConfig) + @property + def reference_config(self) -> ReferenceConfig: + return self.reference + + def iteration_budget(self, optimizer_step_budget: int | None) -> int: + return self.iterations + def optimizer_step_budget(self, training_size=None): return self.iterations @@ -284,7 +322,7 @@ class GRPOEvaluationConfig(OnlineEvaluationConfig): ) -class GRPOPhaseConfig(BatchedPhaseConfig): +class GRPOPhaseConfig(BatchedPhaseConfig, OnlinePhaseConfig): scoring_chunk_size: int = Field(default=64, gt=0) # One preserves the original in-process collector, with no worker startup. simulator_workers: int = Field(default=1, gt=0) @@ -293,7 +331,6 @@ class GRPOPhaseConfig(BatchedPhaseConfig): worker_threads: int = Field(default=1, gt=0) image_feature_cache_bytes: int = Field(default=0, ge=0) algorithm: Literal["grpo"] - environment: OnlineEnvironmentConfig evaluation: GRPOEvaluationConfig reference: ReferenceConfig = Field(default_factory=ReferenceConfig) total_groups: int = Field(gt=0) @@ -308,7 +345,6 @@ class GRPOPhaseConfig(BatchedPhaseConfig): reference_kl_coefficient: float = Field(default=0.04, ge=0) old_policy_target_kl: float | None = Field(default=None, gt=0) reward_scale: float = Field(default=1.0, gt=0) - archive_directory: str | None = None @model_serializer(mode="wrap") def serialize_execution_options(self, handler): @@ -321,10 +357,18 @@ def serialize_execution_options(self, handler): "worker_threads", "image_feature_cache_bytes", ): - if getattr(self, name) == type(self).model_fields[name].default: + if values.get(name) == type(self).model_fields[name].default: values.pop(name, None) return values + @property + def reference_config(self) -> ReferenceConfig: + return self.reference + + def iteration_budget(self, optimizer_step_budget: int | None) -> int: + total, per_iteration = self.total_groups, self.groups_per_iteration + return (total + per_iteration - 1) // per_iteration + def optimizer_step_budget(self, training_size=None): full, remainder = divmod(self.total_groups, self.groups_per_iteration) size = self.batch_size * self.accumulation_batches diff --git a/src/sequifier/config/sources.py b/src/sequifier/config/sources.py index c27b7476..fffeae9a 100644 --- a/src/sequifier/config/sources.py +++ b/src/sequifier/config/sources.py @@ -11,10 +11,13 @@ def phase_sources(phase): """All authored references, including held-out and preference sources.""" - refs = list(getattr(phase, "sources", [])) - refs.extend(getattr(phase.evaluation, "sources", None) or []) - if (preference := getattr(phase, "preferences", None)) is not None: - refs.append(preference) + if phase.algorithm in {"ppo", "grpo"}: + return [] + refs = [] if phase.algorithm == "dpo" else list(phase.sources) + if phase.evaluation is not None: + refs.extend(phase.evaluation.sources or []) + if phase.algorithm == "dpo": + refs.append(phase.preferences) return refs diff --git a/src/sequifier/config/train_config.py b/src/sequifier/config/train_config.py index 629d467b..4d8117f8 100644 --- a/src/sequifier/config/train_config.py +++ b/src/sequifier/config/train_config.py @@ -169,14 +169,15 @@ def validate_distribution(self): raise ValueError("FSDP requires torch_compile 'none' or 'inner'") if self.data_parallelism == "ddp" and self.torch_compile == "inner": raise ValueError("DDP requires torch_compile 'none' or 'outer'") - for name in ("latest_checkpoint_interval_minutes",): - value = getattr(self, name) - if ( - value is not None - and value <= 0 - and os.getenv("SEQUIFIER_TESTING") != "1" - ): - raise ValueError(f"{name} must be larger than zero") + interval = self.latest_checkpoint_interval_minutes + if ( + interval is not None + and interval <= 0 + and os.getenv("SEQUIFIER_TESTING") != "1" + ): + raise ValueError( + "latest_checkpoint_interval_minutes must be larger than zero" + ) return self @@ -1949,11 +1950,27 @@ def interface_build_view( if interface.static_conditioning else [] ), - **{ - name: getattr(interface, name) - for name in SelectedInterfaceConfig.__dataclass_fields__ - if name in ResolvedModelInterface.model_fields - }, + input_columns=interface.input_columns, + image_sources=interface.image_sources, + preconditioning_image=interface.preconditioning_image, + preconditioning_image_encoder=interface.preconditioning_image_encoder, + target_columns=interface.target_columns, + categorical_hashing=interface.categorical_hashing, + categorical_hash_contracts=interface.categorical_hash_contracts, + target_column_types=interface.target_column_types, + column_data_types=interface.column_data_types, + categorical_columns=interface.categorical_columns, + real_columns=interface.real_columns, + categorical_decoder_special_tokens=interface.categorical_decoder_special_tokens, + feature_layout=interface.feature_layout, + depth_layouts=interface.depth_layouts, + n_classes=interface.n_classes, + id_maps=interface.id_maps, + special_token_ids=interface.special_token_ids, + target_decoder_ids=interface.target_decoder_ids, + target_n_classes=interface.target_n_classes, + target_global_to_decoder=interface.target_global_to_decoder, + window_view=interface.window_view, model=SelectedModelComponents( backbone=config.model.backbone, ingestion=interface.ingestion, @@ -2009,7 +2026,11 @@ def dataset_part_view( part=part, curriculum_training=curriculum_column is not None, curriculum_column=curriculum_column, - epochs=sum(getattr(phase, "epochs", 0) for phase in config.training_plan), + epochs=sum( + phase.epochs + for phase in config.training_plan + if isinstance(phase, (ResolvedTrainingPhase, PolicyImitationPhaseConfig)) + ), read_format=part.read_format, window_stride=window_stride, data_path=part.training_data_path, diff --git a/src/sequifier/config/training_resolution.py b/src/sequifier/config/training_resolution.py index da573ec2..c4b53b4e 100644 --- a/src/sequifier/config/training_resolution.py +++ b/src/sequifier/config/training_resolution.py @@ -97,28 +97,30 @@ def resolve(self, metadata_by_part): def _resolve_phase_sources(config, phase, available): """Expand named sources for both metadata fitting and final run resolution.""" preference = phase.algorithm == "dpo" - fields = ( - [(phase, "preferences"), (phase.evaluation, "sources")] - if preference - else [(phase, "sources"), (phase.evaluation, "sources")] - ) - for owner, key in fields: - refs = getattr(owner, key, None) - if refs is None: - continue - single = key == "preferences" + if phase.algorithm in {"ppo", "grpo"}: + return + + def expand(refs): expanded = [ ref.model_copy(update={"source": full}) - for ref in ([refs] if single else refs) + for ref in refs for full, _ in source_parts(config, ref.source) ] if not expanded or any(ref.source not in available for ref in expanded): raise ValueError( f"{phase.name}: select {'preference' if preference else 'sequence'} parts" ) - if single and len(expanded) != 1: + return expanded + + if preference: + expanded = expand([phase.preferences]) + if len(expanded) != 1: raise ValueError(f"{phase.name}: select one training preference part") - setattr(owner, key, expanded[0] if single else expanded) + phase.preferences = expanded[0] + else: + phase.sources = expand(phase.sources) + if phase.evaluation.sources is not None: + phase.evaluation.sources = expand(phase.evaluation.sources) def phase_interface(config, phase): @@ -126,7 +128,7 @@ def phase_interface(config, phase): config.dataset_training[ref.source.partition(".")[0]].model_interface for ref in phase_sources(phase) } - explicit = getattr(phase, "model_interface", None) + explicit = phase.model_interface if phase.algorithm != "supervised" else None if explicit is not None: names.add(explicit) if not names: @@ -204,14 +206,16 @@ def _history_initialized_metadata(config, name, binding, source, storage): static = source.static_conditioning inputs = set(source.input_columns + (static.columns if static else [])) targets = set(config.model.interfaces[name].target_columns) - for key in ( - "column_data_types", - "n_classes", - "id_maps", - "selected_columns_statistics", + for key, values, original in ( + ("column_data_types", metadata.column_data_types, source.column_data_types), + ("n_classes", metadata.n_classes, source.n_classes), + ("id_maps", metadata.id_maps, source.id_maps), + ( + "selected_columns_statistics", + metadata.selected_columns_statistics, + source.selected_columns_statistics, + ), ): - values = getattr(metadata, key) - original = getattr(source, key) for column in inputs: if column in targets and column in values and column in original: if ( @@ -390,27 +394,27 @@ def resolve_training_run( extend_prediction_distributions(source, config.model) if not history_only: config.model = OwnedModelSpec.from_execution(source, config.model) - for field in ( - "context_length", - "training_objective", - "target_offset", - "forward_attention", + incoming = source.options + resolved_fields = dict(training) + inherited = {} + for field, actual in ( + ("context_length", incoming.context_length), + ("training_objective", incoming.training_objective), + ("target_offset", incoming.target_offset), + ("forward_attention", incoming.forward_attention), ): - actual = getattr(source.options, field) if ( history_only and field in {"training_objective", "target_offset"} and field in training.model_fields_set ): continue - if ( - field in training.model_fields_set - and getattr(training, field) != actual - ): + if field in training.model_fields_set and resolved_fields[field] != actual: raise ValueError( f"model.execution.{field} differs from initialize_from" ) - setattr(training, field, actual) + inherited[field] = actual + training = training.model_copy(update=inherited) source_dtypes = source.options.layer_type_dtypes if ( "layer_type_dtypes" in config.run.execution.precision.model_fields_set @@ -446,13 +450,10 @@ def resolve_training_run( training = TrainingRuntimeSettings.model_validate(training.model_dump()) config.runtime_settings = training if needs_policy_contracts: - for field in ( - "context_length", - "training_objective", - "target_offset", - "forward_attention", - ): - setattr(config.model.execution, field, getattr(training, field)) + config.model.execution.context_length = training.context_length + config.model.execution.training_objective = training.training_objective + config.model.execution.target_offset = training.target_offset + config.model.execution.forward_attention = training.forward_attention # Keep each physical metadata read shared by model assertions and source resolution. metadata_cache: dict[Path, DatasetMetadata] = {} @@ -473,7 +474,11 @@ def read_metadata(path): for dataset_name, dataset in config.dataset_training.items(): for part_name, part in dataset.parts.items(): ref = f"{dataset_name}.{part_name}" - path = getattr(part, "metadata_config_path", None) + path = ( + part.metadata_config_path + if not isinstance(part, PreferencePartSpecModel) + else None + ) if metadata_by_part is not None and isinstance(part, DatasetPartSpecModel): if ref in metadata_by_part: part_metadata[ref] = metadata_by_part[ref] @@ -503,10 +508,10 @@ def read_metadata(path): _resolve_phase_sources( config, phase, preferences if phase.algorithm == "dpo" else sequences ) - reference = getattr(phase, "reference", None) + reference = phase.reference_config if reference and reference.path: reference.path = str((root / reference.path).resolve()) - archive = getattr(phase, "archive_directory", None) + archive = phase.rollout_archive_directory if archive: phase.archive_directory = str((root / archive).resolve()) @@ -532,6 +537,7 @@ def read_metadata(path): for ref, value in part_metadata.items(): name = config.dataset_training[ref.partition(".")[0]].model_interface if name in metadata: + expected_fields, actual_fields = dict(metadata[name]), dict(value) for key in ( "column_data_types", "n_classes", @@ -539,7 +545,7 @@ def read_metadata(path): "selected_columns_statistics", "normalize_real_columns", ): - if getattr(metadata[name], key) != getattr(value, key): + if expected_fields[key] != actual_fields[key]: raise ValueError(f"{ref}: interface metadata differs at {key}") metadata[name] = value @@ -565,13 +571,14 @@ def read_metadata(path): interface = source.interface(name) if value is not None: candidate = _resolve_interface(name, spec, value, training) + expected_fields, actual_fields = dict(interface), dict(candidate) for key in ( "column_data_types", "id_maps", "selected_columns_statistics", "target_decoder_ids", ): - if getattr(candidate, key) != getattr(interface, key): + if expected_fields[key] != actual_fields[key]: raise ValueError( f"{name}: metadata differs from initialization at {key}" ) @@ -645,8 +652,8 @@ def read_metadata(path): f"{phase.name}: interface {name!r} requires a policy binding" ) for environment in ( - getattr(phase, "environment", None), - getattr(phase.evaluation, "environment", None), + phase.training_environment, + phase.evaluation_environment, ): if environment and environment.units != policy.units: raise ValueError( diff --git a/src/sequifier/evaluation/service.py b/src/sequifier/evaluation/service.py index 97c322ea..9c9783df 100644 --- a/src/sequifier/evaluation/service.py +++ b/src/sequifier/evaluation/service.py @@ -9,6 +9,7 @@ from typing import Any import torch +from pydantic import BaseModel from torch.nn.functional import one_hot from sequifier.helpers import construct_index_maps @@ -80,12 +81,11 @@ def __init__( def _baseline_definition(dataset: Any) -> str: """Identify inputs that affect raw baseline loss statistics.""" interface = dataset.interface - training = getattr(dataset.objective.config, "runtime_settings", None) + training = dataset.objective.config.runtime_settings def model_value(value: Any) -> Any: - dump = getattr(value, "model_dump", None) - if callable(dump): - return dump(mode="json") + if isinstance(value, BaseModel): + return value.model_dump(mode="json") if is_dataclass(value) and not isinstance(value, type): return asdict(value) return value @@ -93,13 +93,11 @@ def model_value(value: Any) -> Any: definition = { "objective": dataset.objective.name, "objective_options": { - "bert_spec": model_value(getattr(training, "bert_spec", None)), - "next_occurrence_config": model_value( - getattr(training, "next_occurrence_config", None) - ), + "bert_spec": model_value(training.bert_spec), + "next_occurrence_config": model_value(training.next_occurrence_config), }, "window_view": model_value(interface.window_view), - "prediction_length": getattr(interface.decoder, "prediction_length", None), + "prediction_length": interface.decoder.prediction_length, "target_columns": interface.target_columns, "target_column_types": interface.target_column_types, "target_decoder_ids": interface.target_decoder_ids, @@ -124,10 +122,12 @@ def _baseline_output( ).transpose(0, 1)[:, -decoded_length:] if interface.target_column_types[target] == "categorical": global_ids = values.to(torch.int64) - if target in getattr(interface, "categorical_hashing", {}): - codec = network.resolve_interface( - dataset.interface_name - ).decoder.hash_codecs[target] + if target in interface.categorical_hashing: + codec = ( + network.resolve_interface(dataset.interface_name) + .require_decoder() + .hash_codec(target) + ) codes = codec.encode(global_ids) auxiliary_logits[target] = tuple( one_hot(codes[..., index], width).to(torch.float32) @@ -193,9 +193,7 @@ def evaluate( target: torch.zeros( (), device=context.device, dtype=torch.int64 ) - for target in getattr( - dataset.interface, "categorical_hashing", {} - ) + for target in dataset.interface.categorical_hashing if target in dataset.interface.target_decoder_ids } accuracy_count = torch.zeros( diff --git a/src/sequifier/export/onnx.py b/src/sequifier/export/onnx.py index ab38d618..3045d3e2 100644 --- a/src/sequifier/export/onnx.py +++ b/src/sequifier/export/onnx.py @@ -312,7 +312,7 @@ def export( model = copy.deepcopy(model).cpu() for module in model.modules(): if isinstance(module, SelfAttention): - module._sequifier_export_attention = True + module.export_attention = True schema = ExecutionSchema.from_interface( interface, context_length, embedding=interface_name is None ) diff --git a/src/sequifier/helpers.py b/src/sequifier/helpers.py index 6c332dec..af35ad7d 100644 --- a/src/sequifier/helpers.py +++ b/src/sequifier/helpers.py @@ -592,8 +592,11 @@ def resolve_window_sampling_plan( @beartype def configured_window_stride(config: Any) -> Optional[int]: - """Read the optional stride from validated configs or legacy test doubles.""" - value = getattr(config, "window_stride", None) + """Read the optional stride from dataset or inference settings.""" + try: + value = config.window_stride + except AttributeError: + return None return value if isinstance(value, int) else None @@ -967,14 +970,10 @@ def get_torch_dtype(dtype_str: str) -> torch.dtype: "float16": torch.float16, "bfloat16": torch.bfloat16, "float64": torch.float64, + "float8_e4m3fn": torch.float8_e4m3fn, + "float8_e5m2": torch.float8_e5m2, } - # Add float8 support if available in this PyTorch version - if hasattr(torch, "float8_e4m3fn"): - dtype_map["float8_e4m3fn"] = torch.float8_e4m3fn - if hasattr(torch, "float8_e5m2"): - dtype_map["float8_e5m2"] = torch.float8_e5m2 - if dtype_str not in dtype_map: raise ValueError( f"dtype '{dtype_str}' not supported or available. Options: {list(dtype_map.keys())}" diff --git a/src/sequifier/hyperparameter_search.py b/src/sequifier/hyperparameter_search.py index ea79fe15..3faaada8 100644 --- a/src/sequifier/hyperparameter_search.py +++ b/src/sequifier/hyperparameter_search.py @@ -52,7 +52,7 @@ def _monitored_dataset(run_config: Any) -> tuple[str | None, int]: if dataset_count == 1: return None, dataset_count evaluation = run_config.training_plan.phases[-1].evaluation - monitor = getattr(evaluation, "monitor", None) + monitor = evaluation.monitor if evaluation is not None else None if monitor is None: raise ValueError( "Multi-dataset hyperparameter search requires evaluation.monitor" @@ -67,14 +67,18 @@ def _monitored_dataset(run_config: Any) -> tuple[str | None, int]: @beartype def create_sampler(config: Any) -> optuna.samplers.BaseSampler: - strategy = getattr(config, "method", "bayesian") - sampler_seed = getattr(config, "sampler_seed", None) + strategy = config.method + sampler_seed = config.sampler_seed if strategy == "random": return optuna.samplers.RandomSampler(seed=sampler_seed) if strategy == "grid": - if hasattr(optuna.samplers, "BruteForceSampler"): - return optuna.samplers.BruteForceSampler(seed=sampler_seed) - raise RuntimeError("Grid search requires Optuna >= 3.1 for BruteForceSampler.") + try: + from optuna.samplers import BruteForceSampler + except ImportError as error: + raise RuntimeError( + "Grid search requires Optuna >= 3.1 for BruteForceSampler." + ) from error + return BruteForceSampler(seed=sampler_seed) return optuna.samplers.TPESampler( seed=sampler_seed, multivariate=True, diff --git a/src/sequifier/infer.py b/src/sequifier/infer.py index 1d43dbca..9b33463d 100644 --- a/src/sequifier/infer.py +++ b/src/sequifier/infer.py @@ -30,6 +30,7 @@ ) from sequifier.io.output_paths import require_file_destinations, unique_paths from sequifier.io.preprocessing import denormalize_values +from sequifier.model.execution_schema import ExecutionSchema from sequifier.objectives import get_objective_class from sequifier.special_tokens import ( ONNX_CATEGORICAL_TARGET_CODECS_KEY, @@ -371,8 +372,8 @@ def _windowed_inference_batch_from_storage( config.window_view, configured_window_stride(config), ) - if getattr(config, "aligned_data_path", False): - steps = getattr(config, "prediction_alignment_steps", 1) + if config.aligned_data_path: + steps = config.prediction_alignment_steps spacing = config.prediction_length if spacing is None: raise ValueError("Aligned inference requires prediction_length") @@ -647,11 +648,14 @@ def infer_worker( for model_path in model_paths: target_decoder_ids = None route_args = dict(args_config) - for key in ("dataset", "part", "model_interface"): - value = getattr(config, key, None) + for key, value in ( + ("dataset", config.dataset), + ("part", config.part), + ("model_interface", config.model_interface), + ): if value is not None: route_args[key] = value - artifact = getattr(config, "_artifacts", {}).get(model_path) + artifact = config._artifacts.get(model_path) if model_path.lower().endswith(".pt"): if artifact is None: artifact = read_execution_artifact( @@ -726,9 +730,7 @@ def infer_worker( logger.info(f"Inferring for {model_id}") if config.task == "forecast": - if getattr(config, "output_path", None) is not None and getattr( - config, "output_probabilities", False - ): + if config.output_path is not None and config.output_probabilities: part_count = 1 if is_folder_input: total = len( @@ -1096,7 +1098,7 @@ def infer_generative( column_data_types: dict[str, torch.dtype], ): """Write generative predictions/probabilities for each dataset chunk.""" - if getattr(config, "rollout", None) is not None: + if config.rollout is not None: infer_numeric_rollout(config, inferer, model_id, dataset, column_data_types) return data_path = config.data_path @@ -1130,10 +1132,7 @@ def infer_generative( ) ) and configured_window_stride(config) is None - and ( - not getattr(config, "aligned_data_path", False) - or getattr(config, "prediction_alignment_steps", 1) == 1 - ) + and (not config.aligned_data_path or config.prediction_alignment_steps == 1) ): probs, preds = get_probs_preds_from_df( config, @@ -1599,6 +1598,8 @@ def verify_variable_order(data: pl.DataFrame) -> None: class Inferer: """Inference runtime for PT/ONNX sequifier models.""" + execution_schema: "ExecutionSchema | None" = None + @beartype def __init__( self, @@ -1738,27 +1739,9 @@ def __init__( self.infer_with_dropout, artifact=artifact, ) - route_model = getattr( - self.inference_model, - "transformer_model", - self.inference_model, - ) - self.target_decoder_ids = dict(route_model.target_decoder_ids) + self.target_decoder_ids = self.inference_model.target_decoder_ids if self.sample_from_distribution_columns: - from sequifier.model.decoders import ( - AutoregressiveTransformerDecoderBranch, - ) - - network = self.inference_model.network - network = getattr(network, "_orig_mod", network) - route = network.resolve_interface(self.inference_model.interface_name) - autoregressive_targets = { - target - for branch in route.decoder.branches.values() - if isinstance(branch, AutoregressiveTransformerDecoderBranch) - for target in branch.target_columns - } - sampled = autoregressive_targets.intersection( + sampled = self.inference_model.autoregressive_targets.intersection( self.sample_from_distribution_columns ) if sampled: @@ -2046,7 +2029,7 @@ def infer_pure( ) -> list[np.ndarray]: """Run one ONNX batch and flatten sequence-major outputs.""" metadata = metadata or {} - schema = getattr(self, "execution_schema", None) + schema = self.execution_schema if schema is not None: schema.validate( {key: torch.from_numpy(value) for key, value in x.items()}, diff --git a/src/sequifier/integration/callbacks.py b/src/sequifier/integration/callbacks.py index 7e2d63cb..71c2e954 100644 --- a/src/sequifier/integration/callbacks.py +++ b/src/sequifier/integration/callbacks.py @@ -1,6 +1,5 @@ from __future__ import annotations -import importlib from dataclasses import dataclass, replace from typing import Any, Protocol, cast, runtime_checkable @@ -13,6 +12,7 @@ TrainingEvent, ) from sequifier.integration.controls import TrainingDirective +from sequifier.integration.factories import load_factory from sequifier.integration.specifications import ( ExecutionRequirements, IntegrationSpec, @@ -64,13 +64,42 @@ class _RegisteredIntegration: state_policy: StatePolicy rank_policy: str + def __post_init__(self): + instance = self.instance + try: + self.handle = instance.handle + except AttributeError: + self.handle = None + try: + self.on_gradients_unscaled = instance.on_gradients_unscaled + except AttributeError: + self.on_gradients_unscaled = None + try: + self.capture_request = instance.capture_request + except AttributeError: + self.capture_request = None + try: + self.interventions = instance.interventions + except AttributeError: + self.interventions = None + try: + self.local_gradient_summary = instance.local_gradient_summary + except AttributeError: + self.local_gradient_summary = None + try: + self.state_dict = instance.state_dict + except AttributeError: + self.state_dict = None + try: + self.load_state_dict = instance.load_state_dict + except AttributeError: + self.load_state_dict = None + @beartype def _load_factory(path: str) -> Any: - module_name, attribute_name = path.split(":", 1) - module = importlib.import_module(module_name) try: - return getattr(module, attribute_name) + return load_factory(path) except AttributeError as error: raise ImportError(f"Integration factory {path!r} does not exist.") from error @@ -101,7 +130,10 @@ def __init__( continue factory = _load_factory(spec.factory) instance = factory(dict(spec.config)) - actual_id = getattr(instance, "integration_id", None) + try: + actual_id = instance.integration_id + except AttributeError: + actual_id = None if actual_id != spec.integration_id: raise ValueError( f"Factory {spec.factory!r} returned integration_id " @@ -121,7 +153,10 @@ def __init__( self._observers: list[_RegisteredIntegration] = [] self._controller: _RegisteredIntegration | None = None for item in registered: - integration_id = getattr(item.instance, "integration_id", None) + try: + integration_id = item.instance.integration_id + except AttributeError: + integration_id = None if not isinstance(integration_id, str) or not integration_id: raise ValueError( "Every integration must define a non-empty integration_id." @@ -129,9 +164,9 @@ def __init__( if integration_id in ids: raise ValueError(f"Duplicate integration_id: {integration_id!r}.") ids.add(integration_id) - if callable(getattr(item.instance, "handle", None)): + if callable(item.handle): self._observers.append(item) - if callable(getattr(item.instance, "on_gradients_unscaled", None)): + if callable(item.on_gradients_unscaled): if self._controller is not None: raise ValueError( "At most one TrainingController may be configured." @@ -168,7 +203,9 @@ def control_enabled(self) -> bool: @beartype def emit(self, event: TrainingEvent) -> None: for item in self._observers: - item.instance.handle(event) + handle = item.handle + assert callable(handle) + handle(event) @beartype def forward_trace(self, event: BatchPrepared) -> TraceContext | None: @@ -178,7 +215,7 @@ def forward_trace(self, event: BatchPrepared) -> TraceContext | None: if self._controller is not None and self._controller not in items: items.append(self._controller) for item in items: - request_provider = getattr(item.instance, "capture_request", None) + request_provider = item.capture_request if callable(request_provider): request = request_provider(event) if request is not None and not isinstance(request, CaptureRequest): @@ -187,7 +224,7 @@ def forward_trace(self, event: BatchPrepared) -> TraceContext | None: ) if request is not None: requests.append(request) - intervention_provider = getattr(item.instance, "interventions", None) + intervention_provider = item.interventions if callable(intervention_provider): bindings = intervention_provider(event) if not isinstance(bindings, tuple) or not all( @@ -226,7 +263,11 @@ def positions_equal(left: Any, right: Any) -> bool: clone=any(item.clone for item in requests), positions=positions, ) - trace_catalog = getattr(event.access.model, "trace_catalog", ()) + trace_catalog: Any + try: + trace_catalog = event.access.model.trace_catalog + except AttributeError: + trace_catalog = () available = {site.name for site in trace_catalog} requested = set(request.sites).union(binding.site for binding in interventions) unknown = requested.difference(available) @@ -246,9 +287,10 @@ def validate_execution( if self._controller is not None and self._controller not in items: items.append(self._controller) for item in items: - requirements = getattr( - item.instance, "execution_requirements", ExecutionRequirements() - ) + try: + requirements = item.instance.execution_requirements + except AttributeError: + requirements = ExecutionRequirements() if not isinstance(requirements, ExecutionRequirements): raise TypeError( f"Integration {item.instance.integration_id!r} has invalid " @@ -258,8 +300,8 @@ def validate_execution( requirements.activation_tracing or requirements.interventions or requirements.higher_order_gradients - or callable(getattr(item.instance, "capture_request", None)) - or callable(getattr(item.instance, "interventions", None)) + or callable(item.capture_request) + or callable(item.interventions) ) if needs_eager and torch_compile != "none": raise ValueError( @@ -289,9 +331,7 @@ def directive(self, event: GradientsUnscaled) -> TrainingDirective | None: directive: TrainingDirective | None = None controller_event = event if self._controller is not None: - summary_provider = getattr( - self._controller.instance, "local_gradient_summary", None - ) + summary_provider = self._controller.local_gradient_summary if callable(summary_provider): local_summary = summary_provider(event) if not isinstance(local_summary, dict) or not all( @@ -324,9 +364,10 @@ def directive(self, event: GradientsUnscaled) -> TrainingDirective | None: device=parameter.device, ) dist.all_reduce(values, op=dist.ReduceOp.SUM) - reduction = getattr( - self._controller.instance, "gradient_reduction", "mean" - ) + try: + reduction = self._controller.instance.gradient_reduction + except AttributeError: + reduction = "mean" if reduction == "mean": values /= self.world_size elif reduction != "sum": @@ -339,13 +380,14 @@ def directive(self, event: GradientsUnscaled) -> TrainingDirective | None: } controller_event = replace(event, reduced_summary=reduced_summary) if self._controller is not None and (not self.distributed or self.rank == 0): - directive = self._controller.instance.on_gradients_unscaled( - controller_event - ) - if directive is not None and not isinstance(directive, TrainingDirective): + decide = self._controller.on_gradients_unscaled + assert callable(decide) + decision = decide(controller_event) + if decision is not None and not isinstance(decision, TrainingDirective): raise TypeError( "TrainingController must return TrainingDirective or None." ) + directive = decision if self.distributed: if not dist.is_available() or not dist.is_initialized(): @@ -366,7 +408,7 @@ def state_dict(self) -> dict[str, dict[str, Any]]: items.append(self._controller) for item in items: integration_id = item.instance.integration_id - state_dict = getattr(item.instance, "state_dict", None) + state_dict = item.state_dict result[integration_id] = { "factory": item.factory, "state": state_dict() if callable(state_dict) else {}, @@ -442,7 +484,7 @@ def load_state_dict(self, saved: dict[str, Any] | None) -> None: raise ValueError( f"Checkpoint integration state for {integration_id!r} is invalid." ) - load_state_dict = getattr(item.instance, "load_state_dict", None) + load_state_dict = item.load_state_dict if callable(load_state_dict): load_state_dict(state) unknown = set(saved).difference(configured_ids) diff --git a/src/sequifier/integration/factories.py b/src/sequifier/integration/factories.py new file mode 100644 index 00000000..097572e3 --- /dev/null +++ b/src/sequifier/integration/factories.py @@ -0,0 +1,14 @@ +"""Resolve configured module-level factories at the plugin boundary.""" + +from importlib import import_module + + +def load_factory(path: str): + module_name, symbol = path.split(":", 1) + module = import_module(module_name) + try: + return vars(module)[symbol] + except KeyError as error: + raise AttributeError( + f"module {module_name!r} has no attribute {symbol!r}" + ) from error diff --git a/src/sequifier/io/depth_preprocess.py b/src/sequifier/io/depth_preprocess.py index 70f38d6e..6cfa722e 100644 --- a/src/sequifier/io/depth_preprocess.py +++ b/src/sequifier/io/depth_preprocess.py @@ -70,19 +70,17 @@ def _file_items( def source_rows(): if read_format == "csv": - if hasattr(pl.LazyFrame, "collect_batches"): - batches = ( - pl.scan_csv(filename) - .select(columns) - .collect_batches(chunk_size=1024) - ) - for batch in batches: - yield from batch.iter_rows(named=True) - else: + frame = pl.scan_csv(filename).select(columns) + try: + collect_batches = frame.collect_batches + except AttributeError: reader = pl.read_csv_batched(filename, columns=columns, batch_size=1024) while batches := reader.next_batches(1): for batch in batches: yield from batch.iter_rows(named=True) + else: + for batch in collect_batches(chunk_size=1024): + yield from batch.iter_rows(named=True) else: parquet = _parquet_file(filename) for batch in parquet.iter_batches( @@ -776,9 +774,7 @@ def preprocess_depth(owner, selected_columns): if Path(owner.preprocessing_data_path).is_dir() else [owner.preprocessing_data_path] ) - configured_curriculum_columns = curriculum_columns( - getattr(owner, "curriculum_column", None) - ) + configured_curriculum_columns = curriculum_columns(owner.curriculum_column) reserved_columns = { "sequenceId", "itemPosition", diff --git a/src/sequifier/io/episode_features.py b/src/sequifier/io/episode_features.py index c20c1555..5240fb1b 100644 --- a/src/sequifier/io/episode_features.py +++ b/src/sequifier/io/episode_features.py @@ -47,10 +47,13 @@ def values(self, episode, length, column): def fit_categorical_config(self) -> ResolvedPolicyBinding | PolicyFittingSpec: """Fit only declared raw categorical features, using the training episodes.""" - values = { - key: deepcopy(getattr(self.config, key)) - for key in ("id_maps", "observation_n_classes", "static_n_classes") - } + values = deepcopy( + { + "id_maps": self.config.id_maps, + "observation_n_classes": self.config.observation_n_classes, + "static_n_classes": self.config.static_n_classes, + } + ) for column in self.config.categorical_columns: if ( column in self.config.id_maps diff --git a/src/sequifier/io/iql_dataset.py b/src/sequifier/io/iql_dataset.py index cd600f3f..632af227 100644 --- a/src/sequifier/io/iql_dataset.py +++ b/src/sequifier/io/iql_dataset.py @@ -32,8 +32,13 @@ def __init__(self, decisions): source = episode_index transitions = episode.transitions if transitions is None or any( - getattr(transitions, field) is None - for field in ("rewards", "terminated", "truncated", "bootstrap_allowed") + values is None + for values in ( + transitions.rewards, + transitions.terminated, + transitions.truncated, + transitions.bootstrap_allowed, + ) ): raise ValueError( f"IQL source {source!r} episode {episode.episode_id!r} " diff --git a/src/sequifier/io/memory_budget.py b/src/sequifier/io/memory_budget.py index 62a1bace..68c2aae6 100644 --- a/src/sequifier/io/memory_budget.py +++ b/src/sequifier/io/memory_budget.py @@ -49,9 +49,13 @@ def resident_bytes(value: Any, seen: set[Any] | None = None) -> int: ) if isinstance(value, (tuple, list, set, frozenset)): return size + sum(resident_bytes(item, seen) for item in value) - if hasattr(value, "__dict__") and not isinstance(value, type): - return size + resident_bytes(vars(value), seen) - return size + if isinstance(value, type): + return size + try: + fields = vars(value) + except TypeError: + return size + return size + resident_bytes(fields, seen) def _parquet_size(path: Path, string_width: int) -> tuple[int, int]: diff --git a/src/sequifier/io/output_paths.py b/src/sequifier/io/output_paths.py index 7955888c..7e8ed753 100644 --- a/src/sequifier/io/output_paths.py +++ b/src/sequifier/io/output_paths.py @@ -76,7 +76,14 @@ def validate_training_destinations(config, checkpoint, protected, *, phase_offse directories.append( model_log_directory(config.project_root, config.model_name).resolve() ) - phases = getattr(config, "phases", getattr(config, "training_plan", ())) + from sequifier.rl.run_config import ResolvedPolicyRun + + if isinstance(config, ResolvedPolicyRun): + phases, resume = config.phases, config.resume + dataset_names = () + else: + phases, resume = config.training_plan, config.runtime_settings.resume + dataset_names = tuple(config.dataset_training) from sequifier.artifacts.phase import PhaseArtifacts generated = [] @@ -100,9 +107,9 @@ def validate_training_destinations(config, checkpoint, protected, *, phase_offse generated, protected, label="phase output", check_each_other=False ) directories.extend( - Path(phase.archive_directory).resolve() + Path(archive).resolve() for phase in phases - if getattr(phase, "archive_directory", None) + if (archive := phase.rollout_archive_directory) ) directories = unique_paths( directories, "training output directories must be distinct" @@ -119,9 +126,6 @@ def validate_training_destinations(config, checkpoint, protected, *, phase_offse ).resolve(): phase for index, phase in enumerate(phases, phase_offset) } - resume = getattr(config, "resume", None) or getattr( - getattr(config, "runtime_settings", None), "resume", None - ) snapshot_phase = snapshot_phases.get(checkpoint.parent) snapshot_name = snapshot_phase is not None and ( checkpoint.name.startswith(f"{config.model_name}-") @@ -153,7 +157,7 @@ def validate_training_destinations(config, checkpoint, protected, *, phase_offse if any(phase.algorithm == "supervised" for phase in phases): export_directory = (Path(config.project_root) / "models").resolve() names = [config.model_name] + [ - f"{config.model_name}-{name}" for name in config.dataset_training + f"{config.model_name}-{name}" for name in dataset_names ] pattern = re.compile( "(?:" @@ -171,12 +175,13 @@ def validate_training_destinations(config, checkpoint, protected, *, phase_offse def resolve_training_output_paths(config): root = Path(config.project_root) - defaults = { - "model_path": f"models/{config.model_name}.pt", - "report_path": f"outputs/training/{config.model_name}/report.json", - "phase_directory": f"outputs/training/{config.model_name}/phases", - } - for name, default in defaults.items(): - setattr( - config, name, str((root / (getattr(config, name, "") or default)).resolve()) - ) + output_dir = Path("outputs/training") / config.model_name + config.model_path = str( + (root / (config.model_path or f"models/{config.model_name}.pt")).resolve() + ) + config.report_path = str( + (root / (config.report_path or output_dir / "report.json")).resolve() + ) + config.phase_directory = str( + (root / (config.phase_directory or output_dir / "phases")).resolve() + ) diff --git a/src/sequifier/io/policy_materialization.py b/src/sequifier/io/policy_materialization.py index f8b6d4f6..0601c8e6 100644 --- a/src/sequifier/io/policy_materialization.py +++ b/src/sequifier/io/policy_materialization.py @@ -6,6 +6,13 @@ from sequifier.io.batch import SequifierInputs from sequifier.model.execution_schema import ExecutionSchema +_HISTORY_DTYPES = { + "bool": torch.bool, + "float32": torch.float32, + "int64": torch.int64, + "uint8": torch.uint8, +} + def encode_value(value, column, config, *, force_raw=False): # Numeric/image payloads have no categorical codec. Avoid recursively @@ -46,7 +53,7 @@ def materialize_history(config, observations, previous_actions, static): features, metadata = {}, {} for descriptor in schema.inputs: key = descriptor.key - dtype = getattr(torch, descriptor.dtype) + dtype = _HISTORY_DTYPES[descriptor.dtype] if key == "attention_valid_mask": metadata[key] = torch.tensor([[False] * padding + [True] * len(rows)]) continue diff --git a/src/sequifier/io/preference_dataset.py b/src/sequifier/io/preference_dataset.py index 4b363d90..a1346057 100644 --- a/src/sequifier/io/preference_dataset.py +++ b/src/sequifier/io/preference_dataset.py @@ -172,8 +172,10 @@ def validate_features(self): ) contexts[pair.context_id] = context_key left, right = initial - for mapping in ("inputs", "metadata"): - lmap, rmap = getattr(left, mapping), getattr(right, mapping) + for mapping, lmap, rmap in ( + ("inputs", left.inputs, right.inputs), + ("metadata", left.metadata, right.metadata), + ): if lmap.keys() != rmap.keys() or any( not torch.equal(lmap[k], rmap[k]) for k in lmap ): diff --git a/src/sequifier/io/pt_payload.py b/src/sequifier/io/pt_payload.py index cd8658ee..9aa1299d 100644 --- a/src/sequifier/io/pt_payload.py +++ b/src/sequifier/io/pt_payload.py @@ -126,21 +126,14 @@ def validate(self, layouts=None, *, n_classes=None): if first.ndim < 2: raise ValueError("Stored tensors require batch and temporal axes") n, width = first.shape[:2] - for key in ( - "sequence_ids", - "subsequence_ids", - "start_item_positions", - "left_pad_lengths", + for key, value in ( + ("sequence_ids", self.sequence_ids), + ("subsequence_ids", self.subsequence_ids), + ("start_item_positions", self.start_item_positions), + ("left_pad_lengths", self.left_pad_lengths), + ("split_start_item_positions", self.split_start_item_positions), + ("split_end_item_positions", self.split_end_item_positions), ): - value = getattr(self, key) - if ( - not isinstance(value, Tensor) - or value.dtype != torch.int64 - or tuple(value.shape) != (n,) - ): - raise ValueError(f"{key} must be int64 [{n}]") - for key in ("split_start_item_positions", "split_end_item_positions"): - value = getattr(self, key) if ( not isinstance(value, Tensor) or value.dtype != torch.int64 @@ -273,17 +266,12 @@ def save_pt_payload(batch: StoredTensorBatch, path, *, layouts=None, n_classes=N "sample_positions": batch.sample_positions, "curriculum_columns": list(batch.curriculum_columns), }, - **{ - key: getattr(batch, key) - for key in ( - "sequence_ids", - "subsequence_ids", - "start_item_positions", - "left_pad_lengths", - "split_start_item_positions", - "split_end_item_positions", - ) - }, + "sequence_ids": batch.sequence_ids, + "subsequence_ids": batch.subsequence_ids, + "start_item_positions": batch.start_item_positions, + "left_pad_lengths": batch.left_pad_lengths, + "split_start_item_positions": batch.split_start_item_positions, + "split_end_item_positions": batch.split_end_item_positions, }, path, ) @@ -323,13 +311,10 @@ def concatenate_pt_batches(batches: list[StoredTensorBatch]) -> StoredTensorBatc split_end_item_positions=torch.cat( [b.split_end_item_positions for b in batches] ), - **{ - key: torch.cat([getattr(b, key) for b in batches]) - for key in ( - "sequence_ids", - "subsequence_ids", - "start_item_positions", - "left_pad_lengths", - ) - }, + sequence_ids=torch.cat([batch.sequence_ids for batch in batches]), + subsequence_ids=torch.cat([batch.subsequence_ids for batch in batches]), + start_item_positions=torch.cat( + [batch.start_item_positions for batch in batches] + ), + left_pad_lengths=torch.cat([batch.left_pad_lengths for batch in batches]), ) diff --git a/src/sequifier/io/sample_order.py b/src/sequifier/io/sample_order.py index 169b6f85..e21b1d84 100644 --- a/src/sequifier/io/sample_order.py +++ b/src/sequifier/io/sample_order.py @@ -1,6 +1,7 @@ """Deterministic sample ordering for ordinary and curriculum datasets.""" from dataclasses import dataclass +from typing import Any import polars as pl import torch @@ -127,26 +128,35 @@ def indices_for_epoch(self, *, seed: int, epoch: int, shuffle: bool) -> Tensor: return indices -def configured_file_order(config: object) -> str: - """Read the validated per-part file-order policy, tolerating test doubles.""" - value = getattr(getattr(config, "part", None), "file_order", "shuffled") +def configured_file_order(config: Any) -> str: + """Read the optional per-part file-order policy.""" + try: + value = config.part.file_order + except AttributeError: + return "shuffled" return value if value in {"shuffled", "name"} else "shuffled" -def curriculum_training_enabled(config: object) -> bool: +def curriculum_training_enabled(config: Any) -> bool: """Return the validated training-plan curriculum switch.""" - value = getattr(config, "curriculum_training", False) + try: + value = config.curriculum_training + except AttributeError: + return False return value if isinstance(value, bool) else False -def configured_curriculum_column(config: object) -> str | None: +def configured_curriculum_column(config: Any) -> str | None: """Read the validated training-plan curriculum column.""" - value = getattr(config, "curriculum_column", None) + try: + value = config.curriculum_column + except AttributeError: + return None return value if isinstance(value, str) else None def curriculum_sample_positions( - config: object, + config: Any, sample_positions: Tensor | None, source: str, stored_columns: tuple[str, ...] = (), @@ -172,7 +182,7 @@ def curriculum_sample_positions( def curriculum_positions_from_parquet( - config: object, data: pl.DataFrame, source: str + config: Any, data: pl.DataFrame, source: str ) -> Tensor | None: """Read curriculum values only when curriculum training is enabled.""" if not curriculum_training_enabled(config): @@ -183,7 +193,7 @@ def curriculum_positions_from_parquet( ) -def validate_folder_curriculum(config: object, metadata: dict, source: str) -> None: +def validate_folder_curriculum(config: Any, metadata: dict, source: str) -> None: """Fail before iteration when a folder lacks curriculum-capable metadata.""" if not curriculum_training_enabled(config): return diff --git a/src/sequifier/io/sequence.py b/src/sequifier/io/sequence.py index 19d35642..84f0dbdf 100644 --- a/src/sequifier/io/sequence.py +++ b/src/sequifier/io/sequence.py @@ -49,9 +49,7 @@ class TransitionColumns(BaseModel): def at(self, index: int) -> dict: """Read a transition without inventing absent capabilities.""" return { - name: values[index] if values is not None else None - for name in type(self).model_fields - for values in [getattr(self, name)] + name: values[index] if values is not None else None for name, values in self } @@ -96,8 +94,7 @@ def validate_sequence(self): transitions = self.transitions if transitions is None: return self - for field in type(transitions).model_fields: - values = getattr(transitions, field) + for field, values in transitions: if values is not None and len(values) != length: raise ValueError(f"transition {field} must match sequence length") for index in range(length): diff --git a/src/sequifier/io/sequifier_dataset_from_file.py b/src/sequifier/io/sequifier_dataset_from_file.py index 8ee341d8..6c7e8575 100644 --- a/src/sequifier/io/sequifier_dataset_from_file.py +++ b/src/sequifier/io/sequifier_dataset_from_file.py @@ -89,7 +89,10 @@ def __init__(self, data_path: str, config: Any, shuffle: bool = True): if config.read_format == "pt": from sequifier.io.pt_payload import load_pt_payload - part = getattr(config, "part", None) + try: + part = config.part + except AttributeError: + part = None metadata = part.metadata if part is not None else None payload = load_pt_payload( data_path, diff --git a/src/sequifier/io/sequifier_dataset_from_folder_pt.py b/src/sequifier/io/sequifier_dataset_from_folder_pt.py index 9d9f717d..c051c597 100644 --- a/src/sequifier/io/sequifier_dataset_from_folder_pt.py +++ b/src/sequifier/io/sequifier_dataset_from_folder_pt.py @@ -35,9 +35,7 @@ def __init__(self, data_path: str, config: Any, shuffle: bool = True): self.depth_layouts = DepthLayoutRegistryModel.model_validate( metadata.get("depth_layouts", {}) ) - selected_layouts: DepthLayoutRegistryModel | None = getattr( - config, "depth_layouts", None - ) + selected_layouts: DepthLayoutRegistryModel | None = config.depth_layouts if selected_layouts is None: selected_layouts = DepthLayoutRegistryModel() if self.depth_layouts.compatibility_signature( diff --git a/src/sequifier/io/sequifier_dataset_from_folder_pt_lazy.py b/src/sequifier/io/sequifier_dataset_from_folder_pt_lazy.py index eb5fa728..cadb557b 100644 --- a/src/sequifier/io/sequifier_dataset_from_folder_pt_lazy.py +++ b/src/sequifier/io/sequifier_dataset_from_folder_pt_lazy.py @@ -33,9 +33,7 @@ def __init__(self, data_path: str, config: Any, shuffle: bool = True): self.depth_layouts = DepthLayoutRegistryModel.model_validate( metadata.get("depth_layouts", {}) ) - selected_layouts: DepthLayoutRegistryModel | None = getattr( - config, "depth_layouts", None - ) + selected_layouts: DepthLayoutRegistryModel | None = config.depth_layouts if selected_layouts is None: selected_layouts = DepthLayoutRegistryModel() if self.depth_layouts.compatibility_signature( diff --git a/src/sequifier/io/static_features.py b/src/sequifier/io/static_features.py index 191655b6..3c094000 100644 --- a/src/sequifier/io/static_features.py +++ b/src/sequifier/io/static_features.py @@ -54,10 +54,16 @@ def _load( def table_for_config(config: Any) -> StaticFeatureTable | None: - columns: tuple[str, ...] = tuple(getattr(config, "static_columns", ())) + try: + columns = tuple(config.static_columns) + except AttributeError: + return None if not columns: return None - path = getattr(config, "static_features_path", None) + try: + path = config.static_features_path + except AttributeError: + path = None if path is None: raise ValueError("Static conditioning requires static_features_path") dtypes = tuple(config.column_data_types[column] for column in columns) diff --git a/src/sequifier/io/tensors.py b/src/sequifier/io/tensors.py index 34840d40..42832c12 100644 --- a/src/sequifier/io/tensors.py +++ b/src/sequifier/io/tensors.py @@ -16,7 +16,7 @@ def _map_tensors(value, operation): if isinstance(value, TensorRecord): return type(value)( **{ - field.name: _map_tensors(getattr(value, field.name), operation) + field.name: _map_tensors(vars(value)[field.name], operation) for field in fields(value) } ) @@ -67,7 +67,7 @@ def collate_tensor_records(values): return type(first)( **{ field.name: collate_tensor_records( - [getattr(value, field.name) for value in values] + [vars(value)[field.name] for value in values] ) for field in fields(first) } diff --git a/src/sequifier/io/values.py b/src/sequifier/io/values.py index 80bec9c0..07550621 100644 --- a/src/sequifier/io/values.py +++ b/src/sequifier/io/values.py @@ -27,8 +27,12 @@ def numeric_scalar(value, label: str) -> float: def portable_value(value): """Own JSON-compatible copies of array, tensor, and nested input payloads.""" - if hasattr(value, "tolist"): - value = value.tolist() + try: + tolist = value.tolist + except AttributeError: + pass + else: + value = tolist() if isinstance(value, dict): return {key: portable_value(item) for key, item in value.items()} if isinstance(value, (tuple, list)): diff --git a/src/sequifier/model/decoders.py b/src/sequifier/model/decoders.py index a4f0aa5f..1aeb0025 100644 --- a/src/sequifier/model/decoders.py +++ b/src/sequifier/model/decoders.py @@ -30,8 +30,6 @@ def __init__(self, input_dim: int, codec: CategoricalHashCodec): self.layers = nn.ModuleList( nn.Linear(input_dim, width) for width in codec.widths ) - for layer in self.layers: - layer._sequifier_decoder_output = True # type: ignore[attr-defined] def forward(self, hidden: Tensor) -> Tensor: return torch.cat( @@ -99,8 +97,6 @@ def __init__( if target_column in self.hash_codecs else nn.Linear(layer_input_dim, output_dim) ) - if isinstance(output_layer, nn.Linear): - output_layer._sequifier_decoder_output = True # type: ignore[attr-defined] self.output_layers[target_column] = output_layer @staticmethod @@ -342,8 +338,6 @@ def __init__( output = nn.Linear(self.width, 1) else: raise ValueError(f"Unknown target column type {kind!r}.") - if isinstance(output, nn.Linear): - output._sequifier_decoder_output = True # type: ignore[attr-defined] if embedding is not None: self.value_embeddings[target] = embedding self.output_layers[target] = output @@ -374,7 +368,7 @@ def _embed_value( layer = cast(nn.Linear, module) values = values.to(dtype=layer.weight.dtype).unsqueeze(-1) return layer(values) - lookup = getattr(self, self._lookup_buffer_names[target]) + lookup = self.get_buffer(self._lookup_buffer_names[target]) global_ids = values.to(torch.long) if valid_mask is None: effective_valid_mask = torch.ones_like(global_ids, dtype=torch.bool) @@ -694,6 +688,9 @@ def decode(self, target_column: str, x: Tensor) -> Tensor: ) return self.resolve_target(target_column, branch.decode(target_column, x)) + def hash_codec(self, target_column: str) -> CategoricalHashCodec: + return cast(CategoricalHashCodec, self.hash_codecs[target_column]) + def resolve_target(self, target_column: str, raw: Tensor) -> Tensor: if target_column in self.hash_codecs: codec = self.hash_codecs[target_column] @@ -846,7 +843,7 @@ def resolve_decoding_plan(hparams: Any) -> DecodingPlan: f"target_columns: {sorted(missing_columns)}" ) if branch_config.type == "autoregressive_transformer": - if getattr(hparams, "training_objective", None) == "bert": + if hparams.training_objective == "bert": mask_targets = [ target for target in target_columns @@ -919,7 +916,7 @@ def build_target_decoding( target: CategoricalHashCodec( config, hparams.n_classes[target], hparams.target_decoder_ids[target] ) - for target, config in getattr(hparams, "categorical_hashing", {}).items() + for target, config in hparams.categorical_hashing.items() if target in hparams.target_decoder_ids } @@ -956,9 +953,13 @@ def build_target_decoding( n_classes=decoder_n_classes, input_dim=input_dim, hidden_dims=DECODER_HIDDEN_DIMS[branch_config.type](branch_config), - activation=getattr(branch_config, "activation", "relu"), - dropout=getattr(branch_config, "dropout", 0.0), - hidden_weight_l2=getattr(branch_config, "hidden_weight_l2", 0.0), + activation=branch_config.activation + if branch_config.type == "mlp" + else "relu", + dropout=branch_config.dropout if branch_config.type == "mlp" else 0.0, + hidden_weight_l2=branch_config.hidden_weight_l2 + if branch_config.type == "mlp" + else 0.0, hash_codecs={ target: hash_codecs[target] for target in branch.target_columns diff --git a/src/sequifier/model/depth_ingestion.py b/src/sequifier/model/depth_ingestion.py index dea19d7d..1b064b0f 100644 --- a/src/sequifier/model/depth_ingestion.py +++ b/src/sequifier/model/depth_ingestion.py @@ -81,9 +81,6 @@ def __init__( else None ) self.drop = nn.Dropout(config.dropout) - # Ownership affects new optimizer IDs without changing legacy attention ingestions. - for parameter in self.parameters(): - parameter._sequifier_depth_parameter = True def forward(self, src, metadata): valid = metadata[self.mask_key] diff --git a/src/sequifier/model/execution_schema.py b/src/sequifier/model/execution_schema.py index 34006263..037c43c6 100644 --- a/src/sequifier/model/execution_schema.py +++ b/src/sequifier/model/execution_schema.py @@ -45,9 +45,7 @@ def __post_init__(self): def from_interface( cls, interface, context_length, *, embedding=False, embedding_width=None ): - layouts: DepthLayoutRegistryModel | None = getattr( - interface, "depth_layouts", None - ) + layouts: DepthLayoutRegistryModel | None = interface.depth_layouts if layouts is None: layouts = DepthLayoutRegistryModel() inputs = [] @@ -69,7 +67,7 @@ def from_interface( ) static_columns = ( interface.static_conditioning.columns - if getattr(interface, "static_conditioning", None) is not None + if interface.static_conditioning is not None else [] ) for index, column in enumerate(static_columns): @@ -115,7 +113,7 @@ def from_interface( ("batch", context_length, capacity), ) ) - if getattr(interface, "preconditioning_image_encoder", None) is not None: + if interface.preconditioning_image_encoder is not None: source = interface.preconditioning_image inputs.append( InputDescriptor( diff --git a/src/sequifier/model/factory.py b/src/sequifier/model/factory.py index ac0c3dd4..a0b5e0e4 100644 --- a/src/sequifier/model/factory.py +++ b/src/sequifier/model/factory.py @@ -2,22 +2,24 @@ import logging from dataclasses import dataclass -from typing import Any +from typing import Any, cast import torch from torch import nn +from sequifier.config.initialization_config import ModelInitializationConfig from sequifier.helpers import get_torch_dtype from sequifier.model.backbone import TransformerBackbone from sequifier.model.decoders import build_target_decoding from sequifier.model.execution import ModelExecution from sequifier.model.ingestion_compiler import compile_feature_ingestion +from sequifier.model.ingestions import BaseFeatureIngestion, IngestionOutputProjection from sequifier.model.initialization import ( initialize_model_weights, initialize_owned_ingestion, ) from sequifier.model.initialization_seed import derived_seed, isolated_initialization -from sequifier.model.layers import RMSNorm +from sequifier.model.layers import RMSNorm, SequifierEncoderLayer from sequifier.model.network import ComposableTransformerNetwork, ModelInterfaceModule from sequifier.model.static_conditioning import StaticConditionEncoder from sequifier.objectives import BERTObjective, CausalObjective @@ -84,7 +86,7 @@ def validate_hash_contracts(network: ComposableTransformerNetwork, config: Any) f"Resolved hash contract for {column!r} differs from its configuration" ) if column in interface.target_decoder_ids: - codec = route.decoder.hash_codecs[column] + codec = route.require_decoder().hash_codec(column) if codec.contract() != expected: raise ValueError( f"Persisted hash codec for {column!r} differs from its resolved contract" @@ -160,29 +162,35 @@ def _apply_layer_dtypes(network: nn.Module, config: Any) -> None: module.to(dtype=get_torch_dtype(layer_config["norm"])) +@dataclass(frozen=True) +class _ComponentInitialization: + module: nn.Module + policy: ModelInitializationConfig + seed: int | None = None + + @beartype def _initialize_components( - components: dict[str, tuple[nn.Module, Any]], logger: Any + components: dict[str, _ComponentInitialization], logger: Any ) -> None: configured_targets: set[tuple[str, str]] = set() matched_targets: set[tuple[str, str]] = set() - for module, initialization in components.values(): + for component in components.values(): + module, initialization = component.module, component.policy configured_targets.update(initialization.configured_targets()) - initializer = ( - initialize_owned_ingestion - if hasattr(module, "_sequifier_initialization_config") - else initialize_model_weights - ) - with isolated_initialization( - getattr(module, "_sequifier_initialization_seed", None) - ): - matched_targets.update( - initializer( - module, - initialization, - warn_unmatched=False, + with isolated_initialization(component.seed): + if ( + isinstance(module, BaseFeatureIngestion) + and module.initialization_config is not None + ): + matched = initialize_owned_ingestion( + module, initialization, warn_unmatched=False ) - ) + else: + matched = initialize_model_weights( + module, initialization, warn_unmatched=False + ) + matched_targets.update(matched) unmatched = configured_targets.difference(matched_targets) if unmatched: targets = ", ".join(f"{group}.{kind}" for group, kind in sorted(unmatched)) @@ -196,7 +204,7 @@ def _decoder_metadata(view: Any) -> InterfaceRuntimeMetadata: for column in view.target_columns if view.target_column_types[column] == "categorical" } - persisted_decoder_ids = getattr(view, "target_decoder_ids", None) + persisted_decoder_ids = view.target_decoder_ids if persisted_decoder_ids: if set(persisted_decoder_ids) != categorical_targets: raise ValueError( @@ -269,13 +277,15 @@ def _build_composable_network( backbone_seed = config.model.backbone.initialization_seed with isolated_initialization(derived_seed(backbone_seed, (), "constructor")): backbone = TransformerBackbone(config.model.backbone.architecture) - backbone._sequifier_initialization_seed = derived_seed( - backbone_seed, (), "initializer" + backbone_initialization = _ComponentInitialization( + backbone, + config.model.backbone.initialization, + derived_seed(backbone_seed, (), "initializer"), ) routes = {} runtime_metadata = {} - route_initialization_components: dict[str, tuple[nn.Module, Any]] = {} + route_initialization_components: dict[str, _ComponentInitialization] = {} for name, interface in interfaces.items(): view = interface_build_view(config, interface) load_pretrained = ( @@ -283,9 +293,9 @@ def _build_composable_network( ) built_ingestion = compile_feature_ingestion( hparams=view, - direct_real_dtype_provider=lambda: backbone.layers[ - 0 - ].ff.get_first_layer_dtype(), + direct_real_dtype_provider=lambda: cast( + SequifierEncoderLayer, backbone.layers[0] + ).ff.get_first_layer_dtype(), max_concat_tensors=config.options.max_concat_tensors, load_pretrained=load_pretrained, ) @@ -313,12 +323,10 @@ def _build_composable_network( adapter: nn.Module = ( nn.Identity() if built_ingestion.width == backbone.input_dim - else nn.Linear(built_ingestion.width, backbone.input_dim) + else IngestionOutputProjection( + built_ingestion.width, backbone.input_dim + ) ) - adapter._sequifier_initialization_seed = derived_seed( - interface.ingestion.initialization_seed, ("adapter",), "initializer" - ) - adapter._sequifier_layer_group = "ingestion.output_projection" # type: ignore[attr-defined] decoder = None if encoder_interface is None: metadata = _decoder_metadata(view) @@ -354,9 +362,6 @@ def _build_composable_network( interface.static_conditioning.total_embedding_dim, interface.static_conditioning.feature_embedding_dims, ) - static_encoder._sequifier_initialization_seed = derived_seed( - static_seed, ("static_conditioning",), "initializer" - ) image_prefix_encoder = None if interface.preconditioning_image_encoder is not None: from sequifier.model.images import ImagePrefixEncoder @@ -389,47 +394,64 @@ def _build_composable_network( static_condition_encoder=static_encoder, preconditioning_image_encoder=image_prefix_encoder, prefix_projection=prefix_projection, + depth_layouts=interface.depth_layouts, + input_columns=tuple(interface.input_columns), + input_n_classes=interface.n_classes, ) from sequifier.model.prediction_distribution import PredictionScales distributions = interface.prediction_distributions if encoder_interface is None and distributions: routes[name].prediction_scales = PredictionScales(distributions).to(device) - routes[name].depth_layouts = interface.depth_layouts - routes[name].input_columns = tuple(interface.input_columns) - routes[name].input_n_classes = interface.n_classes route_initialization_components.update( { - f"interfaces.{name}.ingestion": ( + f"interfaces.{name}.ingestion": _ComponentInitialization( built_ingestion.module, interface.ingestion.initialization, + built_ingestion.module.initialization_seed, ), - f"interfaces.{name}.ingestion_adapter": ( + f"interfaces.{name}.ingestion_adapter": _ComponentInitialization( adapter, interface.ingestion.initialization, + derived_seed( + interface.ingestion.initialization_seed, + ("adapter",), + "initializer", + ), ), } ) if decoder is not None: route_initialization_components[f"interfaces.{name}.decoder"] = ( - decoder, - interface.decoder.initialization, + _ComponentInitialization( + decoder, + interface.decoder.initialization, + ) ) - if routes[name].static_condition_encoder is not None: + if static_encoder is not None: route_initialization_components[ f"interfaces.{name}.static_condition_encoder" - ] = ( - routes[name].static_condition_encoder, + ] = _ComponentInitialization( + static_encoder, interface.ingestion.initialization, + derived_seed( + interface.ingestion.initialization_seed, + ("static_conditioning",), + "initializer", + ), ) if image_prefix_encoder is not None: route_initialization_components[ f"interfaces.{name}.preconditioning_image_encoder" - ] = (image_prefix_encoder, interface.ingestion.initialization) + ] = _ComponentInitialization( + image_prefix_encoder, interface.ingestion.initialization + ) if prefix_projection is not None: route_initialization_components[f"interfaces.{name}.prefix_projection"] = ( - prefix_projection, - interface.ingestion.initialization, + _ComponentInitialization( + prefix_projection, + interface.ingestion.initialization, + ) ) attention_objective = ( @@ -460,10 +482,7 @@ def _build_composable_network( f"interfaces.{name}.ingestion_adapter" ] ), - "backbone": ( - backbone, - config.model.backbone.initialization, - ), + "backbone": backbone_initialization, } for component in ( "decoder", @@ -478,10 +497,7 @@ def _build_composable_network( ] else: initialization_components = { - "backbone": ( - backbone, - config.model.backbone.initialization, - ), + "backbone": backbone_initialization, **route_initialization_components, } _initialize_components(initialization_components, logger) diff --git a/src/sequifier/model/hash_codec.py b/src/sequifier/model/hash_codec.py index 2e15f6b8..6f9ce418 100644 --- a/src/sequifier/model/hash_codec.py +++ b/src/sequifier/model/hash_codec.py @@ -78,7 +78,7 @@ def __init__(self, config, n_classes: int, decoder_ids: list[int] | None = None) self.kind = config.type self.num_buckets = config.num_buckets self.num_hashes = config.num_hashes - self.seed = getattr(config, "seed", 0) + self.seed = config.seed if config.type == "multi_hash" else 0 self.version = ContractVersions.HASH_CODEC self.widths = ( [config.num_buckets] * config.num_hashes diff --git a/src/sequifier/model/history_transfer.py b/src/sequifier/model/history_transfer.py index e5976681..3ab47a9e 100644 --- a/src/sequifier/model/history_transfer.py +++ b/src/sequifier/model/history_transfer.py @@ -62,14 +62,13 @@ def _history_contract(config, interface_name): "special_token_ids": interface.special_token_ids, "input_metadata": { column: { - key: getattr(interface, key).get(column) - for key in ( - "column_data_types", - "n_classes", - "id_maps", - "categorical_hashing", - "selected_columns_statistics", - ) + "column_data_types": interface.column_data_types.get(column), + "n_classes": interface.n_classes.get(column), + "id_maps": interface.id_maps.get(column), + "categorical_hashing": interface.categorical_hashing.get(column), + "selected_columns_statistics": interface.selected_columns_statistics.get( + column + ), } for column in columns }, @@ -97,8 +96,14 @@ def history_components(network, interface_name): "backbone": network.backbone, **{ f"interfaces.{interface_name}.{name}": component - for name in _HISTORY_COMPONENTS - if (component := getattr(route, name)) is not None + for name, component in ( + ("ingestion", route.ingestion), + ("ingestion_adapter", route.ingestion_adapter), + ("static_condition_encoder", route.static_condition_encoder), + ("preconditioning_image_encoder", route.preconditioning_image_encoder), + ("prefix_projection", route.prefix_projection), + ) + if component is not None }, } diff --git a/src/sequifier/model/ingestion_compiler.py b/src/sequifier/model/ingestion_compiler.py index d661c19a..54acd063 100644 --- a/src/sequifier/model/ingestion_compiler.py +++ b/src/sequifier/model/ingestion_compiler.py @@ -301,7 +301,7 @@ def resolve_ingestion_plan(hparams: Any) -> IngestionPlan: model = hparams.model ingestion_spec = model.ingestion architecture = model.backbone.architecture - objective_name = getattr(hparams, "training_objective", None) + objective_name = hparams.training_objective uses_causal_attention = ( get_objective_class(objective_name).uses_causal_attention if objective_name is not None @@ -312,7 +312,7 @@ def resolve_ingestion_plan(hparams: Any) -> IngestionPlan: ) from sequifier.config.depth_layout import DepthLayoutRegistryModel - layouts: DepthLayoutRegistryModel | None = getattr(hparams, "depth_layouts", None) + layouts: DepthLayoutRegistryModel | None = hparams.depth_layouts if layouts is None: layouts = DepthLayoutRegistryModel() @@ -583,7 +583,7 @@ def _build_image_handler(branch, context): def _depth_columns(hparams, config, allow_default_columns): from sequifier.config.depth_layout import DepthLayoutRegistryModel - layouts: DepthLayoutRegistryModel | None = getattr(hparams, "depth_layouts", None) + layouts: DepthLayoutRegistryModel | None = hparams.depth_layouts if layouts is None: layouts = DepthLayoutRegistryModel() if config.layout not in layouts: @@ -712,10 +712,8 @@ def build(branch, inherited_seed=None, path=()): ) else: module = INGESTION_HANDLERS[branch.config.type].build(branch, context) - module._sequifier_initialization_config = branch.config - module._sequifier_initialization_seed = derived_seed( - inherited_seed, path, "initializer" - ) + module.initialization_config = branch.config + module.initialization_seed = derived_seed(inherited_seed, path, "initializer") return module built_branches = { @@ -739,8 +737,8 @@ def build(branch, inherited_seed=None, path=()): merge_dim=plan.transformer_input_width, merge_softmax=cast(str, plan.merge_softmax), ) - module._sequifier_initialization_config = root_config - module._sequifier_initialization_seed = derived_seed(root_seed, (), "initializer") + module.initialization_config = root_config + module.initialization_seed = derived_seed(root_seed, (), "initializer") return BuiltIngestion(module, plan.transformer_input_width) diff --git a/src/sequifier/model/ingestions.py b/src/sequifier/model/ingestions.py index a8fd6600..4dd61d86 100644 --- a/src/sequifier/model/ingestions.py +++ b/src/sequifier/model/ingestions.py @@ -1,5 +1,5 @@ import math -from collections.abc import Callable +from collections.abc import Callable, Iterable from dataclasses import dataclass from itertools import product from typing import Any, Optional, cast @@ -10,6 +10,8 @@ from torch.nn import ModuleDict from sequifier.accurate_softmax import accurate_softmax +from sequifier.config.components import IngestionComponentBase +from sequifier.config.identifiers import validate_module_dict_key from sequifier.model.dtypes import ( cast_floating_to_dtype, cast_floating_to_module_dtype, @@ -48,18 +50,6 @@ def embedding_safe_indices(indices: Tensor) -> Tensor: return indices.to(dtype=target_dtype) -@beartype -def _validate_module_dict_key(key: str, usage: str) -> None: - if key == "": - raise ValueError(f"{usage} cannot be empty") - if "." in key: - raise ValueError(f"{usage} cannot contain '.'") - from torch import nn - - if hasattr(nn.ModuleDict(), key): - raise ValueError(f"{usage} collides with a ModuleDict attribute") - - @beartype def get_feature_embedding_dims( total_embedding_dim: int, @@ -141,6 +131,12 @@ def get_feature_embedding_dims( class BaseFeatureIngestion(nn.Module): + initialization_config: IngestionComponentBase | None = None + initialization_seed: int | None = None + + def initialization_children(self) -> Iterable["BaseFeatureIngestion"]: + return () + @conditional_beartype def forward(self, src: dict[str, Tensor], metadata: dict[str, Tensor]) -> Tensor: raise NotImplementedError @@ -151,6 +147,10 @@ def _scale_embedding(x: Tensor, embedding_dim: int) -> Tensor: return x * math.sqrt(embedding_dim) +class IngestionOutputProjection(nn.Linear): + """Adapt an ingestion representation to its consumer's width.""" + + class RealFeatureProjection(nn.Linear): """Project one standardized real-valued feature into an embedding space.""" @@ -681,7 +681,7 @@ def __init__( categorical_set = set(categorical_columns) real_set = set(real_columns) for group_name, group_columns in self.groups.items(): - _validate_module_dict_key( + validate_module_dict_key( group_name, f"grouped ingestion group {group_name!r}" ) self.group_ingestions[group_name] = FeaturePoolFeatureIngestion( @@ -1647,7 +1647,7 @@ class CompositeFeatureIngestion(BaseFeatureIngestion): def __init__( self, *, - branches: dict[str, nn.Module], + branches: dict[str, BaseFeatureIngestion], branch_widths: dict[str, int], merge_type: str, merge_dim: int, @@ -1655,7 +1655,7 @@ def __init__( ): super().__init__() for branch_name in branches: - _validate_module_dict_key( + validate_module_dict_key( branch_name, f"Composite ingestion branch {branch_name!r}" ) self.branches = ModuleDict(branches) @@ -1666,6 +1666,9 @@ def __init__( merge_softmax, ) + def initialization_children(self) -> Iterable[BaseFeatureIngestion]: + return cast(Iterable[BaseFeatureIngestion], self.branches.values()) + @conditional_beartype def forward(self, src: dict[str, Tensor], metadata: dict[str, Tensor]) -> Tensor: branch_outputs = { diff --git a/src/sequifier/model/initialization.py b/src/sequifier/model/initialization.py index c1544bba..f971498a 100644 --- a/src/sequifier/model/initialization.py +++ b/src/sequifier/model/initialization.py @@ -15,12 +15,19 @@ ModelInitializationConfig, ParameterKind, ) -from sequifier.model.decoders import TargetDecoderBranch, TargetDecoding from sequifier.model.ingestions import ( + BaseFeatureIngestion, RealFeatureProjection, TemporalConvFeatureIngestion, ) from sequifier.model.layers import FeedForward, RMSNorm, SelfAttention +from sequifier.model.module_roles import ( + decoder_module, + decoder_output_ids, + ingestion_projection_ids, + position_embedding_ids, + range_position_projection, +) from sequifier.typechecking import beartype EMBEDDING_INIT_STD = 0.02 @@ -59,16 +66,9 @@ def __init__( self.initialization = initialization or ModelInitializationConfig() self.initialized_parameter_ids: set[int] = set() self.matched_override_targets: set[InitializationTarget] = set() - self.position_embedding_module_ids = self._position_embedding_module_ids() - self.ingestion_projection_module_ids = self._ingestion_projection_module_ids() - self.decoder_output_module_ids = self._decoder_output_module_ids() - - @beartype - def _decoder_module(self) -> Optional[nn.Module]: - if isinstance(self.model, (TargetDecoding, TargetDecoderBranch)): - return self.model - decoder = getattr(self.model, "decoder", None) - return decoder if isinstance(decoder, nn.Module) else None + self.position_embedding_module_ids = position_embedding_ids(model) + self.ingestion_projection_module_ids = ingestion_projection_ids(model) + self.decoder_output_module_ids = decoder_output_ids(decoder_module(model)) @beartype def initialize(self) -> None: @@ -82,79 +82,6 @@ def initialize(self) -> None: self._initialize_remaining_modules() self._initialize_free_parameters() - @beartype - def _position_embedding_module_ids(self) -> set[int]: - position_embeddings: set[int] = set() - for module in self.model.modules(): - for attribute in ( - "pos_encoder", - "position_embedding", - "global_position_encoder", - "axis_embedding_layers", - ): - value = getattr(module, attribute, None) - if not isinstance(value, nn.Module): - continue - value_module = cast(nn.Module, value) - position_embeddings.update( - id(child) - for child in value_module.modules() - if isinstance(child, nn.Embedding) - ) - return position_embeddings - - @beartype - def _ingestion_projection_module_ids(self) -> set[int]: - projections: set[int] = set() - for module in self.model.modules(): - if ( - isinstance(module, nn.Linear) - and getattr(module, "_sequifier_layer_group", None) - == "ingestion.output_projection" - ): - projections.add(id(module)) - for attribute in ( - "output_projection_layer", - "input_projection", - "concat_projection", - "branch_projections", - ): - value = getattr(module, attribute, None) - if not isinstance(value, nn.Module): - continue - value_module = cast(nn.Module, value) - projections.update( - id(child) - for child in value_module.modules() - if isinstance(child, nn.Linear) - ) - return projections - - @beartype - def _decoder_output_module_ids(self) -> set[int]: - decoder = self._decoder_module() - if decoder is None: - return set() - marked = { - id(module) - for module in decoder.modules() - if isinstance(module, nn.Linear) - and getattr(module, "_sequifier_decoder_output", False) - } - output_module_ids = { - id(output_layer) - for module in decoder.modules() - if isinstance(module, TargetDecoderBranch) - for output_layer in module.output_layers.values() - if isinstance(output_layer, nn.Linear) - } - if output_module_ids or marked: - return output_module_ids | marked - # Compatibility with the original per-target ModuleDict decoder. - return { - id(module) for module in decoder.modules() if isinstance(module, nn.Linear) - } - @beartype def _is_initialized(self, parameter: Optional[Tensor]) -> bool: return parameter is None or id(parameter) in self.initialized_parameter_ids @@ -440,7 +367,7 @@ def _initialize_feed_forward(self, feed_forward: FeedForward) -> None: @beartype def _initialize_decoder(self) -> None: - decoder = self._decoder_module() + decoder = decoder_module(self.model) if decoder is None: return @@ -456,32 +383,31 @@ def _initialize_decoder(self) -> None: @beartype def _initialize_range_position_projection(self) -> None: - projection = getattr(self.model, "range_position_projection", None) - if projection is None: - projection = getattr(self.model, "range_projection", None) + projection = range_position_projection(self.model) if projection is None: return if not isinstance(projection, nn.Linear): raise TypeError("range_position_projection must be an nn.Linear") - if projection.out_features + 1 != projection.in_features: + linear_projection = cast(nn.Linear, projection) + if linear_projection.out_features + 1 != linear_projection.in_features: raise ValueError( "range_position_projection must map model_dim + 1 to model_dim" ) if not self._apply_override( - "position.range_projection", "weight", [projection.weight] + "position.range_projection", "weight", [linear_projection.weight] ): - if not self._is_initialized(projection.weight): + if not self._is_initialized(linear_projection.weight): self._identity_plus_normal_raw( - projection.weight, + linear_projection.weight, mean=0.0, std=EMBEDDING_INIT_STD, ) - self._mark_initialized(projection.weight) + self._mark_initialized(linear_projection.weight) if not self._apply_override( - "position.range_projection", "bias", [projection.bias] + "position.range_projection", "bias", [linear_projection.bias] ): - self._zero_(projection.bias) + self._zero_(linear_projection.bias) @beartype def _initialize_real_feature_projections(self) -> None: @@ -564,12 +490,12 @@ def _initialize_remaining_modules(self) -> None: nn.BatchNorm3d, ), ): - weight = getattr(module, "weight", None) + weight = module.weight if not self._apply_override("normalization", "weight", [weight]): if isinstance(weight, Tensor) and not self._is_initialized(weight): nn.init.ones_(weight) self._mark_initialized(weight) - bias = getattr(module, "bias", None) + bias = module.bias if not self._apply_override("normalization", "bias", [bias]): self._zero_(bias) elif isinstance(module, nn.Linear): @@ -622,16 +548,16 @@ def initialize_model_weights( return initializer.matched_override_targets -def initialize_owned_ingestion(model, initialization, *, warn_unmatched=True): - from sequifier.config.initialization_config import ModelInitializationConfig +def initialize_owned_ingestion( + model: BaseFeatureIngestion, initialization, *, warn_unmatched=True +): from sequifier.model.initialization_seed import isolated_initialization def active(node): - config = getattr(node, "_sequifier_initialization_config", None) - if getattr(node, "_sequifier_initialization_seed", None) is not None: + if node.initialization_seed is not None: return True - for child in getattr(node, "branches", {}).values(): - child_config = getattr(child, "_sequifier_initialization_config", config) + for child in node.initialization_children(): + child_config = child.initialization_config if ( child_config is not None and child_config.initialization.configured_targets() @@ -648,7 +574,7 @@ def active(node): requests = {} def visit(node, inherited): - config = getattr(node, "_sequifier_initialization_config", None) + config = node.initialization_config values = inherited.model_dump(mode="python", exclude_none=True) if config is not None: for group, targets in config.initialization.model_dump( @@ -656,12 +582,12 @@ def visit(node, inherited): ).items(): values[group] = {**values.get(group, {}), **targets} effective = ModelInitializationConfig.model_validate(values) - children = getattr(node, "branches", {}) - excluded = {id(p) for child in children.values() for p in child.parameters()} + branches = node.initialization_children() + excluded = {id(p) for child in branches for p in child.parameters()} owned = {id(p) for p in node.parameters()} - excluded request = ( effective.model_dump(mode="python"), - getattr(node, "_sequifier_initialization_seed", None), + node.initialization_seed, ) for identity in owned: if identity in requests and requests[identity] != request: @@ -674,9 +600,12 @@ def visit(node, inherited): groups = semantic_parameter_groups(node) biases = { - id(module.bias) + id(parameter) for module in node.modules() - if isinstance(getattr(module, "bias", None), Tensor) + for name, parameter in module.named_parameters( + recurse=False, remove_duplicate=False + ) + if name == "bias" } unmatched = { (group, kind) @@ -700,13 +629,11 @@ def visit(node, inherited): if not parameter.requires_grad } ) - with isolated_initialization( - getattr(node, "_sequifier_initialization_seed", None) - ): + with isolated_initialization(node.initialization_seed): initializer.initialize() visited.update(id(p) for p in node.parameters() if id(p) not in excluded) all_matched.update(initializer.matched_override_targets) - for child in children.values(): + for child in branches: visit(child, effective) visit(model, initialization) diff --git a/src/sequifier/model/layers.py b/src/sequifier/model/layers.py index d9d46883..4c82d735 100644 --- a/src/sequifier/model/layers.py +++ b/src/sequifier/model/layers.py @@ -15,6 +15,7 @@ def __init__(self, dim: int, eps: float = 1e-6): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) + self.bias = None @conditional_beartype def forward(self, x): @@ -159,6 +160,7 @@ def __init__( output_projection=True, ): super().__init__() + self.export_attention = False self.num_heads = num_heads self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads self.model_dim = model_dim @@ -240,7 +242,7 @@ def forward( analysis_requested = trace is not None and any( trace.requires(f"{site_prefix}.{suffix}") for suffix in analysis_sites ) - if getattr(self, "_sequifier_export_attention", False): + if self.export_attention: # Explicit export lowering retains attention-weight dropout independently # of residual and feed-forward dropout. Eager SDPA is unchanged. scores = torch.matmul(xq, xk.transpose(-2, -1)) / math.sqrt(self.head_dim) diff --git a/src/sequifier/model/module_roles.py b/src/sequifier/model/module_roles.py new file mode 100644 index 00000000..d1a630ad --- /dev/null +++ b/src/sequifier/model/module_roles.py @@ -0,0 +1,92 @@ +"""Structural module roles shared by initialization and parameter selection.""" + +from torch import nn + +from sequifier.model.decoders import ( + AutoregressiveTransformerDecoderBranch, + HashTargetHead, + TargetDecoderBranch, + TargetDecoding, +) +from sequifier.model.ingestions import IngestionOutputProjection + +_POSITION_MODULES = { + "pos_encoder", + "position_embedding", + "global_position_encoder", + "axis_embedding_layers", +} +_INGESTION_PROJECTION_MODULES = { + "output_projection_layer", + "input_projection", + "concat_projection", + "branch_projections", +} + + +def decoder_module(model: nn.Module) -> nn.Module | None: + if isinstance(model, (TargetDecoding, TargetDecoderBranch)): + return model + try: + return model.get_submodule("decoder") + except AttributeError: + return None + + +def position_embedding_ids(model: nn.Module) -> set[int]: + return { + id(module) + for name, module in model.named_modules(remove_duplicate=False) + if isinstance(module, nn.Embedding) + and _POSITION_MODULES.intersection(name.split(".")) + } + + +def ingestion_projection_ids(model: nn.Module) -> set[int]: + return { + id(module) + for name, module in model.named_modules(remove_duplicate=False) + if isinstance(module, nn.Linear) + and ( + isinstance(module, IngestionOutputProjection) + or _INGESTION_PROJECTION_MODULES.intersection(name.split(".")) + ) + } + + +def decoder_output_ids(decoder: nn.Module | None) -> set[int]: + if decoder is None: + return set() + ids = { + id(output) + for branch in decoder.modules() + if isinstance( + branch, (TargetDecoderBranch, AutoregressiveTransformerDecoderBranch) + ) + for output in branch.output_layers.values() + if isinstance(output, nn.Linear) + } + ids.update( + id(layer) + for head in decoder.modules() + if isinstance(head, HashTargetHead) + for layer in head.layers + ) + # Legacy decoders were per-target ModuleDicts. + return ids or { + id(module) for module in decoder.modules() if isinstance(module, nn.Linear) + } + + +def range_position_projection(model: nn.Module) -> object: + # Accept both historical names for standalone backbones. + try: + projection = model.range_position_projection + except AttributeError: + projection = None + if projection is not None: + return projection + try: + return model.range_projection + except AttributeError: + return None diff --git a/src/sequifier/model/network.py b/src/sequifier/model/network.py index 1f053618..5f26eb2e 100644 --- a/src/sequifier/model/network.py +++ b/src/sequifier/model/network.py @@ -6,7 +6,16 @@ import torch from torch import Tensor, nn +from sequifier.config.depth_layout import DepthLayoutRegistryModel +from sequifier.model.backbone import TransformerBackbone +from sequifier.model.decoders import ( + AutoregressiveTransformerDecoderBranch, + TargetDecoderBranch, + TargetDecoding, +) from sequifier.model.dtypes import cast_floating_to_module_dtype +from sequifier.model.layers import SequifierEncoderLayer +from sequifier.model.prediction_distribution import PredictionScales from sequifier.model.tracing import ( CaptureRequest, ForwardContext, @@ -59,7 +68,7 @@ def __init__( *, ingestion: nn.Module, ingestion_adapter: nn.Module, - decoder: nn.Module | None, + decoder: TargetDecoding | None, decoder_input_width: int, decoding_support: int, prediction_length: int, @@ -68,14 +77,20 @@ def __init__( static_condition_encoder: nn.Module | None = None, preconditioning_image_encoder: nn.Module | None = None, prefix_projection: nn.Module | None = None, + depth_layouts: DepthLayoutRegistryModel | None = None, + input_columns: tuple[str, ...] = (), + input_n_classes: dict[str, int] | None = None, ) -> None: super().__init__() if decoding_support <= 0: raise ValueError("decoding_support must be positive.") self.ingestion = ingestion self.ingestion_adapter = ingestion_adapter - if decoder is not None: - self.decoder = decoder + self.decoder = decoder + self.prediction_scales: PredictionScales | None = None + self.depth_layouts = depth_layouts + self.input_columns = input_columns + self.input_n_classes = input_n_classes self.decoder_input_width = decoder_input_width self.decoding_support = decoding_support self.prediction_length = prediction_length @@ -85,6 +100,18 @@ def __init__( self.preconditioning_image_encoder = preconditioning_image_encoder self.prefix_projection = prefix_projection + def require_decoder(self) -> TargetDecoding: + if self.decoder is None: + raise ValueError("This model interface has no prediction decoder.") + return self.decoder + + def prediction_scale(self, target: str): + if self.prediction_scales is None: + raise ValueError( + f"{target}: scale: model requires a prediction_distributions entry" + ) + return self.prediction_scales.scale(target) + def prefix(self, features: dict[str, Tensor]) -> Tensor | None: parts = [] if self.static_condition_encoder is not None: @@ -106,7 +133,7 @@ def prefix(self, features: dict[str, Tensor]) -> Tensor | None: def ingest( self, features: dict[str, Tensor], metadata: dict[str, Tensor] ) -> Tensor: - layouts = getattr(self, "depth_layouts", None) + layouts = self.depth_layouts if ( layouts and not torch.compiler.is_compiling() @@ -207,7 +234,7 @@ def decode_with_auxiliary( axes=("batch", "time", "channel"), width=self.decoder_input_width, ) - decoded = self.decoder( + decoded = self.require_decoder()( decoder_input, teacher_targets=teacher_targets, teacher_valid_mask=teacher_valid_mask, @@ -241,7 +268,7 @@ class ComposableTransformerNetwork(nn.Module): def __init__( self, *, - backbone: nn.Module, + backbone: TransformerBackbone, interfaces: dict[str, ModelInterfaceModule], attention_mask_policy: Tensor, attention_mask_policy_with_prefix: Tensor, @@ -289,12 +316,11 @@ def regularization_loss(self, interface_name: str | None = None) -> Tensor: """Return decoder regularization through an explicit model contract.""" route = self.resolve_interface(interface_name) - regularization = getattr(route.decoder, "regularization_loss", None) - if not callable(regularization): - parameter = next(route.decoder.parameters(), None) - device = parameter.device if parameter is not None else torch.device("cpu") - return torch.zeros((), device=device) - return regularization() + if route.decoder is not None: + return route.decoder.regularization_loss() + parameter = next(route.parameters(), None) + device = parameter.device if parameter is not None else torch.device("cpu") + return torch.zeros((), device=device) def structural_metadata(self) -> dict[str, object]: return { @@ -309,11 +335,16 @@ def trace_catalog_for( self, interface_name: str | None = None ) -> tuple[TraceSite, ...]: route = self.resolve_interface(interface_name) + decoder = route.require_decoder() branch_counts = { - name: tuple(branch.hidden_dims) - for name, branch in route.decoder.branches.items() + name: tuple( + cast( + TargetDecoderBranch | AutoregressiveTransformerDecoderBranch, branch + ).hidden_dims + ) + for name, branch in decoder.branches.items() } - first_attention = self.backbone.layers[0].attn + first_attention = cast(SequifierEncoderLayer, self.backbone.layers[0]).attn sites = list( trace_sites( num_layers=len(self.backbone.layers), @@ -321,13 +352,12 @@ def trace_catalog_for( attention_width=int(first_attention.head_dim), decoder_input_width=route.decoder_input_width, decoder_branches=branch_counts, - target_branches=dict(route.decoder.target_to_branch), + target_branches=dict(decoder.target_to_branch), ) ) - for branch_name, branch in route.decoder.branches.items(): - describe = getattr(branch, "trace_sites", None) - if callable(describe): - sites.extend(cast(tuple[TraceSite, ...], describe(branch_name))) + for branch_name, branch in decoder.branches.items(): + if isinstance(branch, AutoregressiveTransformerDecoderBranch): + sites.extend(branch.trace_sites(branch_name)) return tuple(sites) @property diff --git a/src/sequifier/model/parameter_catalog.py b/src/sequifier/model/parameter_catalog.py index def23e0c..9134a0d3 100644 --- a/src/sequifier/model/parameter_catalog.py +++ b/src/sequifier/model/parameter_catalog.py @@ -49,16 +49,21 @@ def __init__(self, model: nn.Module): # Component-local discovery preserves decoder and range-position semantics # when the catalog is built for a complete multi-interface network. from sequifier.model.decoders import TargetDecoderBranch, TargetDecoding + from sequifier.model.depth_ingestion import DepthTransformerIngestion + from sequifier.model.module_roles import range_position_projection + depth_parameters = { + id(parameter) + for module in model.modules() + if isinstance(module, DepthTransformerIngestion) + for parameter in module.parameters() + } for name, module in model.named_modules(): component = name.rsplit(".", 1)[-1] if ( component not in {"backbone", "ingestion", "decoder"} and not isinstance(module, (TargetDecoding, TargetDecoderBranch)) - and not any( - hasattr(module, key) - for key in ("range_position_projection", "range_projection") - ) + and range_position_projection(module) is None ): continue for group, component_parameters in semantic_parameter_groups( @@ -109,9 +114,7 @@ def __init__(self, model: nn.Module): shape=tuple(parameter.shape), dtype=parameter.dtype, shared_parameter_id=parameter_id if len(names) > 1 else None, - depth_parameter=bool( - getattr(parameter, "_sequifier_depth_parameter", False) - ), + depth_parameter=bool(identity in depth_parameters), branch_path=tuple( part for i, part in enumerate(canonical_name.split(".")) diff --git a/src/sequifier/model/parameter_groups.py b/src/sequifier/model/parameter_groups.py index 270d0c3c..34a3324c 100644 --- a/src/sequifier/model/parameter_groups.py +++ b/src/sequifier/model/parameter_groups.py @@ -2,17 +2,23 @@ from collections import defaultdict from collections.abc import Iterable -from typing import Optional, cast +from typing import Optional from torch import Tensor, nn from sequifier.config.layer_groups import LayerGroup -from sequifier.model.decoders import TargetDecoderBranch, TargetDecoding from sequifier.model.ingestions import ( RealFeatureProjection, TemporalConvFeatureIngestion, ) from sequifier.model.layers import FeedForward, RMSNorm, SelfAttention +from sequifier.model.module_roles import ( + decoder_module, + decoder_output_ids, + ingestion_projection_ids, + position_embedding_ids, + range_position_projection, +) from sequifier.typechecking import beartype CONVOLUTION_TYPES = (nn.Conv1d, nn.Conv2d, nn.Conv3d) @@ -26,9 +32,9 @@ def __init__(self, model: nn.Module): self.model = model self._claimed_parameter_ids: set[int] = set() self._groups: dict[LayerGroup, list[nn.Parameter]] = defaultdict(list) - self._position_embedding_module_ids = self._position_embedding_ids() - self._ingestion_projection_module_ids = self._ingestion_projection_ids() - self._decoder_output_module_ids = self._decoder_output_ids() + self._position_embedding_module_ids = position_embedding_ids(model) + self._ingestion_projection_module_ids = ingestion_projection_ids(model) + self._decoder_output_module_ids = decoder_output_ids(decoder_module(model)) @beartype def collect(self) -> dict[LayerGroup, tuple[nn.Parameter, ...]]: @@ -59,92 +65,17 @@ def _claim( self._claimed_parameter_ids.add(parameter_id) self._groups[group].append(parameter) - @beartype - def _decoder_module(self) -> Optional[nn.Module]: - if isinstance(self.model, (TargetDecoding, TargetDecoderBranch)): - return self.model - decoder = getattr(self.model, "decoder", None) - return decoder if isinstance(decoder, nn.Module) else None - - @beartype - def _position_embedding_ids(self) -> set[int]: - position_embeddings: set[int] = set() - for module in self.model.modules(): - for attribute in ( - "pos_encoder", - "position_embedding", - "global_position_encoder", - "axis_embedding_layers", - ): - value = getattr(module, attribute, None) - if not isinstance(value, nn.Module): - continue - position_embeddings.update( - id(child) - for child in cast(nn.Module, value).modules() - if isinstance(child, nn.Embedding) - ) - return position_embeddings - - @beartype - def _ingestion_projection_ids(self) -> set[int]: - projections: set[int] = set() - for module in self.model.modules(): - if ( - isinstance(module, nn.Linear) - and getattr(module, "_sequifier_layer_group", None) - == "ingestion.output_projection" - ): - projections.add(id(module)) - for attribute in ( - "output_projection_layer", - "input_projection", - "concat_projection", - "branch_projections", - ): - value = getattr(module, attribute, None) - if not isinstance(value, nn.Module): - continue - projections.update( - id(child) - for child in cast(nn.Module, value).modules() - if isinstance(child, nn.Linear) - ) - return projections - - @beartype - def _decoder_output_ids(self) -> set[int]: - marked = { - id(module) - for module in self.model.modules() - if isinstance(module, nn.Linear) - and getattr(module, "_sequifier_decoder_output", False) - } - decoder = self._decoder_module() - if decoder is None: - return marked - output_module_ids = { - id(output_layer) - for module in decoder.modules() - if isinstance(module, TargetDecoderBranch) - for output_layer in module.output_layers.values() - if isinstance(output_layer, nn.Linear) - } - if output_module_ids or marked: - return output_module_ids | marked - return { - id(module) for module in decoder.modules() if isinstance(module, nn.Linear) - } - @beartype def _claim_linear(self, layer: nn.Linear, group: LayerGroup) -> None: self._claim(group, (layer.weight, layer.bias)) @beartype - def _claim_convolution(self, layer: nn.Module, group: LayerGroup) -> None: + def _claim_convolution( + self, layer: nn.Conv1d | nn.Conv2d | nn.Conv3d, group: LayerGroup + ) -> None: self._claim( group, - (getattr(layer, "weight", None), getattr(layer, "bias", None)), + (layer.weight, layer.bias), ) @beartype @@ -175,7 +106,7 @@ def _collect_transformer_layers(self) -> None: @beartype def _collect_decoder(self) -> None: - decoder = self._decoder_module() + decoder = decoder_module(self.model) if decoder is None: return for module in decoder.modules(): @@ -190,9 +121,7 @@ def _collect_decoder(self) -> None: @beartype def _collect_range_position_projection(self) -> None: - projection = getattr(self.model, "range_position_projection", None) - if projection is None: - projection = getattr(self.model, "range_projection", None) + projection = range_position_projection(self.model) if isinstance(projection, nn.Linear): self._claim_linear(projection, "position.range_projection") @@ -253,7 +182,7 @@ def _collect_remaining_modules(self) -> None: ): self._claim( "normalization", - (getattr(module, "weight", None), getattr(module, "bias", None)), + (module.weight, module.bias), ) elif isinstance(module, nn.Linear): group = ( diff --git a/src/sequifier/model/static_conditioning.py b/src/sequifier/model/static_conditioning.py index b87b05c1..47d78c4d 100644 --- a/src/sequifier/model/static_conditioning.py +++ b/src/sequifier/model/static_conditioning.py @@ -6,7 +6,10 @@ from torch import Tensor, nn from sequifier.model.dtypes import cast_floating_to_module_dtype -from sequifier.model.ingestions import get_feature_embedding_dims +from sequifier.model.ingestions import ( + IngestionOutputProjection, + get_feature_embedding_dims, +) class StaticConditionEncoder(nn.Module): @@ -53,8 +56,7 @@ def __init__( raise ValueError( "Every static conditioning column must be categorical or real" ) - self.projection = nn.Linear(input_width, model_dim) - self.projection._sequifier_layer_group = "ingestion.output_projection" + self.projection = IngestionOutputProjection(input_width, model_dim) def forward(self, features: dict[str, Tensor]) -> Tensor: parts = [ diff --git a/src/sequifier/model/tracing.py b/src/sequifier/model/tracing.py index d682b377..aef797b7 100644 --- a/src/sequifier/model/tracing.py +++ b/src/sequifier/model/tracing.py @@ -7,6 +7,7 @@ import torch from torch import Tensor +from torch._dynamo import OptimizedModule from sequifier.typechecking import beartype @@ -178,9 +179,13 @@ def analysis_execution( trace: CaptureRequest | None = None, interventions: tuple[InterventionBinding, ...] = (), ): - if hasattr(model, "_orig_mod"): + if isinstance(model, OptimizedModule): raise ValueError("analysis_execution requires an eager, uncompiled model.") - available = {site.name for site in getattr(model, "trace_catalog", ())} + try: + trace_catalog = model.trace_catalog + except AttributeError: + trace_catalog = () + available = {site.name for site in trace_catalog} requested = set(trace.sites if trace is not None else ()).union( binding.site for binding in interventions ) @@ -216,7 +221,7 @@ def functional_forward( *args: Any, **kwargs: Any, ) -> Any: - if hasattr(model, "_orig_mod"): + if isinstance(model, OptimizedModule): raise ValueError("functional_forward requires an eager, uncompiled model.") return torch.func.functional_call(model, (parameters, buffers), args, kwargs) diff --git a/src/sequifier/objectives.py b/src/sequifier/objectives.py index 9d130a1f..ebdf2bd5 100644 --- a/src/sequifier/objectives.py +++ b/src/sequifier/objectives.py @@ -551,7 +551,7 @@ def apply_bert_masking( ) masked_data = dict(data_batch) - depth_layouts = getattr(config, "depth_layouts", None) + depth_layouts = config.depth_layouts depth_column_layout = depth_layouts.column_to_layout if depth_layouts else {} for col, tensor in data_batch.items(): diff --git a/src/sequifier/preprocess.py b/src/sequifier/preprocess.py index 5ebf15bf..f80b77b8 100644 --- a/src/sequifier/preprocess.py +++ b/src/sequifier/preprocess.py @@ -606,7 +606,11 @@ def _validate_declared_roles_against_metadata( @beartype def _dtype_is_numeric(dtype: Any) -> bool: - return dtype.is_numeric() if hasattr(dtype, "is_numeric") else False + try: + is_numeric = dtype.is_numeric + except AttributeError: + return False + return is_numeric() def _is_temporal_dtype_name(dtype_name: str) -> bool: @@ -1266,6 +1270,8 @@ def __init__( version=ContractVersions.STORED_WINDOW_LAYOUT, ) self.has_sample_positions = False + self.output_n_classes = {} + self.output_column_data_types = {} self.image_manifest = self._validate_image_sources() self._setup_directories() @@ -2803,7 +2809,7 @@ def _process_batches_multiple_files( "seed": self.seed, "normalize_real_columns": self.normalize_real_columns, "sequence_split_assignments": sequence_split_assignments, - "curriculum_column": getattr(self, "curriculum_column", None), + "curriculum_column": self.curriculum_column, "split_column": self.split_column, "split_values": self.split_values, "normalize_on_all_data": self.normalize_on_all_data, @@ -2951,11 +2957,11 @@ def _write_or_validate_resume_manifest( "mask_column": self.mask_column, **( {"curriculum_column": self.curriculum_column} - if getattr(self, "curriculum_column", None) is not None + if self.curriculum_column is not None else {} ), "use_precomputed_maps": self.use_precomputed_maps, - "cardinality_config": getattr(self, "cardinality_config", {}), + "cardinality_config": self.cardinality_config, "n_classes": n_classes, "id_maps": id_maps, "column_data_types": col_types, @@ -3183,7 +3189,7 @@ def _create_metadata_for_folder( payload = load_pt_payload( file_path, layouts=self.depth_layouts, - n_classes=getattr(self, "output_n_classes", None), + n_classes=self.output_n_classes, ) sequences_dict = payload.sequences left_pad_lengths = payload.left_pad_lengths @@ -3274,8 +3280,8 @@ def _create_metadata_for_folder( metadata = { "split_index": split_index, - "n_classes": getattr(self, "output_n_classes", {}), - "column_data_types": getattr(self, "output_column_data_types", {}), + "n_classes": self.output_n_classes, + "column_data_types": self.output_column_data_types, "id_maps": self.output_id_maps, "selected_columns_statistics": self.output_selected_columns_statistics, "normalize_real_columns": self.normalize_real_columns, @@ -3285,7 +3291,7 @@ def _create_metadata_for_folder( **self._layout_metadata(), } metadata["window_length"] = self.window_lengths[split_index] - if getattr(self, "has_sample_positions", False): + if self.has_sample_positions: metadata["curriculum_column"] = self.curriculum_column metadata_path = directory / "metadata.json" diff --git a/src/sequifier/rl/checkpoint.py b/src/sequifier/rl/checkpoint.py index 40591b29..0f7727bb 100644 --- a/src/sequifier/rl/checkpoint.py +++ b/src/sequifier/rl/checkpoint.py @@ -50,7 +50,7 @@ def policy_model_artifact( policy.execution_config, provenance={ **(metadata or {}), - "transfer": getattr(policy, "transfer_report", None), + "transfer": policy.transfer_report, }, state_dict=weights, ) diff --git a/src/sequifier/rl/environment.py b/src/sequifier/rl/environment.py index eb134613..6eedf1f3 100644 --- a/src/sequifier/rl/environment.py +++ b/src/sequifier/rl/environment.py @@ -2,7 +2,6 @@ import json from dataclasses import dataclass, field -from importlib import import_module from typing import Any, Literal, Mapping, Protocol import numpy as np @@ -13,6 +12,7 @@ require_contract_version, ) from sequifier.fingerprints import fingerprint +from sequifier.integration.factories import load_factory from sequifier.io.values import ( DomainValue, numeric_scalar, @@ -97,42 +97,67 @@ def reset_group(self, token: object, rollout_seed: int) -> OnlineResetResult: .. def load_online_adapter(spec): - module, attribute = spec.factory.split(":") - adapter = getattr(import_module(module), attribute)(**spec.config) + adapter = load_factory(spec.factory)(**spec.config) try: validate_online_adapter(adapter) return adapter except BaseException: - close = getattr(adapter, "close", None) + try: + close = adapter.close + except AttributeError: + close = None if callable(close): close() raise def validate_online_adapter(adapter): - if getattr(adapter, "protocol", None) != ONLINE_POLICY_PROTOCOL: + try: + protocol = adapter.protocol + except AttributeError: + protocol = None + if protocol != ONLINE_POLICY_PROTOCOL: raise ValueError(f"adapter.protocol must be {ONLINE_POLICY_PROTOCOL!r}") + try: + protocol_version = adapter.protocol_version + except AttributeError: + protocol_version = None require_contract_version( "online policy environment adapter protocol", - getattr(adapter, "protocol_version", None), + protocol_version, ContractVersions.ONLINE_POLICY_ADAPTER, ) - implementation_id = getattr(adapter, "implementation_id", None) + try: + implementation_id = adapter.implementation_id + except AttributeError: + implementation_id = None if not isinstance(implementation_id, str) or not implementation_id.strip(): raise ValueError("online adapter requires a nonempty implementation_id") - if not isinstance(getattr(adapter, "capabilities", None), EnvironmentCapabilities): + try: + capabilities = adapter.capabilities + except AttributeError: + capabilities = None + if not isinstance(capabilities, EnvironmentCapabilities): raise ValueError("online adapter requires explicit EnvironmentCapabilities") - for method in ("reset", "step", "close"): - if not callable(getattr(adapter, method, None)): - raise ValueError(f"online adapter requires {method}") + try: + methods = (adapter.reset, adapter.step, adapter.close) + except AttributeError as error: + raise ValueError(f"online adapter requires {error.name}") from error + for name, method in zip(("reset", "step", "close"), methods): + if not callable(method): + raise ValueError(f"online adapter requires {name}") def require_exact_snapshot(adapter): if adapter.capabilities.exact_snapshot is not True: raise ValueError("exact PPO resume requires capabilities.exact_snapshot") - for method in ("snapshot_state", "restore_state"): - if not callable(getattr(adapter, method, None)): - raise ValueError(f"exact snapshot adapter requires {method}") + try: + methods = (adapter.snapshot_state, adapter.restore_state) + except AttributeError as error: + raise ValueError(f"exact snapshot adapter requires {error.name}") from error + for name, method in zip(("snapshot_state", "restore_state"), methods): + if not callable(method): + raise ValueError(f"exact snapshot adapter requires {name}") def require_group_start(adapter, *, exact=False): @@ -140,9 +165,13 @@ def require_group_start(adapter, *, exact=False): raise ValueError( "GRPO requires explicit group_start: snapshot or scenario capability; a repeated seed is insufficient" ) - for method in ("create_group_start", "reset_group"): - if not callable(getattr(adapter, method, None)): - raise ValueError(f"group-start adapter requires {method}") + try: + methods = (adapter.create_group_start, adapter.reset_group) + except AttributeError as error: + raise ValueError(f"group-start adapter requires {error.name}") from error + for name, method in zip(("create_group_start", "reset_group"), methods): + if not callable(method): + raise ValueError(f"group-start adapter requires {name}") if exact and adapter.capabilities.reproducible_group_start is not True: raise ValueError( "exact GRPO resume requires documented reproducible_group_start capability" @@ -225,9 +254,11 @@ def validate_step(result, units, *, compact_images=False): def require_seeded_reset(adapter): - if not adapter.capabilities.seeded_reset or not callable( - getattr(adapter, "reset_from_observation", None) - ): + try: + reset = adapter.reset_from_observation + except AttributeError: + reset = None + if not adapter.capabilities.seeded_reset or not callable(reset): raise ValueError( "policy rollout requires seeded_reset and reset_from_observation(initial, seed)" ) diff --git a/src/sequifier/rl/inference.py b/src/sequifier/rl/inference.py index f9a1472b..6e66308c 100644 --- a/src/sequifier/rl/inference.py +++ b/src/sequifier/rl/inference.py @@ -134,7 +134,7 @@ def policy_windows(data, spec, config): @torch.no_grad() def predict_policy(policy, data, config): spec = policy.config - if getattr(config, "output_path", None) is not None and config.output_probabilities: + if config.output_path is not None and config.output_probabilities: from sequifier.infer import validate_probability_destinations validate_probability_destinations( diff --git a/src/sequifier/rl/joint_distribution.py b/src/sequifier/rl/joint_distribution.py index 32c348bc..09581a42 100644 --- a/src/sequifier/rl/joint_distribution.py +++ b/src/sequifier/rl/joint_distribution.py @@ -87,7 +87,7 @@ def _field(self, name, output): if stats is not None: location = denormalize_values(location, stats) scale = ( - self.route.prediction_scales.scale(name) + self.route.prediction_scale(name) if field.scale == "model" else field.scale.fixed ) diff --git a/src/sequifier/rl/learner.py b/src/sequifier/rl/learner.py index 8f624e93..2766e817 100644 --- a/src/sequifier/rl/learner.py +++ b/src/sequifier/rl/learner.py @@ -103,7 +103,7 @@ def _initialize_history_actor(config): raise ValueError("history_only requires initialize_from") actor = build_actor(config) name = actor.interface_name - decoder = actor.network.resolve_interface(name).decoder + decoder = actor.network.resolve_interface(name).require_decoder() history_ids = history_parameter_ids(actor.network, name) if any(id(parameter) in history_ids for parameter in decoder.parameters()): raise ValueError(f"{name}: history_only requires an untied action decoder") @@ -268,7 +268,7 @@ def _execute_policy(config, prepared, group, identity, on_checkpoint): adapter_contract, { "path": config.initialize_from, - "transfer": getattr(actor, "transfer_report", None), + "transfer": actor.transfer_report, "weight_hash": weight_fingerprint(actor.network.state_dict()), }, clone_weights(actor.network), @@ -307,7 +307,7 @@ def run_phase(transition): dependencies.revalidate() incoming_weight_hash = weight_fingerprint(actor.network.state_dict()) initialization = {} - if getattr(phase, "history_initialization", None) == "phase_start_actor": + if phase.uses_actor_history: from sequifier.model.history_transfer import history_components initialization["history_initialization"] = { diff --git a/src/sequifier/rl/offline_data.py b/src/sequifier/rl/offline_data.py index add81c1f..741bb81d 100644 --- a/src/sequifier/rl/offline_data.py +++ b/src/sequifier/rl/offline_data.py @@ -71,10 +71,16 @@ def prepare_offline(run, phase, catalog, cache, membership): raise ValueError( f"phases.{phase.name}: nonempty train and validation partitions required" ) - for attribute in ("all_episode_ids", "group_ids", "demonstration_keys"): - if (set(getattr(training, attribute)) & set(getattr(validation, attribute))) - { - None - }: + for attribute, train_ids, validation_ids in ( + ("all_episode_ids", training.all_episode_ids, validation.all_episode_ids), + ("group_ids", training.group_ids, validation.group_ids), + ( + "demonstration_keys", + training.demonstration_keys, + validation.demonstration_keys, + ), + ): + if (set(train_ids) & set(validation_ids)) - {None}: raise ValueError( f"phases.{phase.name}: {attribute} crosses train/validation partitions" ) @@ -93,9 +99,7 @@ def phase_validation_fraction(run, phase): """A combined leakage-safe split requires a single agreed fraction.""" refs = [phase.preferences] if phase.algorithm == "dpo" else phase.sources fractions = { - getattr(run, "source_validation_fractions", {}).get( - ref.source, run.validation_fraction - ) + run.source_validation_fractions.get(ref.source, run.validation_fraction) for ref in refs } if len(fractions) != 1: diff --git a/src/sequifier/rl/policy.py b/src/sequifier/rl/policy.py index eb227a51..facdedb7 100644 --- a/src/sequifier/rl/policy.py +++ b/src/sequifier/rl/policy.py @@ -30,9 +30,15 @@ class Policy(nn.Module): interface_name: str network: ComposableTransformerNetwork units: dict[str, str] + transfer_report: dict | None ppo_policy_version: int grpo_policy_version: int + def __init__(self): + super().__init__() + self.units = {} + self.transfer_report = None + @classmethod def from_network(cls, network, execution_config, interface_name: str): interface = execution_config.interface(interface_name) @@ -63,10 +69,10 @@ def backbone(self): def _resolve_actions(self): route = self.network.resolve_interface(self.interface_name) - self.action_plan = ActionExecutionPlan(self, route.decoder) + self.action_plan = ActionExecutionPlan(self, route.require_decoder()) self.action_order = self.action_plan.order self.autoregressive_actions = self.action_plan.has_autoregression - if not hasattr(route, "prediction_scales"): + if route.prediction_scales is None: route.prediction_scales = PredictionScales( self.interface.prediction_distributions ).to(self.device) diff --git a/src/sequifier/rl/ppo_evaluation.py b/src/sequifier/rl/ppo_evaluation.py index 4debd850..bac4697a 100644 --- a/src/sequifier/rl/ppo_evaluation.py +++ b/src/sequifier/rl/ppo_evaluation.py @@ -16,8 +16,9 @@ class EvaluationTimeout(TimeoutError): def require_evaluation_timer(): - if threading.current_thread() is not threading.main_thread() or not hasattr( - signal, "setitimer" + if ( + threading.current_thread() is not threading.main_thread() + or "setitimer" not in vars(signal) ): raise ValueError( "bounded local PPO evaluation requires the main thread and a POSIX interval timer" @@ -56,7 +57,7 @@ def evaluate_policy(actor, evaluation, *, forbidden_adapter=None): Adapters must release resources promptly from close(). """ require_evaluation_timer() - if evaluation.environment.units != getattr(actor, "units", {}): + if evaluation.environment.units != actor.units: raise ValueError( "evaluation environment units differ from the portable actor representation" ) diff --git a/src/sequifier/rl/preparation.py b/src/sequifier/rl/preparation.py index 94fda96a..0fa6772a 100644 --- a/src/sequifier/rl/preparation.py +++ b/src/sequifier/rl/preparation.py @@ -41,9 +41,9 @@ def validate_output_paths(config, dependencies): outputs = unique_paths(outputs, message) archives = unique_paths( [ - Path(phase.archive_directory).resolve() + Path(archive).resolve() for phase in config.phases - if getattr(phase, "archive_directory", None) + if (archive := phase.rollout_archive_directory) ], message, ) @@ -136,12 +136,13 @@ def preflight_environments(config): try: identities = {} for index, phase in enumerate(config.phases, config.phase_offset): - if not hasattr(phase, "environment"): + training_environment = phase.training_environment + if training_environment is None: continue require_evaluation_timer() for role, spec in ( - ("training", phase.environment), - ("evaluation", phase.evaluation.environment), + ("training", training_environment), + ("evaluation", phase.evaluation_environment), ): adapter = load_online_adapter(spec) try: diff --git a/src/sequifier/rl/prepared_inputs.py b/src/sequifier/rl/prepared_inputs.py index 93d3ec8d..d0c722a1 100644 --- a/src/sequifier/rl/prepared_inputs.py +++ b/src/sequifier/rl/prepared_inputs.py @@ -22,18 +22,16 @@ class RunInputs: sources: dict[str, ResolvedSequenceSource] checksums: dict[Path, str] - _identity: dict[str, str] = field(init=False, repr=False) + _identity: dict[str, str] = field(repr=False) - def __post_init__(self): - object.__setattr__( - self, - "_identity", - {str(path.resolve()): value for path, value in self.checksums.items()}, - ) + @classmethod + def from_sources(cls, sources, checksums): + identity = {str(path.resolve()): value for path, value in checksums.items()} + return cls(sources, checksums, identity) @classmethod def resolve(cls, config): - prepared = getattr(config, "_prepared_inputs", None) or PreparedTrainingInputs() + prepared = config._prepared_inputs or PreparedTrainingInputs() return prepared.resolve(config) @property @@ -145,7 +143,7 @@ def run_scope(self): @classmethod def for_run(cls, config): - if getattr(config, "_prepared_inputs", None) is None: + if config._prepared_inputs is None: config._prepared_inputs = cls() return config._prepared_inputs @@ -218,7 +216,7 @@ def resolve(self, config): if path not in self._checksums: self._checksums[path] = file_checksum(path) checksums[path] = self._checksums[path] - self._inputs[key] = RunInputs(sources, checksums) + self._inputs[key] = RunInputs.from_sources(sources, checksums) return self._inputs[key] @contextmanager @@ -269,9 +267,7 @@ def _prepare_phase(self, config, phase, policy, inputs, *, fitting=False): "preferences": preferences, "identity": checksums, "validation_fraction": config.validation_fraction, - "source_validation_fractions": getattr( - config, "source_validation_fractions", {} - ), + "source_validation_fractions": config.source_validation_fractions, "seed": config.seed, } ) diff --git a/src/sequifier/rl/references.py b/src/sequifier/rl/references.py index 92b24ede..bb92b7dc 100644 --- a/src/sequifier/rl/references.py +++ b/src/sequifier/rl/references.py @@ -31,7 +31,7 @@ def __init__(self, config, recovery, *, resuming): "resume.initialize_from.weights", ) for index, phase in enumerate(config.phases): - reference = getattr(phase, "reference", None) + reference = phase.reference_config if reference is None or reference.source != "artifact": continue if Path(reference.path).is_file(): @@ -50,10 +50,10 @@ def __init__(self, config, recovery, *, resuming): ) # Completed references and initialization are no longer needed in memory. pending = { - phase.reference.path + reference.path for phase in config.phases[recovery.phase_index :] - if getattr(phase, "reference", None) is not None - and phase.reference.source == "artifact" + if (reference := phase.reference_config) is not None + and reference.source == "artifact" } self.artifacts = { path: value for path, value in self.artifacts.items() if path in pending @@ -79,9 +79,7 @@ def _artifact(self, path): def resolve(self, actor, index): recovery = self.recovery - declaration = getattr( - self.config.phases[recovery.phase_index], "reference", None - ) + declaration = self.config.phases[recovery.phase_index].reference_config if declaration is None: return None, None if recovery.phase_state is not None: @@ -107,7 +105,7 @@ def resolve(self, actor, index): def validate_reference_state(phase, recovery, index): state = recovery.phase_state if state is not None: - reference = getattr(phase, "reference", None) + reference = phase.reference_config if (reference is None) != (state["reference"] is None): raise ValueError("checkpoint reference disagrees with the phase") if reference is not None: diff --git a/src/sequifier/rl/run_checkpoint.py b/src/sequifier/rl/run_checkpoint.py index 3d6fb50a..eea3eb05 100644 --- a/src/sequifier/rl/run_checkpoint.py +++ b/src/sequifier/rl/run_checkpoint.py @@ -291,7 +291,7 @@ def save(self, path, device, state=None): "artifact_type": "sequifier_policy_run", "format_version": ContractVersions.POLICY_RECOVERY, **{ - f.name: getattr(self, f.name) + f.name: vars(self)[f.name] for f in fields(self) if f.name not in {"representation", "completed"} }, diff --git a/src/sequifier/rl/run_state.py b/src/sequifier/rl/run_state.py index 4cb3ebcb..d5c6e86b 100644 --- a/src/sequifier/rl/run_state.py +++ b/src/sequifier/rl/run_state.py @@ -68,6 +68,10 @@ def from_actor(cls, actor, units): from sequifier.artifacts.model_export import model_execution_config + try: + units = actor.units + except AttributeError: + pass model = actor.execution_config.model.model_dump(mode="json") model["backbone"]["repository"] = None return cls( @@ -79,7 +83,7 @@ def from_actor(cls, actor, units): "policy": actor.config.model_dump(mode="json"), "model": model, "statistics": actor.interface.selected_columns_statistics, - "units": dict(getattr(actor, "units", units)), + "units": dict(units), }, sort_keys=True, allow_nan=False, @@ -218,13 +222,10 @@ def observe(self, actor, metric, transaction): def state_dict(self): return { - key: getattr(self, key) - for key in ( - "best_metric", - "best_weights", - "best_transaction", - "last_metric", - ) + "best_metric": self.best_metric, + "best_weights": self.best_weights, + "best_transaction": self.best_transaction, + "last_metric": self.last_metric, } def finish(self, context, transaction: int, clocks, provenance): @@ -291,17 +292,8 @@ def __init__(self, context): self.sampler = torch.Generator().manual_seed(context.identity.sampling_seed) self.state = context.resume_state or {} self.transaction = self.state.get("transaction", 0) - phase = context.phase - self.iteration_budget = ( - phase.iterations - if hasattr(phase, "iterations") - else (phase.total_env_steps + phase.rollout_env_steps - 1) - // phase.rollout_env_steps - if hasattr(phase, "total_env_steps") - else (phase.total_groups + phase.groups_per_iteration - 1) - // phase.groups_per_iteration - if hasattr(phase, "total_groups") - else context.optimization.step_budget or 1 + self.iteration_budget = context.phase.iteration_budget( + context.optimization.step_budget ) self.stopping = StoppingState(**self.state.get("stopping", {})) self.initial_evaluated = self.state.get("initial_evaluated", False) diff --git a/src/sequifier/rl/runtime.py b/src/sequifier/rl/runtime.py index 4c476e57..ed032d05 100644 --- a/src/sequifier/rl/runtime.py +++ b/src/sequifier/rl/runtime.py @@ -42,7 +42,7 @@ def collect_policy_rollout( if action_selection not in {"deterministic", "stochastic"}: raise ValueError("action_selection must be deterministic or stochastic") spec, units = policy.config, config.environment.units - if units != getattr(policy, "units", {}): + if units != policy.units: raise ValueError("rollout environment units differ from the policy") seed = SeedHistory.model_validate(seed.model_dump()) if ( diff --git a/src/sequifier/rl/simulator_workers.py b/src/sequifier/rl/simulator_workers.py index 0736ef4c..95a9cb56 100644 --- a/src/sequifier/rl/simulator_workers.py +++ b/src/sequifier/rl/simulator_workers.py @@ -18,7 +18,11 @@ def require_parallel_group_start(adapter): require_group_start(adapter) - if getattr(adapter, "supports_parallel_group_start", False) is not True: + try: + supported = adapter.supports_parallel_group_start + except AttributeError: + supported = False + if supported is not True: raise ValueError( "parallel GRPO requires adapter.supports_parallel_group_start = True; " "its portable group-start tokens must work in independent adapter instances" @@ -35,13 +39,14 @@ def _serve(connection, spec, threads): adapter = load_online_adapter(spec) require_parallel_group_start(adapter) connection.send((request, True, adapter_identity(adapter))) + operations = {"reset_group": adapter.reset_group, "step": adapter.step} while True: request, method, args = connection.recv() if method == "close": break - if method not in {"reset_group", "step"}: + if method not in operations: raise ValueError(f"unsupported simulator operation: {method}") - connection.send((request, True, getattr(adapter, method)(*args))) + connection.send((request, True, operations[method](*args))) except (EOFError, BrokenPipeError): pass except BaseException: diff --git a/src/sequifier/rl/window_storage.py b/src/sequifier/rl/window_storage.py index cf194563..36cdb3dc 100644 --- a/src/sequifier/rl/window_storage.py +++ b/src/sequifier/rl/window_storage.py @@ -20,7 +20,7 @@ def tensor_storage_bytes(value, seen=None): return sum(tensor_storage_bytes(item, seen) for item in value.values()) if isinstance(value, TensorRecord): return sum( - tensor_storage_bytes(getattr(value, field.name), seen) + tensor_storage_bytes(vars(value)[field.name], seen) for field in fields(value) ) storage = value.untyped_storage() diff --git a/src/sequifier/rollout/factories.py b/src/sequifier/rollout/factories.py index b7d84bc6..9b04e3b2 100644 --- a/src/sequifier/rollout/factories.py +++ b/src/sequifier/rollout/factories.py @@ -1,6 +1,5 @@ """Persistent adapter loading; factories receive their config as keyword arguments.""" -from importlib import import_module from typing import cast from sequifier.config.rollout_config import AdapterConfig @@ -9,26 +8,39 @@ ContractVersions, require_contract_version, ) +from sequifier.integration.factories import load_factory from sequifier.rollout.contracts import EnvironmentAdapter def validate_adapter(adapter): - if getattr(adapter, "protocol", None) != FORECAST_ROLLOUT_PROTOCOL: + try: + protocol = adapter.protocol + except AttributeError: + protocol = None + if protocol != FORECAST_ROLLOUT_PROTOCOL: raise ValueError(f"adapter.protocol must be {FORECAST_ROLLOUT_PROTOCOL!r}") - protocol_version = getattr(adapter, "protocol_version", None) + try: + protocol_version = adapter.protocol_version + except AttributeError: + protocol_version = None require_contract_version( "forecast rollout adapter protocol", protocol_version, ContractVersions.FORECAST_ROLLOUT_ADAPTER, ) - for method in ("reset", "step", "close"): - if not callable(getattr(adapter, method, None)): - raise ValueError(f"rollout adapter requires a callable {method} method") + try: + methods = (adapter.reset, adapter.step, adapter.close) + except AttributeError as error: + raise ValueError( + f"rollout adapter requires a callable {error.name} method" + ) from error + for name, method in zip(("reset", "step", "close"), methods): + if not callable(method): + raise ValueError(f"rollout adapter requires a callable {name} method") def load_adapter(spec: AdapterConfig) -> EnvironmentAdapter: - module, attribute = spec.factory.split(":") - factory = getattr(import_module(module), attribute) + factory = load_factory(spec.factory) if not callable(factory): raise ValueError(f"Adapter factory {spec.factory!r} is not callable") # Runtime validation happens inside collect_rollout's cleanup boundary. diff --git a/src/sequifier/train.py b/src/sequifier/train.py index 3b152c50..f31bbf5e 100644 --- a/src/sequifier/train.py +++ b/src/sequifier/train.py @@ -47,6 +47,7 @@ class LoadedInferenceModel: config: Any interface_name: str embedding: bool + autoregressive_targets: frozenset[str] = frozenset() @property def interface(self) -> Any: @@ -275,6 +276,19 @@ def load_inference_model( if isinstance(module, (nn.Dropout, nn.MultiheadAttention, SelfAttention)): module.train() + from sequifier.model.decoders import AutoregressiveTransformerDecoderBranch + + route = network.resolve_interface(interface_name) + autoregressive_targets = ( + frozenset( + target + for branch in route.decoder.branches.values() + if isinstance(branch, AutoregressiveTransformerDecoderBranch) + for target in branch.target_columns + ) + if route.decoder is not None + else frozenset() + ) if not device.startswith("mps"): network = torch.compile(network) if model_type not in {"generative", "embedding"}: @@ -284,6 +298,7 @@ def load_inference_model( config=config, interface_name=interface_name, embedding=model_type == "embedding", + autoregressive_targets=autoregressive_targets, ) diff --git a/src/sequifier/training/distributed_strategy.py b/src/sequifier/training/distributed_strategy.py index 61e356b4..482c8688 100644 --- a/src/sequifier/training/distributed_strategy.py +++ b/src/sequifier/training/distributed_strategy.py @@ -155,8 +155,14 @@ def prepare_network(self, network: nn.Module) -> PreparedNetwork: if self.cpu_offload: kwargs["offload_policy"] = CPUOffloadPolicy() seen: set[int] = set() - backbone = getattr(network, "backbone", None) - for layer in getattr(backbone, "layers", ()): + from sequifier.model.network import ComposableTransformerNetwork + + layers = ( + network.backbone.layers + if isinstance(network, ComposableTransformerNetwork) + else () + ) + for layer in layers: if id(layer) not in seen: fully_shard(layer, **kwargs) seen.add(id(layer)) diff --git a/src/sequifier/training/loss.py b/src/sequifier/training/loss.py index 7e07c4ac..6d18edf4 100644 --- a/src/sequifier/training/loss.py +++ b/src/sequifier/training/loss.py @@ -149,7 +149,7 @@ def calculate( for target in target_names: kind = interface.target_column_types[target] if ( - target in getattr(interface, "categorical_hashing", {}) + target in interface.categorical_hashing and target not in output.auxiliary_logits ): raise RuntimeError( @@ -172,9 +172,11 @@ def calculate( target_for_loss = lookup[safe_global_ids] excluded = target_for_loss < 0 if target in output.auxiliary_logits: - codec = network.resolve_interface( - dataset.interface_name - ).decoder.hash_codecs[target] + codec = ( + network.resolve_interface(dataset.interface_name) + .require_decoder() + .hash_codec(target) + ) if bool((excluded & flat_mask).any()): raise ValueError( f"Categorical target {target!r} contains excluded special " @@ -275,7 +277,7 @@ def calculate( location = denormalize_values(location, statistics) observed = denormalize_values(observed, statistics) distribution = RealPredictionDistribution( - location, route.prediction_scales.scale(target) + location, route.prediction_scale(target) ) sums[target] = -distribution.log_prob(observed).sum() elif isinstance(criterion, AccurateCrossEntropyLoss): diff --git a/src/sequifier/training/metrics.py b/src/sequifier/training/metrics.py index 5641b43f..316c2968 100644 --- a/src/sequifier/training/metrics.py +++ b/src/sequifier/training/metrics.py @@ -12,6 +12,7 @@ import numpy as np from loguru import logger +from torch import Tensor from sequifier.contract_versions import ContractVersions, require_contract_version from sequifier.logging_paths import dataset_artifact_prefix @@ -108,8 +109,9 @@ def _timestamp_utc() -> str: @beartype def _metric_value(value: Any) -> float: """Convert scalar tensor/NumPy/Python values without display rounding.""" - item = getattr(value, "item", None) - scalar: Any = item() if callable(item) else value + scalar: Any = ( + value.item() if isinstance(value, (Tensor, np.generic, np.ndarray)) else value + ) return float(scalar) diff --git a/src/sequifier/training/optimization.py b/src/sequifier/training/optimization.py index a43b0bc7..076a7e7c 100644 --- a/src/sequifier/training/optimization.py +++ b/src/sequifier/training/optimization.py @@ -9,7 +9,7 @@ from torch import dtype, float16, nn from torch.amp.grad_scaler import GradScaler from torch.optim import Optimizer -from torch.optim.lr_scheduler import ReduceLROnPlateau +from torch.optim.lr_scheduler import OneCycleLR, ReduceLROnPlateau from sequifier.artifacts.run_checkpoint import OptimizationState from sequifier.config.optimizer_config import ( @@ -273,7 +273,7 @@ def step_scheduler(self, metric: float | None = None) -> bool: self.skip_next_scheduler_step = False return False if ( - hasattr(self.scheduler, "total_steps") + isinstance(self.scheduler, OneCycleLR) and self.scheduler.last_epoch >= self.scheduler.total_steps ): return False diff --git a/src/sequifier/training/runtime.py b/src/sequifier/training/runtime.py index cb72a4c2..aa97b575 100644 --- a/src/sequifier/training/runtime.py +++ b/src/sequifier/training/runtime.py @@ -24,6 +24,7 @@ ) from sequifier.helpers import normalize_path from sequifier.io.batch import SequifierBatch +from sequifier.io.folder_dataset import FolderDataset from sequifier.io.memory_budget import DataLoadingBudget from sequifier.io.sequifier_dataset_from_file import SequifierDatasetFromFile from sequifier.io.sequifier_dataset_from_folder_parquet import ( @@ -413,9 +414,9 @@ def _loaders(self, split: Literal["training", "validation"]): @beartype def set_epoch(self, epoch: int, split: Literal["training", "validation"]): for _, loader in self._loaders(split): - set_epoch = getattr(loader.dataset, "set_epoch", None) - if callable(set_epoch): - set_epoch(epoch) + dataset = loader.dataset + if isinstance(dataset, (SequifierDatasetFromFile, FolderDataset)): + dataset.set_epoch(epoch) @beartype def iter_batches( @@ -480,7 +481,7 @@ def _criterion_modules( criterion_class = ( AccurateCrossEntropyLoss if dataset.criterion[target] == "AccurateCrossEntropyLoss" - else getattr(torch.nn, dataset.criterion[target]) + else vars(torch.nn)[dataset.criterion[target]] ) kwargs: dict[str, Any] = {"reduction": "none"} if dataset.class_weights is not None and target in dataset.class_weights: diff --git a/src/sequifier/trajectory/export.py b/src/sequifier/trajectory/export.py index 5775f6be..5771bed3 100644 --- a/src/sequifier/trajectory/export.py +++ b/src/sequifier/trajectory/export.py @@ -197,10 +197,9 @@ def flush(): ), GROUP_COLUMN: [episode.group_id] * length, REWARD_COLUMN: transitions.rewards or [None] * length, - **{ - field: getattr(transitions, field) or [None] * length - for field in TRANSITION_FLAGS - }, + "terminated": transitions.terminated or [None] * length, + "truncated": transitions.truncated or [None] * length, + "bootstrap_allowed": transitions.bootstrap_allowed or [None] * length, } episode_types = dict(types) for name, column in zip( diff --git a/src/sequifier/trajectory/importers.py b/src/sequifier/trajectory/importers.py index 9ac97fce..d17d0024 100644 --- a/src/sequifier/trajectory/importers.py +++ b/src/sequifier/trajectory/importers.py @@ -51,16 +51,11 @@ def sequence_from_trajectory( ), static_inputs=episode.seed.static_inputs, transitions=TransitionColumns( - **{ - field: tuple(getattr(step, attribute) for step in steps) - for field, attribute in { - "rewards": "reward", - "terminated": "terminated", - "truncated": "truncated", - "successors": "successor_observation", - "bootstrap_allowed": "bootstrap_allowed", - }.items() - } + rewards=tuple(step.reward for step in steps), + terminated=tuple(step.terminated for step in steps), + truncated=tuple(step.truncated for step in steps), + successors=tuple(step.successor_observation for step in steps), + bootstrap_allowed=tuple(step.bootstrap_allowed for step in steps), ), ) diff --git a/src/sequifier/visualize_training.py b/src/sequifier/visualize_training.py index ec6ec141..dfab5bd3 100644 --- a/src/sequifier/visualize_training.py +++ b/src/sequifier/visualize_training.py @@ -553,7 +553,7 @@ def generate_html_report( output_dir = os.path.join(args.project_root, "outputs", "visualization") os.makedirs(output_dir, exist_ok=True) - yaxis_type = "log" if getattr(args, "log_scale", False) else "linear" + yaxis_type = "log" if args.log_scale else "linear" if len(models) == 1 and report_name is None: model = models[0] @@ -584,7 +584,7 @@ def visualize_training(args: argparse.Namespace) -> None: if not models: raise ValueError("No models provided to visualize.") - bucket_batches = getattr(args, "bucket_training_batches", None) + bucket_batches = args.bucket_training_batches all_data: dict[str, Any] = {} invalid_runs: dict[str, str] = {} diff --git a/tools/check_contract_versions.py b/tools/check_contract_versions.py index f2cef2a1..5ca3e64a 100644 --- a/tools/check_contract_versions.py +++ b/tools/check_contract_versions.py @@ -84,7 +84,7 @@ def hardcoded_versions(tree): assignments = () if isinstance(node, ast.Dict): assignments = ( - (getattr(key, "value", None), value) + (key.value if isinstance(key, ast.Constant) else None, value) for key, value in zip(node.keys, node.values) ) elif isinstance(node, ast.Assign): diff --git a/tools/stream_folder_via_scp.py b/tools/stream_folder_via_scp.py index f5bcf543..5071394f 100644 --- a/tools/stream_folder_via_scp.py +++ b/tools/stream_folder_via_scp.py @@ -69,6 +69,7 @@ def main(): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + sftp = None try: ssh.connect(args.remote_host, username=args.remote_user, password=password) @@ -125,10 +126,9 @@ def main(): except Exception as e: print(f"Error: {e}") finally: - if "sftp" in locals(): - sftp.close() # type: ignore - if "ssh" in locals(): - ssh.close() + if sftp is not None: + sftp.close() + ssh.close() if __name__ == "__main__":