From 21809ca6597cfda1331bea8f7be034802f217a74 Mon Sep 17 00:00:00 2001 From: aldbr Date: Tue, 30 Jun 2026 18:14:18 +0200 Subject: [PATCH 1/7] docs: include ADRs for the migration --- docs/adr/IC-ADR-001_computing_elements.md | 964 ++++++++++++++++++++++ docs/adr/IC-ADR-002_integration_tests.md | 398 +++++++++ docs/adr/IC-ADR-003_credentials.md | 301 +++++++ docs/adr/IC-ADR-XXX_template.md | 45 + docs/adr/index.md | 13 + docs/notes/reuse-survey.md | 68 ++ 6 files changed, 1789 insertions(+) create mode 100644 docs/adr/IC-ADR-001_computing_elements.md create mode 100644 docs/adr/IC-ADR-002_integration_tests.md create mode 100644 docs/adr/IC-ADR-003_credentials.md create mode 100644 docs/adr/IC-ADR-XXX_template.md create mode 100644 docs/adr/index.md create mode 100644 docs/notes/reuse-survey.md diff --git a/docs/adr/IC-ADR-001_computing_elements.md b/docs/adr/IC-ADR-001_computing_elements.md new file mode 100644 index 0000000..b7535cd --- /dev/null +++ b/docs/adr/IC-ADR-001_computing_elements.md @@ -0,0 +1,964 @@ +# IC-ADR-001: Computing Element interfaces and the DIRAC to interCEde migration + +## Metadata + +- **Created By:** Alexandre Boyer +- **Date:** 2026-06-30 +- **Status:** Draft +- **Decision Maker(s):** Federico Stagni, Christophe Haen, Chris Burr +- **Stakeholders:** DIRAC/DiracX SiteDirector, PushJobAgent and Pilot maintainers; extension + communities operating Computing Elements and batch systems; interCEde contributors + +> **Scope and altitude.** This is a *direction-setting* ADR. It commits to the **shape** of +> interCEde's Computing Element interfaces and records what is ported from DIRAC's +> `Resources/Computing` and what is deliberately left out. Illustrative code sketches pin down the +> shape; the exhaustive method-by-method signatures, the wire/serialisation details, and the +> per-backend porting mechanics are deferred to follow-up ADRs and to the implementation itself. +> interCEde is a standalone library in the [DIRACGrid](https://github.com/DIRACGrid) ecosystem +> designed to back the Computing Element layer used by +> [DiracX](https://github.com/DIRACGrid/diracx); this ADR is intended to be read alongside the +> DiracX transition ADRs but governs interCEde's own API. (Why a standalone library rather than a +> `diracx` subpackage is recorded in Rejected Ideas.) +> +> **How to read this.** To review the decision, read the Abstract, Motivation, §1–§4 and §7. +> The rest is supporting reference and can be skimmed: §8 (migration map), §9 (consumer sketches), +> the Rejected-Ideas entries you don't want to contest, and the Open Issues backlog. + +## Abstract + +interCEde gives the workload management layer one payload-agnostic way to **submit, monitor and +retrieve** units of work on heterogeneous Computing Elements (CEs) and batch systems, validated +against containerised backends. This ADR fixes the shape of that interface with five decisions: + +1. **The contract is a set of small typed interfaces (`typing.Protocol`s), not a base class.** + Backends are defined by structural conformance, implement only the capabilities they have, and + can be mocked without inheriting interCEde types. +2. **Access and scheduling compose instead of multiplying.** A `Transport` (how you reach the + resource) and a `Scheduler` (how work is queued) combine in one generic `BatchBackend`, so N + transports × M schedulers costs N+M classes; genuinely monolithic backends (ARC, HTCondor-CE, + Cloud) implement the contract directly. +3. **DIRAC's in-process CEs (`InProcess`, `Singularity`, `Pool`) leave the library.** They belong + to the pilot/worker-node domain and get their own `Runner`/`ExecutorPool` subsystem, + disconnected from this contract. +4. **The data plane handles arbitrary input/output sandboxes**, not only stdout/stderr — the + direct consequence of never interpreting the payload. +5. **The extension surface is explicit and three-tiered:** stable contracts third parties + implement, provided implementations they may use but not subclass, and internals with no + stability guarantee. + +## Motivation + +### What DIRAC has today + +DIRAC's CE machinery lives in `DIRAC/Resources/Computing/`. A base class `ComputingElement` is +subclassed by nine concrete CEs (`AREX`, `HTCondorCE`, `SSH`, `SSHBatch`, `Local`, `Cloud`, +`InProcess`, `Singularity`, `Pool`), instantiated through a `ComputingElementFactory` that +string-loads modules via `ObjectLoader`. A parallel `BatchSystems/` package (`SLURM`, `Condor`, +`LSF`, `GE`, `OAR`, `Torque`, `Host`) provides scheduler plugins that the `SSH`/`Local` CEs +compose via `loadBatchSystem()`. + +### Why it needs to change + +1. **The base class is a fat class that conflates contract with policy.** `ComputingElement` + mixes the job-lifecycle contract (submit/status/output/kill) with cross-cutting policy: + CPU-slot accounting (`available()`), proxy renewal (`_monitorProxy`), and a layered + configuration hierarchy. Every CE drags this along whether it needs it or not. + +2. **The contract is "abstract by convention", and unenforced.** There is no `abc.ABC`, no + `@abstractmethod`, no Protocol — it is a plain `class ComputingElement:`. What stubs exist + (`submitJob`, `getCEStatus`) merely `return S_ERROR("…should be implemented in a subclass")`; + the rest of the lifecycle (`getJobStatus`, `getJobOutput`, `killJob`) is not declared on the + base **at all**, so nothing — not even a stub — stops a subclass from silently omitting + `getJobOutput`. + +3. **The interface has drifted across implementations.** `submitJob` alone appears as + `(executableFile, proxy, numberOfJobs, inputs, outputs)` (AREX), `(…, numberOfJobs=1)` + (HTCondorCE/SSH), `(…, proxy=None, inputs=None, **kwargs)` (InProcess/Pool), and + `(…, proxy=None, **kwargs)` (Singularity); `proxy` is required in some and optional in others. + `getJobOutput` appears with four different signatures. Not all CEs implement the same methods. + +4. **Three incompatible semantics are crammed under one base.** The base docstring itself + distinguishes **Remote** (async, pollable, multi-job), **Inner** (synchronous, blocking, one + job), and **Inner Pool** CEs. They serve *different actors* — Remote CEs are driven by the + central SiteDirector/PushJobAgent, Inner CEs by the JobAgent inside a pilot — and are never + substituted for one another. Their *mechanics* don't even overlap (`CGroups2.systemCall` + + periodic proxy refresh on the inner side vs batch submit/poll/REST on the remote side). The + shared supertype is a false abstraction. + +5. **The good idea — composition — is applied inconsistently.** `SSH`/`Local` compose a transport + with a `BatchSystems/*` plugin, but `AREX`/`HTCondorCE`/`Cloud` hard-code their scheduler + interaction, and the `BatchSystems/*` plugins are a separate hierarchy with their own ad-hoc, + string-generating interface. + +6. **The data plane is too narrow.** Only `AREXComputingElement` retrieves arbitrary output files + (it lists the ARC session directory and streams every file to disk — though even it still + *returns* only stdout/stderr in memory, `S_OK((output, error))`); every other CE handles only + stdout/stderr, returning their contents in memory (SSH/Local can instead write them to a + directory and return paths, but nothing generalises to an arbitrary sandbox). Input staging is + similarly limited. A payload-agnostic library cannot assume payloads emit only stdout/stderr. + +7. **Untyped results and discovery.** Everything is `S_OK`/`S_ERROR` dicts; backends are + discovered by string-to-module loading. Neither is checkable or pluggable in a typed, + third-party-friendly way. + +### Drivers + +- **Payload-agnostic.** interCEde must not care whether the payload is a DIRAC job, a DIRAC + pilot, or anything else. It submits an opaque executable + sandbox and lets the caller + poll/fetch/kill. +- **Orchestration-agnostic.** Whether work is *pulled* (SiteDirector) or *pushed* (PushJobAgent) + is a workload management system concern above interCEde. +- **One contract, many backends**, with *combinations* (`SSH + Slurm`, `SSH + HTCondor`, + `Local + HTCondor`) as first-class. +- **Tested against the real thing** — backends verified against containerised schedulers, so the + contract must be trivially implementable and mockable. +- **Extensible by third parties** — VOs must be able to add their own transports, schedulers and + CEs without forking interCEde. +- **Typed and async** — DiracX is async; results and errors should be typed. + +## Specification + +### 1. Scope — what interCEde is and is not + +interCEde is the **delegate-and-poll** side of running work on a resource: *submit a payload to a +scheduler/resource, get a handle, then poll it, fetch its sandbox, or kill it.* The boundary with +the rest of the system, stated without any pilot/job vocabulary: + +- **In scope:** reaching a resource (local shell, SSH, REST/ARC, HTCondor-CE, cloud API); + queueing work on a scheduler; tracking a remote handle; staging input/output sandboxes. +- **Out of scope — execute-here-and-now:** running a payload *in this process* on a worker node + (DIRAC's `InProcess`/`Singularity`/`Pool`). This is the pilot/worker-node domain (see §6). +- **Out of scope — orchestration:** pull vs push, matching, pilot lifecycle. interCEde exposes + mechanism; DiracX/DIRAC decides policy. +- **Out of scope — payload interpretation:** interCEde never inspects payload contents. +- **Out of scope — persistence:** interCEde keeps no store of jobs or handles; the caller + persists handles and owns idempotency/reconciliation (see §9). +- **Out of scope — other resource types:** Storage Elements, File Catalogs, and the like are + *different contracts* (data movement; namespace/metadata) with different backends, testing, and + consumers. They are **sibling libraries, not part of interCEde** — see Rejected Ideas. + +The submitted thing is a **`SubmissionSpec`** — the description of an opaque payload (name +provisional; see Open Issues). Submitting a spec, optionally `count` identical times, returns one +**`Submission`**, which bundles one **`JobHandle`** per created job plus any per-copy failures. + +**Throughout this ADR, "job" means the *scheduler's* job (a Slurm/Condor job) — explicitly *not* +a DIRAC Job.** The naming deliberately reserves `Job*` for the backend handle and avoids +`Payload`, which collides with the pilot-side payload concept (see Open Issues). + +A `JobHandle` is the durable, serialisable identity of one job: the caller persists it and later +replays it into status/output/kill calls. It must therefore be stable across processes and carry +whatever *routing* the backend needs to find the job again (e.g. the target host that DIRAC's +`sshcondor://…` scheme used to encode — see §8). + +### 2. The contract — capability-segmented Protocols + +The caller-facing contract is a set of small `typing.Protocol`s, and **every operation works on +many jobs at once**: submit fans a single spec into `count` identical jobs; status/output/kill/ +purge take a sequence of ids and return a mapping keyed by `JobID`, so every per-item outcome is +addressable (see the partial-failure rule below). + +The contract is sized by its consumers. DiracX splits the old monolithic SiteDirector into +independent, separately-scheduled **tasks** — one submits, one polls status, one fetches outputs +(§9 sketches them). The **essential** capabilities of a CE are two — *submit a payload (with +inputs)* and *get the jobs' status* — each driven by its own task. *Retrieving outputs* is a +third task and its own protocol, but it is **optional, not essential**: some backends genuinely +cannot serve a pull-style sandbox retrieval — the canonical case is **Cloud**, where a booted VM +has no server-side filesystem interCEde controls, so exporting outputs is the payload's own +responsibility (e.g. pushing them to an object store) — and the ADR's rule is *never force a +backend to stub a capability it lacks*. So `OutputRetriever` sits with the optional capabilities, +and the output task narrows to it structurally. + +```python +StatusMap = Mapping[JobID, JobStatus] # named alias, stable (Tier A) + +@runtime_checkable +class Submitter(Protocol): # the submission task + async def submit(self, spec: SubmissionSpec, count: int = 1) -> Submission: ... + # submit the SAME spec `count` times (DIRAC's numberOfJobs); returns one Submission + # bundling one JobHandle per successful copy, plus per-copy failures + +@runtime_checkable +class Monitor(Protocol): # the status task + async def status(self, ids: Sequence[JobID]) -> StatusMap: ... + # ids the backend no longer knows come back as JobStatus.UNKNOWN — never dropped + +@runtime_checkable +class JobBackend(Submitter, Monitor, Protocol): ... +``` + +`OutputRetriever` and the other capabilities are additive and optional (below). `JobBackend` is +the composition of the two essential protocols; it names "a complete backend" and is what the +registry returns and validates. But **consumers depend on the narrow protocol they use, not on +`JobBackend`** — the submission task takes a `Submitter`, the status task a `Monitor`, the output +task an `OutputRetriever` (which it must confirm structurally, since a backend may lack it). A +backend implements the slices it can, and each task is handed the same object but sees only its +slice. + +> **Naming — why `JobBackend`, not `ComputingElement`.** interCEde types name *what interCEde +> provides* — a backend the WMS drives — never the external resource. The composed contract is +> `JobBackend` (it pairs with `JobHandle`/`JobID`/`JobStatus`); a Slurm-over-SSH `BatchBackend` +> is emphatically *not* a Computing Element in the WLCG sense, and the concrete drivers +> (`ARCBackend`, `HTCondorCEBackend`, `CloudBackend`) are clients *of* resources, not the +> resources themselves. The domain term "Computing Element / CE" stays in prose and docstrings. +> The one retained "CE" is `HTCondorCEBackend`: there "CE" names the external product +> (HTCondor-CE, a grid gateway distinct from the local `HTCondorScheduler`), not a claim that the +> class *is* a CE. + +> **The registry gate is a smoke-test, not the drift guard.** `isinstance` against a +> `@runtime_checkable` Protocol checks method **presence only, never signatures** — a backend +> whose `submit` has an extra required argument, or that forgot `async`, passes the gate and +> fails only at the first call. The real defence against DIRAC-style signature drift is (a) +> static typing for first-party code and (b) the **container conformance suite**, which is +> therefore a *requirement* of registration, not optional (see §5, Evolution). (Two PEP 544 +> mechanics for implementers: keep `Protocol` in the base list when composing protocols, or the +> subclass silently becomes a concrete class; and `@runtime_checkable` is not inherited — the +> composed `JobBackend` must carry the decorator itself.) + +**Everything else is optional** — additive, independent protocols, *not* part of `JobBackend`, +each justified by a real caller and by backends that have it and don't. All are bulk and return a +per-`JobID` outcome map (`kill`/`purge`/`fetch_output` never return bare `None`, so a +partial-batch failure is always reportable): + +```python +@runtime_checkable +class OutputRetriever(Protocol): # the (on-demand) output task — OPTIONAL + # Bulk: materialise each job's whole output sandbox (incl. the CE/scheduler log) into + # `dest`. No separate log fetch — the log is a manifest member. Retrieval is IDEMPOTENT + # by default (ARC/SSH/Local keep remote state and can be re-fetched); a backend whose + # retrieval is physically one-shot DECLARES it via `destructive` — e.g. HTCondor + # (>= 25.8), where a completed spooled job leaves the queue after `condor_transfer_data` + # (overridable via `leave_in_queue`). interCEde never *forces* destructiveness on a + # re-fetchable backend; the consumer reads `destructive` to know whether `dest` is a + # hard commit point or a re-fetchable cache. + destructive: bool # False for ARC/SSH/Local; True for HTCondor (>= 25.8) + async def fetch_output(self, ids: Sequence[JobID], dest: Path) -> Mapping[JobID, JobOutput]: ... + +@runtime_checkable +class Cancellable(Protocol): + async def kill(self, ids: Sequence[JobID]) -> Mapping[JobID, OpOutcome]: ... + +@runtime_checkable +class Purgeable(Protocol): + # Delete a job's outputs/remote state WITHOUT fetching them — the counterpart to the + # destructive fetch. Needed on its own where a site mandates scratch/spool cleanup + # even for outputs no one will retrieve. + async def purge(self, ids: Sequence[JobID]) -> Mapping[JobID, OpOutcome]: ... + +@runtime_checkable +class LoadReporter(Protocol): + # Aggregate counts of *our* submitted jobs — NOT a capacity number, NOT "available + # slots" (that is the caller's throttling policy). HTCondor-CE cannot provide this at + # all (`getCEStatus()` returns an explicit `S_ERROR("… not supported")`). + async def counts(self) -> JobCounts: ... # {running, waiting} + +@runtime_checkable +class SupportsLiveDiagnostics(Protocol): + # An independent, repeatable diagnostics fetch *before* completion — ARC is the + # clear case (its `diagnose/errors` endpoint); whether HTCondorCE's spool log-peek + # qualifies is an Open Issue. The *final* log is always a member of the + # fetch_output manifest, so there is no standalone "get the final log" method. + async def diagnostics(self, ids: Sequence[JobID]) -> Mapping[JobID, Diagnostics]: ... +``` + +The comment on `OutputRetriever.destructive` above is the **single normative statement** of the +destructive-retrieval model; every other mention in this ADR cross-references it. + +**Partial failure is per item, not per batch.** A bulk call raises a typed exception only for a +whole-operation failure (transport down, auth rejected); anything that can succeed for some ids +and fail for others reports it in the returned map — `status` yields `JobStatus.UNKNOWN` for +unknown/expired ids, and `kill`/`purge`/`fetch_output` yield a per-id `OpOutcome` (ok / error). +Callers never have to guess which ids a partial batch touched. + +**Backends have an explicit lifecycle, because consumers cache them.** DIRAC already proves both +halves: `QueueCECache` caches CE objects keyed on a hash of their parameters, reuses them across +agent cycles, and calls `shutdown()` on eviction (with an explicit "eviction must never fail" +guard), and the SSH CE's `shutdown()` closes per-host connections *and the ssh jump-gateway* — +yet the base class declares no lifecycle contract at all, so eviction relies on `shutdown()` +being "polymorphic by convention". interCEde backends and transports hold the same kind of state +(SSH connections, HTTP sessions), so the lifecycle is part of the Tier-A contract: **backends and +transports are async context managers** — construction is cheap and side-effect-free (no I/O +before first use), close is idempotent, and a consumer that caches backends owns closing them on +eviction. This is also what makes the statelessness rule (§9) honest: the *only* resources a +backend holds are connections, released deterministically, never job state. + +**Decision rule for segmentation:** one protocol per capability a *specific caller wants in +isolation*. The DiracX-task split makes that concrete — submit and status are two separate tasks +(so two essential protocols), and output-retrieval is a third task narrowed to an *optional* +`OutputRetriever` (an earlier draft bundled status+fetch into one "operator"; the task topology +shows that was too coarse). `kill` and `purge` are management actions some flows never use; the +submission task's throttling wants `LoadReporter`; the WMS matcher wants none of this. Do not +segment further than a caller exists for — that discipline is what prevents drift back into a +god-interface. + +```mermaid +classDiagram + direction LR + class Submitter { + <> + +submit(spec, count) Submission + } + class Monitor { + <> + +status(ids) StatusMap + } + class OutputRetriever { + <> + +fetch_output(ids, dest) OutputMap + } + class JobBackend { + <> + } + Submitter <|-- JobBackend + Monitor <|-- JobBackend + class Cancellable { + <> + +kill(ids) OutcomeMap + } + class Purgeable { + <> + +purge(ids) OutcomeMap + } + class LoadReporter { + <> + +counts() JobCounts + } + class SupportsLiveDiagnostics { + <> + +diagnostics(ids) DiagnosticsMap + } + class Transport { + <> + +run(argv) CommandResult + +put(local, remote) + +get(remote, local) + } + class Scheduler { + <> + +stages_own_files bool + +submit_cmd(spec) Command + +parse_status(raw) StatusMap + +kill_cmd(ids) Command + } + class BatchBackend + JobBackend <|.. BatchBackend + OutputRetriever <|.. BatchBackend + Cancellable <|.. BatchBackend + Purgeable <|.. BatchBackend + LoadReporter <|.. BatchBackend + BatchBackend o-- Transport : uses + BatchBackend o-- Scheduler : uses + Transport <|.. SSHTransport + Transport <|.. LocalTransport + Scheduler <|.. Slurm + Scheduler <|.. HTCondorScheduler + class ARCBackend + class HTCondorCEBackend + class CloudBackend + JobBackend <|.. ARCBackend + JobBackend <|.. HTCondorCEBackend + JobBackend <|.. CloudBackend + OutputRetriever <|.. ARCBackend + OutputRetriever <|.. HTCondorCEBackend + Cancellable <|.. ARCBackend + Purgeable <|.. ARCBackend + LoadReporter <|.. ARCBackend + SupportsLiveDiagnostics <|.. ARCBackend + Cancellable <|.. HTCondorCEBackend +``` + +(The two essential protocols compose into `JobBackend`; output-retrieval and the rest are +optional and picked up à la carte. `ARCBackend` adds `OutputRetriever` plus all four optional +capabilities. `HTCondorCEBackend` adds `OutputRetriever` and `Cancellable` only — it reports no +counts, offers no independent diagnostics, and its retrieval is destructive (§2). +`CloudBackend` implements only the essential `Submitter`+`Monitor` — it deliberately does *not* +claim `OutputRetriever` (§2).) + +### 3. Composition — `Transport` × `Scheduler` + +Most backends are really two orthogonal choices: **access/transport** (how you reach the +resource) and **scheduler** (what queues the work). `Transport` and `Scheduler` are +*collaborator* protocols — they are **not** sub-protocols of `JobBackend`. Their relationship to +the contract is realised by composition: + +```python +class BatchBackend: # satisfies JobBackend + def __init__(self, transport: Transport, scheduler: Scheduler): ... +``` + +This one class covers `SSH + Slurm`, `SSH + HTCondor`, `Local + Slurm`, … — the matrix collapses +from *transports × schedulers* to *transports + schedulers*. Backends that are not +"transport + scheduler" (ARC is a REST service fronting an opaque batch system; Cloud boots a VM +and has no scheduler) implement `JobBackend` directly. The Protocol is what lets composed and +monolithic implementations sit behind one caller-facing type. + +Cloud is the clearest case of *why* composition does not always apply. Batch has **two +independent axes** — any `Scheduler` behind any `Transport` — which is what makes the N×M → N+M +collapse worthwhile. Cloud has **one** axis, the provider (OpenStack/EC2/OpenNebula/…), and "how +you reach it" and "what it is" are *fused* into a single Apache Libcloud driver — there is no +matrix to collapse. So `CloudBackend` is **not** `BatchBackend(libcloud, opennebula)`: Libcloud +is not a `Transport` (no shell to `run` argv, no filesystem to `put`/`get`) and a provider like +OpenNebula is not a `Scheduler` (no queue, no submit/parse-status commands). The provider axis +*is* a composition, but Libcloud already owns it (its `get_driver`/`set_driver` registry); +`CloudBackend` is a monolithic `JobBackend` that composes Libcloud **internally** (Tier C, §8), +which is why the provider drivers do not appear as collaborators in the class diagram. + +**Staging is a Transport × Scheduler interaction, and the `Scheduler` carries it.** Who moves the +sandbox is not purely a transport choice: HTCondor declares its file transfer *in the submit +description at submit time* (`should_transfer_files`, `transfer_output_files`) and stages +worker→schedd itself, whereas a Slurm-over-SSH combination declares nothing to the scheduler and +interCEde stages the sandbox over the transport (`put`/`get`) at fetch time. `BatchBackend` must +not hard-code this per scheduler, so the `Scheduler` protocol exposes it as data — a +`stages_own_files` flag (and, where needed, `stage_inputs`/`collect_outputs` hooks). +`BatchBackend.submit()` consults the flag instead of branching on scheduler identity, which is +what keeps "add a scheduler by implementing the protocol, no core change" true even for a +scheduler whose transfer model differs from the reference one. + +**Shared batch-system knowledge.** A backend family's command construction and output parsing — +building a `condor_submit` description, parsing `condor_q`/`condor_history` ClassAds, mapping +native states to `JobStatus` — lives in *one* internal module (`_htcondor`, `_slurm`, …) consumed +by **both** the `Scheduler` adapter and any related monolithic CE. `HTCondorScheduler` (used by +`BatchBackend` for SSH/Local + HTCondor) and `HTCondorCEBackend` (a remote schedd reached over a +grid/CE transport, with tokens and the grid universe) share that core and differ only in +transport/auth and submit-description routing. That overlap is itself a hint that +`HTCondorCEBackend` might eventually be expressed as +`BatchBackend(HTCondorCETransport, HTCondorScheduler)` rather than a monolith (see Open Issues). +The rule is the same one from §6: **share implementation as functions/internal modules, never by +making one a subtype of the other.** + +**No interpreter on the remote host.** DIRAC's SSH/Local CEs work by *shipping* a Python driver +to the resource (`BatchSystems/.py` concatenated with `executeBatch.py`, uploaded and executed +by whatever Python the host provides — the file itself warns "support for py2 and py3 is +necessary"), which is why those drivers are stdlib-only, DIRAC-free, and stuck on 2014-era +conventions. interCEde deliberately drops the ship-and-exec model: `Scheduler`s build commands +*locally* and only argv (plus staged sandbox files) crosses the `Transport`. The remote host +needs the scheduler CLI and a shell — no Python, no shipped code, no py2 constraint. This is a +portability gain worth advertising and a behavioural change worth testing: the integration suite +should assert it with a batch-host variant that has no usable interpreter (IC-ADR-002 §7). + +### 4. Data plane — payload and sandbox + +DIRAC's stdout/stderr-only model is generalised to **arbitrary input/output sandboxes**. This is +not a feature bolt-on; it is the *same requirement* as payload-agnosticism — if interCEde does +not interpret the payload, it cannot assume the payload emits only stdout/stderr. stdout/stderr +become two well-known **members** of the output sandbox; the executable becomes one distinguished +member of the input sandbox. + +```python +@dataclass +class SubmissionSpec: # the opaque payload (name provisional — see Open Issues) + executable: str + arguments: list[str] + input_sandbox: list[FileRef] # files/dirs (or streams) staged in before the run + output_sandbox: OutputSpec # explicit names | globs | "everything produced" + resources: Resources + environment: Mapping[str, str] + tag: str | None = None # consumer-set token, reserved for post-crash + # reconciliation (list-by-tag — see Open Issues) +``` + +Rules: + +- **Declared at submit, materialised at fetch.** HTCondor needs `transfer_output_files` *in the + submit description*, so the output-sandbox spec lives in `SubmissionSpec`, not only as a + fetch-time argument. Permissive backends (ARC lists the session dir; SSH globs the remote + workdir) may also discover outputs at fetch time, so `OutputSpec` expresses both explicit lists + and globs/"all". +- **Caller-owned inputs are consumed at submit.** `submit()` stages (or copies) the executable + and every `input_sandbox` member *during the call* and retains no reference to caller-owned + paths after it returns — the caller may delete its temp files immediately. This rule removes a + DIRAC wart by construction: today the SiteDirector deletes the submitted executable *unless* + the CE returns `ExecutableToKeep` (HTCondorCE does, because it still needs the file on disk + afterwards), a hidden ownership handshake no other CE participates in. +- **Return paths, not contents — and stream.** `fetch_output(ids, dest)` streams each member into + `dest` and returns, per job, a **manifest** (`JobOutput` with `.stdout`, `.stderr`, + `.files[...]`, and `.log`); it never loads file contents into memory. This is the one model + that survives multi-GB outputs. +- **Staging is delegated or synthesised.** "Stage a sandbox" is either delegated to the backend's + native mechanism (ARC session dir, HTCondor file transfer) or synthesised by interCEde over the + transport (`Transport.put`/`get`), selected by the scheduler's `stages_own_files` flag (§3). + The composed CEs get arbitrary sandboxes *for free* because the transport already provides + `put`/`get`. +- **One manifest, one retrieval.** `fetch_output` materialises stdout, stderr, output files + **and** the CE/scheduler log together into `dest`, as members of a single manifest. The + manifest separates **CE/scheduler artifacts** (the log — infra-level, interCEde's to expose) + from **payload artifacts** (fetched, never interpreted). Whether the fetch also *releases* the + remote state is the backend's declared `destructive` property (§2) — when it is declared, the + consumer must treat the local `dest` as the commit point (§9). There is no standalone "get the + final log" call; a *while-running* diagnostics fetch exists only where it is genuinely + independent and non-destructive (`SupportsLiveDiagnostics`, §2). `purge(ids)` on its own still + deletes a job's outputs/remote state *without* fetching — the path a site takes when it + mandates scratch cleanup for outputs no one will retrieve (`Purgeable`, §2). +- **Bounded materialisation, enforced where interCEde does the copy.** "Everything produced" plus + arbitrary filenames from a payload interCEde never inspects means `fetch_output` needs a size + ceiling, a file-count ceiling, a per-transfer timeout, and path containment (every member lands + **under** `dest`; `..`/absolute members are rejected). How much is enforceable inline depends + on who does the copy: where interCEde streams file-by-file itself (ARC, SSH) all four limits + apply mid-stream; where the backend's own tool does the transfer (HTCondor) enforcement is + coarser — a submit-time allow-list, a timeout on the transfer command, and a post-transfer + size/count check. The limits are stated per backend rather than assumed uniform. + +Results are typed dataclasses/models; whole-operation failures raise a **typed exception +hierarchy** (replacing `S_OK`/`S_ERROR`) while per-item outcomes are returned in the bulk map +(§2). The contract is **`async`, and async is the single source of truth**: the backends are all +I/O-bound (SSH, ARC/cloud REST, subprocess) and the contract is bulk, so async is the natural +model for polling/fetching many jobs concurrently. A sync-only backend library (the `htcondor` +bindings, Libcloud, a batch CLI) is wrapped in a thread internally (`asyncio.to_thread`), never +surfaced as a second contract — backends implement only the async protocols. Sync *callers* +(DIRAC during the transition) are served by a thin sync **facade** that runs the async calls to +completion, **not** by a parallel sync API (see Open Issues); a sync caller still benefits from +the bulk fan-out, which runs concurrently *inside* each run-to-completion call. + +### 5. Discovery — registry and entry points + +`ComputingElementFactory` + `ObjectLoader` string-to-module loading is replaced by a typed +**registry** populated through Python **entry points**. Three groups, one per pluggable kind: + +```toml +# a backend package's pyproject.toml +[project.entry-points."intercede.backends"] +arc = "intercede.backend.arc:ARCBackend" +htcondor-ce = "intercede.backend.htcondor:HTCondorCEBackend" + +[project.entry-points."intercede.transports"] +ssh = "intercede.transport.ssh:SSHTransport" +local = "intercede.transport.local:LocalTransport" + +[project.entry-points."intercede.schedulers"] +slurm = "intercede.scheduler.slurm:Slurm" +htcondor = "intercede.scheduler.htcondor:HTCondorScheduler" +``` + +How it resolves: + +- **Lazy, typed lookup.** The registry reads the entry-point groups on first use but imports only + the *one* target a request names — no eager import of every backend. Each resolved object is + checked for structural conformance (`isinstance` against the relevant `@runtime_checkable` + protocol) at the boundary, so a misregistered class fails fast with a clear error instead of at + the first method call. +- **A CE is described by data, not a class name.** A request is a small typed config — + `{"type": "arc", ...}` for a monolithic backend, or `{"transport": "ssh", "scheduler": "slurm", + ...}` for a composed one. For the composed form the registry instantiates the named transport + and scheduler and wraps them in `BatchBackend`; the *combination* never needs its own + registered class, which is what keeps the matrix at N+M. +- **Third parties register without forking.** A VO ships a package advertising any of the three + groups (an in-house scheduler, a site-specific transport, a bespoke CE); it is discovered + automatically once installed in the same environment. Nothing in interCEde is edited, and the + package depends only on Tier-A protocols. +- **Discoverable and versioned.** Unlike `ObjectLoader`, the mapping from a short stable name + (`"slurm"`) to an implementation is owned by the providing package's metadata, versioned with + it, and enumerable (`importlib.metadata.entry_points`) for tooling and diagnostics ("what + backends does this environment offer?"). +- **Explicit override precedence.** Name collisions resolve by a documented rule (e.g. a + configured allow-list / last-installed-wins) so a site can shadow a built-in scheduler with its + own without patching interCEde. + +### 6. The severed "inner" subsystem (out of scope, recorded for completeness) + +`InProcess`, `Singularity` and `Pool` move to the **pilot/worker-node domain** with their own +contract, completely disconnected from `JobBackend`, and deliberately **not** modelled as +backends at all (in DIRAC they were lumped under the same `ComputingElement` base, and that +shared name is what created the false unity). Their relationship is a **depth-1 aggregator** — +*not* the full (recursive) Composite pattern, because pools-of-pools are unwanted: + +- `Runner`: execute one payload. `HostRunner` (was `InProcess`) runs it directly; + `ContainerRunner` (was `Singularity`) is a **decorator** that wraps another `Runner`'s command + in a container. Decoration is a *fixed-depth* `Runner → Runner` edge (a container around a host + runner), not aggregation — it composes exactly one runner, never a collection. +- `ExecutorPool` (was `Pool`): aggregates a collection of `Runner`s and adds bounded concurrency. + The restriction that matters is that **`ExecutorPool` is not itself a `Runner`** (the child + edge is typed `ExecutorPool → Runner`); since a pool is not a runner, it cannot be a child of + another pool. That is what makes pools-of-pools unrepresentable — without forbidding the + bounded, non-recursive `ContainerRunner`-over-`HostRunner` decoration. + +```mermaid +classDiagram + direction LR + class Runner { + <> + +run(payload) PayloadResult + } + class HostRunner + class ContainerRunner + class ExecutorPool { + -limit: int + } + Runner <|.. HostRunner + Runner <|.. ContainerRunner + ContainerRunner o-- Runner : wraps (fixed depth) + ExecutorPool o-- Runner : aggregates (pool is not a Runner) +``` + +This subsystem is tracked as separate work (likely against the Pilot repo); it is named here only +so the migration map is complete. One scope warning for that work, from the consumer analysis: +the JobAgent-side contract is richer than `run(payload)`. It also covers +*description-for-matching* (`getDescription()` — where Pool returns a **list** of CE dicts +implementing the MultiProcessor tag strategy), *filling-mode accounting* (`setCPUTimeLeft()`), +*asynchronous result harvesting* (the mutable `taskResults` dict plus the `AsyncSubmission` +flag), and Pool's per-job writes to `pilot.cfg`. The sibling ADR must scope all four, not just +execution. + +### 7. Extension surface — what is extendable, and what is not + +interCEde is a library implemented and mocked by others, so its public surface is defined +**explicitly** and governed by SemVer. Every public module declares `__all__`; anything not in a +public module's `__all__`, and any module or name prefixed with `_`, is internal and may change +without notice. Three tiers: + +**Tier A — Stable contracts (implement these).** The extension points third parties build +against. Breaking changes are SemVer-major. + +- The protocols: the essential `Submitter`, `Monitor` and their composition `JobBackend`; the + collaborators `Transport`, `Scheduler`; and the optional capability protocols + (`OutputRetriever`, `Cancellable`, `Purgeable`, `LoadReporter`, `SupportsLiveDiagnostics`, …). +- The data types and enums: `SubmissionSpec` (name provisional — see Open Issues), `Submission` + (bundles `.handles` and `.failures`), `JobHandle` (durable, serialisable, routing-carrying) and + its identity `JobID`, `JobStatus` (incl. `UNKNOWN`), `StatusMap`, `JobOutput`, `OpOutcome`, + `JobCounts`, `Diagnostics`, `Resources`, `FileRef`, `OutputSpec`. +- The exception hierarchy (a single rootable `InterCEdeError`). +- The registry API and entry-point group names. +- The lifecycle: backends and transports are async context managers (§2) — cheap, + side-effect-free construction; idempotent close; consumers that cache them own eviction-time + closing. + +**Tier B — Provided implementations (instantiate and register; do not subclass).** Public to +*use*, not a subclassing contract. Extend by **implementing a protocol or composing**, never by +subclassing these. + +- `BatchBackend`; the concrete transports (`SSHTransport`, `LocalTransport`, …); the concrete + schedulers (`Slurm`, `HTCondorScheduler`, `LSF`, `SGE`, `OAR`, `Torque`, `Direct`); the + monolithic CEs (`ARCBackend`, `HTCondorCEBackend`, `CloudBackend`). +- An **optional** convenience base, `BaseBackend(ABC)`, may carry *shared mechanics only* (config + normalisation, logging, retry/timeout helpers) and exposes a small, documented set of + overridable hooks. It is **not** the contract: a fully-conformant CE can be written without it, + and policy (slot accounting, credential renewal) is **not** placed in it. Because "no policy, + by convention" is exactly the DIRAC weakness this ADR indicts, two guardrails keep it from + re-accreting into a fat base: (1) the helpers it may contain are a **closed, enumerated set**, + and a unit test asserts it exposes *no* lifecycle method (`submit`/`status`/`fetch_output`/ + `kill`) and *no* policy method (anything like DIRAC's `available()`/`_monitorProxy`); (2) + interCEde's **own** monolithic CEs do **not** subclass it — so it can never become load-bearing + inside interCEde the way DIRAC's base became load-bearing in nine places. It exists for + third-party convenience and is tested against the same conformance suite as any other backend. + +**Tier C — Internal (do not import).** Command builders, output parsers, transport internals, +materialisation/retry machinery, anything under a `_`-prefixed module. No stability guarantee. + +The guiding rule, and the single most important lesson from DIRAC's fat base: **the preferred +extension mechanism is composition + structural typing, not inheritance.** You add a backend by +implementing a Tier-A protocol and registering it — not by subclassing a Tier-B class. + +### 8. DIRAC → interCEde migration map + +The map records *where each DIRAC piece lands*. Method-by-method porting mechanics (DIRAC's +job-ref formats, `S_OK` payload keys, CS parameter names) belong to the migration mapping +document owned by the sync-facade work (see Open Issues), not to this table. + +| DIRAC (`Resources/Computing`) | interCEde destination | Notes | +| --- | --- | --- | +| `AREXComputingElement` | `ARCBackend` (`JobBackend` + all four optional capabilities + `OutputRetriever`) | REST; reference implementation of the full sandbox model; `killJob` → `kill`, `cleanJob` → `purge`; the `diagnose` endpoint → `SupportsLiveDiagnostics` | +| `HTCondorCEComputingElement` | `HTCondorCEBackend` (monolithic; `JobBackend` + `OutputRetriever` + `Cancellable`) | native file transfer; destructive fetch (§2); no `LoadReporter` (reports no counts); shares the internal `_htcondor` core with `HTCondorScheduler` — composition candidate (Open Issues) | +| `SSHComputingElement` | `BatchBackend(SSHTransport, )` | composition; sandbox via `Transport.put`/`get` | +| `SSHBatchComputingElement` | `BatchBackend(SSHMultiHostTransport, Direct)` | host routing moves into `JobHandle`; host-spreading is placement *policy* (Open Issues) | +| `LocalComputingElement` | `BatchBackend(LocalTransport, )` | remote-family despite the name; its spoofed `ssh://` job-ID hack is dropped — routing lives in `JobHandle` (§1) | +| `CloudComputingElement` | `CloudBackend` (monolithic; essential protocols only) | no `OutputRetriever` (§2); `cleanupPilots` → `purge` | +| `CloudProviders/{OpenNebula,…}` | custom Apache Libcloud `NodeDriver`s — Tier-C internals of `CloudBackend` | provider selection stays delegated to Libcloud (§3, Rejected Ideas); VO-pluggability open | +| `BatchSystems/{SLURM,Condor,LSF,GE,OAR,Torque,Host}` | `Scheduler` implementations (`Slurm`, `HTCondorScheduler`, `LSF`, `SGE`, `OAR`, `Torque`, `Direct`) | promoted from ad-hoc string plugins to the typed protocol; ship-a-python-driver model dropped (§3) | +| `BatchSystems/TimeLeft/*` | **out of scope as an interface** → pilot/worker-node side | queried from *inside* an allocation — a different vantage than the submission-side `Scheduler` (Open Issues) | +| `ComputingElementFactory` (`ObjectLoader`) | typed registry + entry points (§5) | | +| `submitJob` (all remote CEs) | essential `Submitter.submit` | inputs consumed at submit — the `ExecutableToKeep` handshake disappears (§4) | +| `getJobStatus` (all remote CEs) | essential `Monitor.status` | | +| `getJobOutput` (all remote CEs) | **optional** `OutputRetriever.fetch_output` | optional because some backends cannot serve pull-retrieval (§2) | +| `killJob` / `cleanJob` (remote CEs) | optional `Cancellable.kill` / `Purgeable.purge` | not essential — some flows never use them | +| `ComputingElement` base — `getCEStatus()` counts | optional `LoadReporter` (counts only) | the *availability* computation (counts vs `Max*Jobs`) is **caller policy**, removed from interCEde | +| `ComputingElement` base — `shutdown()` (+ `QueueCECache` eviction) | async context-manager lifecycle (§2) | | +| `ComputingElement` base — `setProxy`/`setToken`/`_monitorProxy` | **split**: backend auth → [IC-ADR-003](IC-ADR-003_credentials.md); *payload* proxy renewal → pilot/runner side | | +| `ComputingElement` base — config hierarchy | a loader utility (Tier C), not a base-class responsibility | | +| `InProcessComputingElement` | **out of scope** → `HostRunner` (pilot/runner subsystem, §6) | severed | +| `SingularityComputingElement` | **out of scope** → `ContainerRunner` (decorator over a runner, §6) | severed | +| `PoolComputingElement` | **out of scope** → `ExecutorPool` (depth-1 aggregator, §6) | severed | + +### 9. Consumer interface — how DiracX uses interCEde + +The consumer surface is deliberately small: the Tier-A protocols plus the registry factory. +DiracX splits the old monolithic SiteDirector into **separate tasks**, and each one is typed to +the *narrowest* protocol it needs — never to `JobBackend`, never to a concrete class. The +registry resolves the same backend for all of them; each task sees only its slice. One sketch — +the submission task; the status and output tasks follow the same resolve-narrow-drive pattern: + +```python +from intercede import registry, Submitter, LoadReporter + +async def submission_task(resource, want, make_payload): + ce: Submitter = registry.backend(resource) # isinstance-checked at the boundary + if isinstance(ce, LoadReporter): # optional -> structural narrowing + want = min(want, policy.slots(await ce.counts())) # throttling policy is DiracX's + spec = make_payload() # -> one SubmissionSpec + sub = await ce.submit(spec, count=want) # same spec, `want` identical copies + await store.record(sub.handles) # DiracX owns this store + # sub.failures carries any copies the backend rejected, addressable per copy +``` + +- **The status task** depends on `Monitor` only: it polls and records (`store.update(await + ce.status(handles))`), with unknown ids coming back as `JobStatus.UNKNOWN` so nothing is + silently lost. It literally *cannot* call `fetch_output` — which both documents intent and + keeps output transfers out of the polling loop. +- **The output task** first confirms the backend has the *optional* `OutputRetriever` at all + (`isinstance` narrowing; Cloud lacks it, §2) and fetches into a durable `dest`. When the + backend declares `destructive` (§2), the remote copy is gone once the call returns, so `dest` + is the commit point — upload from there, and on a crash resume from `dest`, never re-fetch. + +What this fixes on the contract side: + +- **Restart re-drives handles; durability is DiracX's.** Tasks re-resolve the backend and drive + their slice on handles **reloaded from the DiracX store** — they never submitted anything in + this process, and interCEde never stored anything. This is why `JobHandle` must be serialisable + and self-contained (§1). +- **The submit→record window is the consumer's to close (interCEde is stateless).** A crash after + `submit` returns but before `store.record` commits leaves jobs on the backend that DiracX has + no handle for. This is not new — DIRAC's SiteDirector has the identical window today — and + interCEde cannot fix it, having no store; the candidate closing mechanism (tag-at-submit + + list-by-tag, using the reserved `tag` field from §4) is an Open Issue. +- **Pull vs push is invisible to interCEde.** A pull SiteDirector and a push PushJobAgent run the + *same* submission task; only *where the payload comes from and when* differs, and that lives in + the consumer. +- **Policy is the consumer's, mechanism is interCEde's.** Every `policy.*` call (throttling, + expiry, retries, "what is a pilot") is consumer-side; interCEde supplies only the verbs. + Optional capabilities are reached by structural narrowing, never assumed. + +## Rationale + +- **Protocols over a fat base class.** The thing callers depend on is a *contract*; a Protocol + expresses it without coupling implementations to interCEde's base, which is exactly what a + library implemented by third parties and mocked against containers needs. DIRAC's pain came + from the opposite: a fat base plus "abstract by convention" stubs with no enforcement and + drifting signatures. `@runtime_checkable` keeps an `isinstance` gate at the registry boundary; + a real `@abstractmethod` ABC is available where forcing an override is genuinely wanted. +- **Composition over inheritance for access × scheduler.** Inheritance forces N×M leaf classes or + fragile multiple inheritance; composition makes it N+M and is the literal expression of the + project's "backends combine" promise. DIRAC already proved the bone is sound (`SSH` + + `BatchSystems`); interCEde just applies it uniformly and gives the scheduler side a real typed + contract. This is well-trodden ground: HTCondor's **blahp** translates one protocol into + per-LRMS submit/status/cancel scripts, and ALICE's JAliEn has per-batch `BatchQueue` drivers — + the `Scheduler` protocol is the same idea, in typed Python. Their async + request-id/`RESULTS`-poll model is also why interCEde's contract is async, bulk, and poll-based + (§2) rather than blocking (see Rejected Ideas for why we reimplement rather than reuse them). +- **Capability segmentation, sized by real callers.** Interface Segregation, with the DiracX-task + split as the evidence: a status task wants `Monitor` and an output task wants `OutputRetriever`, + each *without* `submit` and without each other — so these are separate protocols, not one + "operator". `submit` and `status` are the two *essential* ones (every usable backend has them); + output-retrieval is a real task but optional, because a backend that cannot serve + pull-retrieval must not be forced to stub it — the same reasoning that makes `LoadReporter` and + independent diagnostics optional. Segment to match real callers, no further — and keep *policy* + (the counts → "is there room?" decision) in the caller, not the backend. (Counts-only was + checked against DIRAC: no remote-family caller consumes more than `{running, waiting}` — the + processors-cap math in the base `available()` is fed only by the severed inner CEs, so it moves + with the pilot-side subsystem.) +- **Severing the inner CEs.** Liskov is the test: no caller ever holds a `JobBackend` and uses it + without knowing whether it is remote or inner, because remote and inner serve *different + actors* with disjoint mechanics. The shared base bought only the factory and some config + boilerplate — both replaceable. Removing it deletes a false abstraction at near-zero cost. +- **Sandbox generalisation.** Payload-agnostic ⟹ sandbox-general. Returning a streamed manifest + of paths (not in-memory contents) is the only model that scales to real output, and it unifies + stdout/stderr/output-files/CE-log behind one retrieval. +- **Explicit extension surface.** Uncontrolled subclassing of concrete classes is how the DIRAC + fat base became load-bearing in nine places. A defined three-tier surface lets interCEde evolve + Tier-B/C freely while third parties depend only on Tier-A — which is what makes SemVer + meaningful for a plugin ecosystem. + +## Evolution & non-conforming backends + +No abstraction is eternal; the design's job is to make change cheap and localised, not to predict +the future. Three cases: + +1. **A new backend that fits the shape — trivial, by construction.** A scheduler/transport/CE + that is still "delegate-and-poll" (most new batch systems and CEs are) is added by + implementing the relevant protocol, registering an entry point, and shipping a container + conformance test. No core change — this is the designed-for case, and the reason discovery is + data-driven. + +2. **A backend that does not fit the shape — branch, don't contort.** A future resource may be + fundamentally different: serverless/FaaS with no persistent handle, a callback/push resource + that calls the consumer, a long-lived interactive session, or something with no output-sandbox + notion. The rule — the same one applied when this ADR severed the inner CEs — is **when the + semantics differ, add a sibling abstraction; never overload `JobBackend`.** Additive + differences become optional capability protocols; a genuinely different lifecycle gets its own + protocol and its own registry group. The monolithic-CE escape hatch already lets a + weird-but-pollable backend (e.g. Kubernetes: submit a Job object, poll, fetch logs, delete) + implement the contract directly with whatever internals it needs. + +3. **The contract itself stops making sense — manage it deliberately.** This is what ADRs, SemVer + and the narrow surface are for: + - Consumers depend only on small Tier-A protocols resolved by data, so implementations + (Tier B/C) can be rewritten freely — schedulers moving from CLI to REST changes only + `Transport`/`Scheduler` bodies, not the contract. + - A breaking change is a **new ADR superseding this one** plus a SemVer-major with a + deprecation window. + - Protocols are structural and versionable: introduce `JobBackendV2`, expose both through the + registry, bridge with adapters, and migrate backends one at a time — old and new run side by + side. + - **Prefer additive evolution** — a new optional capability protocol over a core change. The + core is kept deliberately small precisely to minimise what can break. + - The container conformance suite is the early-warning system: change an interface and the + per-backend tests show what breaks before any user does. + + The meta-principle: keep the core minimal, push variation into optional capabilities and + composition, and treat "this does not fit" as a signal to branch a new abstraction rather than + bend the old one. + +## Rejected Ideas + +- **Keep DIRAC's single fat `ComputingElement` base.** Conflates contract and policy, enforces + nothing, and is the documented source of the signature drift and the Remote/Inner/Pool + confusion. +- **One god-interface with every method mandatory.** Forces backends that lack a capability to + ship `S_ERROR`/`NotImplementedError` stubs — exactly what DIRAC's + `HTCondorCEComputingElement.getCEStatus()` already is. Additive optional protocols replace it — + even output-retrieval is optional (§2). +- **A separate `fetch_log` retrieval operation.** Breaks on destructive-fetch backends (§2), + where retrieval deletes the spool — the log must come out in the *same* operation as the + output. The log is therefore a manifest member of the single `fetch_output`; only ARC's + genuinely independent, non-destructive `diagnose` endpoint justifies a separate (optional) + diagnostics call. +- **Model access × scheduler with inheritance** (N×M classes, or a mixin lattice). Combinatorial + and fragile; composition is strictly better here. +- **Keep the inner CEs under `JobBackend`** (even as a sibling branch). They are never + substituted for remote CEs and share no mechanics; the common supertype is fictitious. +- **Use the full recursive Composite pattern for the pool.** Its defining feature — a composite + containing components, hence composites-of-composites — yields pools-of-pools, which we do not + want. A depth-1 aggregator with a leaf-typed child edge is the right restriction. +- **stdout/stderr-only output (DIRAC's default).** Incompatible with payload-agnosticism; ARC + already shows the general model is feasible, and HTCondor/SSH can do it natively or via the + transport. +- **Subclassing concrete CEs as the extension mechanism.** Re-creates the fat-base coupling; the + registry + protocol path keeps the provided implementations free to change. +- **`ObjectLoader` string-to-module discovery.** Untyped and undiscoverable; entry points give + typed, packageable, third-party-friendly registration. +- **Keep the CE layer inside DIRAC/diracx (no standalone library).** Rejected. A standalone + library must serve **both** DIRAC (sync, throughout the migration) and DiracX (async) from one + contract — an in-`diracx` subpackage cannot back the sync DIRAC side — and it lets + backend/CE-version support ship on its own cadence, without a DiracX release. It also isolates + the containerised test matrix from DiracX's CI, gives third-party VOs an entry-point extension + path that does not fork DiracX, and turns the Tier-A/B/C boundary into a packaging fact rather + than a convention. The cost — a second release train, SemVer ceremony, and diracx↔intercede + version skew — is accepted in exchange. (Were DiracX ever the sole consumer *and* the DIRAC + transition complete, folding it back in-tree would be worth revisiting.) +- **One repo for all DIRAC resources — or adding Storage/Catalog to interCEde.** Rejected: the + library boundary is a *coherent contract + testing domain*, so the repo count follows the + number of distinct contracts — contract-driven, not taxonomy-driven — and that count is small + and stable. + - *submit / monitor / retrieve* is one contract with one collaborator model + (`Transport` × `Scheduler`), one test rig (containerised schedulers), one consumer (the WMS). + Storage Elements (`put`/`get`/space, over XRootD/S3/gsiftp) and File Catalogs + (`register`/`query`/metadata, over DFC/Rucio) are *different contracts* with different + backends, test rigs and consumers (the Data Management System) — **sibling libraries**, not + tenants of interCEde. + - Merging them is the repo-level form of the fat-base false-unity this ADR rejects ("when + semantics differ, add a sibling abstraction, never overload the type" — §6, Evolution). DIRAC + itself never unified them: `Resources/{Computing,Storage,Catalog,…}` have distinct base + classes and factories, with no common `Resource` supertype. + - What siblings *may* share is **plumbing, never a contract** — the entry-point registry + mechanism, the typed-error base, and the async/sync-facade conventions could later factor + into a small `dirac-resources-core`. That factoring is a separate, deferrable decision, not a + reason to merge contracts now. +- **Reuse an existing job-submission abstraction instead of defining our own.** Surveyed and + rejected — nothing covers interCEde's target set (SSH+batch, ARC-CE REST, HTCondor-CE, Cloud) + as a modern async, typed Python library. HTCondor's blahp/GAHP is a process protocol, not a + library, and has no SSH story; PanDA/Harvester has the right plugin shape but is inseparable + from the PanDA server; JAliEn's batchqueue layer is Java, unreleased and licence-unclear; and + **none** of the generic scheduler libraries (RADICAL-SAGA, PSI/J, DRMAA, Parsl, AiiDA, + Dask-Jobqueue) speaks ARC at all, in any interface generation. The genuinely reusable layer is + the WLCG grid-CE ecosystem itself plus first-party per-backend clients (`pyarcrest`, the + `htcondor` bindings, Apache Libcloud), which interCEde composes behind a typed async contract. + The per-tool evidence is in the [reuse survey note](../notes/reuse-survey.md); blahp's per-LRMS + scripts and async poll model are worth learning from (see Rationale). + +## Open Issues + +Items tagged **(blocking)** need an answer before this ADR moves to Accepted; **(deferred)** +items are follow-up work that does not gate acceptance. + +- **Naming at the API boundary (blocking).** Keep `Job*` only for the *scheduler* handle + (`JobHandle`, `JobID`, `JobStatus` — accurate and namespaced). The submitted description is + provisionally `SubmissionSpec` — `JobSpec` clashes with DiracX Jobs, `Payload` with the + pilot-side payload. Confirm, or pick another neutral name. +- **Destructive-fetch confirmation (blocking).** Confirm the HTCondor ≥ 25.8 one-shot spool + behaviour (§2) against the release notes, whether setting `leave_in_queue` restores ARC-like + re-fetchability, and that every consumer reads `destructive` before assuming it can re-fetch. +- **HTCondorCE live pilot-log fetch (blocking).** DIRAC operators fetch running-pilot logs from + both AREX *and* HTCondorCE (`dirac-admin-get-pilot-logging-info`, via the CE's `getJobLog`). + Under this ADR the final log is a `fetch_output` manifest member and only ARC gets + `SupportsLiveDiagnostics` — which would regress the operator flow for running HTCondorCE + pilots, because HTCondorCE's log fetch rides the same spool-transfer machinery as output + retrieval. Decide: implement `SupportsLiveDiagnostics` on `HTCondorCEBackend` over that + machinery (documenting that a fetch against a *completed* spooled job shares the destructive + semantics, §2), or record the regression as accepted. +- **Submit idempotency / list-by-tag (deferred).** The submit→record crash window (§9) is the + consumer's to close; the realistic mechanism is **tag-at-submit + list-by-tag** — stamp a token + on the jobs at submit, list jobs carrying it after a crash, reconcile before re-submitting. The + `tag` field is already reserved on `SubmissionSpec` (§4, additive and non-breaking); the + deferrable part is the enumeration capability (a future optional protocol). `LoadReporter`'s + aggregate counts cannot substitute for it. +- **Cloud staging (deferred).** Whether `CloudBackend` later grows a limited `OutputRetriever` + (a transport *into* the VM), and whether "declared staging capability" deserves its own + optional protocol (§2 records why it has none today). +- **Cloud-provider drivers (deferred).** interCEde keeps *composing* Libcloud + (`get_driver`/`set_driver` stays the provider-selection mechanism) rather than defining its own + driver protocol (§3). Open: whether VO-pluggable clouds warrant a thin + `intercede.cloud_drivers` entry-point group over Libcloud's registry, and whether the + OpenNebula driver is maintained in-tree (Tier C) or upstreamed to Libcloud. +- **Home of the severed subsystem (deferred).** Where `Runner`/`ExecutorPool` live (Pilot repo vs + a new small library) — a separate ADR/tracking item (§6 lists the scope warnings for it). +- **`HTCondorCEBackend` as composition (deferred).** It shares the `_htcondor` core with + `HTCondorScheduler`; expressing it as `BatchBackend(HTCondorCETransport, HTCondorScheduler)` + would need a transport-level notion of a "destructive get", and the CE fronts a *different* + scheduler than a raw schedd (its own blahp hides the site's real batch system). Share the + internals now; revisit the shape later. +- **Multi-host SSH placement (deferred).** DIRAC's `SSHBatch` spreads a submission across hosts + by free slots and encodes the host in the job ref. The routing belongs in `JobHandle` (§8); the + spreading decision is placement **policy**. Decide when this backend is scheduled: caller + policy (consistent with the mechanism/policy split) vs a declared multi-host transport + capability. +- **`get_time_left()` placement (deferred — confirm).** It is queried from *inside* a running + allocation — a different vantage and consumer than the submission-side `Scheduler`. Kept + **off** the `Scheduler` protocol to preserve role cohesion; the batch-system parsing is shared + internal code, exposed via a separate pilot-facing capability if the pilot needs it. Confirm + this split (and payload-proxy renewal, likewise pilot-side). +- **DIRAC compatibility shim (deferred).** DIRAC is synchronous and `S_OK`/`S_ERROR`-based; the + resolution is a sync **facade** generated over the async core (run-to-completion + + `S_OK`/`S_ERROR` translation), usable only by sync callers outside an event loop — **not** a + hand-maintained parallel sync protocol, which would re-introduce the two-surfaces drift this + ADR fights. One design constraint is fixed now, because getting it wrong breaks cached + backends: the facade must own a **single long-lived background event loop** in a dedicated + thread and dispatch every call onto it (`run_coroutine_threadsafe`) — never per-call + `asyncio.run()`. Backend connection state (HTTP sessions, SSH connections) is bound to the + event loop it was created on, so a fresh loop per call breaks any backend reused across agent + cycles (the §2 lifecycle/caching model); the facade likewise exposes a sync `close()` over the + async close so cache eviction works. Sync callers inside DIRAC's Tornado-based services are + expected to be safe (handlers run in worker threads, with no running loop in the calling + thread) — verify once. Confirm the facade's lifetime (transition-only vs kept for sync + third-party tools). The facade work also owns the **migration mapping document** — the + translation table from DIRAC's load-bearing conventions (job-ref formats, `PilotStampDict`, + `S_OK` payload keys, CS parameter names, `Tag: Token[:vo]` vs IC-ADR-003 requirements) to + interCEde's. The mapping document lives in interCEde docs; the adapter code lives DIRAC-side. +- **Heterogeneous submit batches (deferred).** `submit(spec, count)` submits `count` *identical* + copies, which map to native array submission (HTCondor `queue N`, Slurm `--array`). A + payload-agnostic push consumer (PushJobAgent) may want a *heterogeneous* batch of distinct + specs. Decide whether to add a `submit(specs: Sequence[SubmissionSpec])` overload (which loses + native-array efficiency on most backends) or leave heterogeneous batches as N separate calls. +- **Staging as a third collaborator (deferred).** Staging currently rides on `Scheduler` + (`stages_own_files` + hooks, §3), and the HTCondorCE-as-composition idea would need a + "destructive get" on the *transport* — two cross-axis concerns before any code exists. + Evaluate promoting staging to its own collaborator protocol (a `Stager` strategy composed + alongside `Transport`/`Scheduler`). +- **Capability declaration vs structural sniffing (deferred).** Dispatch narrows by `isinstance` + against `@runtime_checkable` protocols, which check method *presence only* (§2). Evaluate an + explicit capability declaration (registry metadata, or a `capabilities()` set) as a sturdier + gate, keeping `isinstance` as the smoke-test. +- **Loud partial-failure results (deferred).** Bulk verbs return per-`JobID` outcome maps, which + a caller can silently ignore (unlike a raised exception). Consider a result type that makes + undrained failures loud (a `.raise_for_failures()` helper, or logging on drop) plus single-job + convenience wrappers over the bulk verbs. + +Note: the backend credential/auth model was an open issue here and is now its own ADR — +[IC-ADR-003](IC-ADR-003_credentials.md) (typed credentials, backend-declared requirements, +provider-based supply). Only *backend* auth is interCEde's; payload credential renewal stays +pilot-side. diff --git a/docs/adr/IC-ADR-002_integration_tests.md b/docs/adr/IC-ADR-002_integration_tests.md new file mode 100644 index 0000000..1599795 --- /dev/null +++ b/docs/adr/IC-ADR-002_integration_tests.md @@ -0,0 +1,398 @@ +# IC-ADR-002: Integration testing against containerized backends + +## Metadata + +- **Created By:** Alexandre Boyer +- **Date:** 2026-07-02 +- **Status:** Draft +- **Decision Maker(s):** Federico Stagni, Christophe Haen, Chris Burr +- **Stakeholders:** interCEde contributors and maintainers; DIRACGrid CI maintainers +- **Depends on:** IC-ADR-001 (core architecture: Protocols, Transport × Scheduler composition, capability segmentation) + +> **Scope and altitude.** Direction-setting for the *test architecture*: what runs (real backend +> daemons in per-stack compose environments), where (GitHub-hosted CI, on every PR), and how +> versions are managed (pins + Renovate + support windows). The contents of individual tests, the +> exact Dockerfiles, and the CI YAML are implementation, governed by the architecture decided +> here. + +## Abstract + +interCEde's contract is expressed as structural `Protocol`s exercised against fakes, which structurally cannot catch the bugs interCEde exists to absorb: version-specific daemon behaviour, destructive output retrieval, and status-mapping quirks. This ADR decides to add integration tests that run **real backend daemons** — in the combinations interCEde supports — in GitHub-hosted CI on every pull request, within a standard runner. The unit of everything is a **stack**: a named, self-contained backend environment defined by one `docker compose` file plus versioned configuration, declared in a single `stacks.yml` manifest from which the CI matrix is generated. A single, backend-agnostic contract suite is dispatched across stacks by capability (protocol narrowing), mirroring how the library itself handles capability variance. Backend images build from pinned RPMs, are prebuilt and pushed to GHCR, and every version — image tags, Dockerfile `ARG`s, Python dependencies — is pinned and tracked by Renovate, so an upstream release that breaks a contract surfaces as a red, bisectable Renovate PR rather than tribal knowledge. Credentials are generated ephemerally per run and never committed. A non-blocking weekly canary runs the same matrix against unpinned tags for early warning. Adding a backend or configuration is a manifest entry plus files, with no workflow edits. + +## Motivation + +interCEde's core architecture is deliberately mock-friendly: everything is a `Protocol`, backends +are composed, and the unit test suite exercises contracts against fakes. That is also its greatest +testing risk. A structural protocol plus a fake will happily agree with each other forever while +the real ARC REST endpoint, the real `condor_submit -spool` handshake, or the real +`sbatch`-over-SSH quoting rules drift away underneath. The bugs interCEde exists to absorb — +version-specific daemon behaviour, destructive output retrieval, status-mapping quirks — are +precisely the bugs that unit tests structurally cannot see. + +DIRAC's history makes the point concretely: HTCondor ≥ 25.8 silently changed output retrieval for +spooled jobs into a one-shot operation. No unit test anywhere would have caught that; only a test +that submits a real job to a real schedd and fetches its output twice would go red. + +We therefore need integration tests that: + +1. run real backend daemons, in the combinations interCEde actually supports; +2. run in GitHub-hosted CI, on every pull request, within the resources of a standard runner + (4 vCPU / 16 GB on public repositories); +3. surface **new upstream versions** (ARC, HTCondor, Slurm, OpenSSH, and our own Python + dependencies) as reviewable Renovate pull requests whose CI result *is* the compatibility + verdict; +4. support **multiple configurations per backend combination** eventually (token vs. proxy auth, + shared vs. non-shared filesystem, alternative queue setups) while starting with exactly one + basic configuration each. + +### Initial target combinations + +| Stack ID | Backend under test | interCEde components exercised | +|---|---|---| +| `arc-slurm` | ARC 7 CE (REST) → Slurm LRMS | `ARCBackend` (AREX REST), end-to-end through a real LRMS | +| `arc-condor` | ARC 7 CE (REST) → HTCondor LRMS | `ARCBackend` against the other major LRMS | +| `htcondor` | HTCondor-CE → HTCondor pool | `HTCondorCEBackend`; plain-schedd Condor scheduler tests reuse the same pool | +| `ssh-slurm` | sshd → Slurm | SSH transport × Slurm scheduler | +| `ssh-condor` | sshd → HTCondor (schedd) | SSH transport × Condor scheduler | +| `local-slurm` | Slurm, tests run *inside* the container | Local transport × Slurm scheduler | + +`ssh-*` and `local-*` share the same images — the local variant simply executes pytest inside the +batch container instead of connecting over port 22. Which of `ssh-slurm` / `local-slurm` lands +first is an implementation detail; the architecture treats them identically. Local × HTCondor +(first-class in IC-ADR-001's driver list) is deliberately not an initial stack: the Local +transport is exercised by `local-slurm` and the Condor scheduler by `ssh-condor`, so the +combination adds no new axis; it can be added later as a manifest entry reusing the `ssh-condor` +stack (`exec_in_container: true`). + +### Facts constraining the design + +- **HTCondor** publishes official role images on Docker Hub (`htcondor/mini`, `htcondor/cm`, + `htcondor/execute`, `htcondor/submit`), version-tagged (`-el9`, `lts-el9`). These are + Renovate-trackable out of the box. +- **HTCondor-CE** has no first-party image with clean semantic tags (the OSG images use date tags + and OSG-series semantics); an in-repo Dockerfile installing a pinned `htcondor-ce` RPM is more + Renovate-friendly than chasing OSG tags. +- **ARC** has no maintained official Docker image. The supported path is installing pinned + `nordugrid-arc-*` RPMs on EL9 in our own Dockerfile. interCEde targets **ARC 7 and later + only** — ARC 6 is out of scope (see §8). Crucially, ARC ships *zero configuration*: + a minimal working CE with a **Test-CA and host certificate generated at install time** + (`arcctl test-ca`), which removes the entire grid-PKI problem from CI. +- **Slurm** has no official image; community images are unmaintained to varying degrees. An + in-repo Dockerfile with a pinned Slurm package version (single node running `slurmctld` + + `slurmd` + `munged`) is small and fully under our control. +- **Renovate** is already configured in this repository (pip/pep621/github-actions managers). It + additionally supports `dockerfile` and `docker-compose` managers (image tags), and + `customManagers` (regex) for versions embedded in Dockerfiles as `ARG`s — the standard + `# renovate: datasource=... depName=...` comment convention. +- GitHub Actions **service containers** cannot express what we need (no compose, no build step, + no ordered startup, poor log access). Docker and `docker compose` are preinstalled on + `ubuntu-latest`, and compose v2 supports `up --wait` gating on healthchecks. + +## Specification + +### 1. A *stack* is the unit of everything + +A **stack** is a named, self-contained backend environment: + +``` +tests/integration/ +├── stacks.yml # the manifest: single source of truth for the CI matrix +├── conftest.py # stack-aware fixtures, capability-based skipping +├── test_submit.py # ONE backend-agnostic contract suite … +├── test_status.py +├── test_output.py # … including the fetch-twice destructiveness test +├── test_kill.py +└── stacks/ + ├── _images/ # Dockerfiles shared across stacks + │ ├── arc/Dockerfile # EL9 + pinned nordugrid-arc RPMs (ARG ARC_VERSION) + │ ├── slurm/Dockerfile # EL9 + pinned slurm packages (ARG SLURM_VERSION) + │ └── htcondor-ce/Dockerfile # EL9 + pinned htcondor-ce RPM (ARG HTCONDOR_CE_VERSION) + ├── arc-slurm/ + │ ├── compose.yml + │ └── config/ + │ └── basic/ # arc.conf, slurm.conf, intercede-client.toml + ├── arc-condor/… + ├── htcondor/… + └── ssh-slurm/… # sshd enabled; local-slurm reuses this stack +``` + +Rules: + +- One `compose.yml` per stack. Shared plumbing (network, credential volume, healthcheck blocks) + is factored with YAML anchors or compose `include:`, **not** with a generated mega-file. + A stack must be runnable locally with exactly + `docker compose -f tests/integration/stacks/arc-slurm/compose.yml up --wait` — CI does nothing + a developer cannot do on a laptop. +- Every configurable surface lives under `config//`. Today each stack has only + `config/basic/`. A *configuration* is mounted into the containers (arc.conf, condor config, + slurm.conf) **and** consumed by the test client (`intercede-client.toml` describing endpoint, + credential type, queue names). Backend config and client config change together, atomically, + under one name — this is what makes future configuration-matrix growth mechanical rather than + a refactor. +- `stacks.yml` is the manifest the CI matrix is generated from: + + ```yaml + stacks: + - id: arc-slurm + configs: [basic] + versions: [latest] # ARC 7.x leading edge (support window in §8) + markers: "remote and arc" + - id: htcondor + configs: [basic] + versions: [lts, latest] # HTCondor 24.0 LTS anchor + leading edge + markers: "remote and htcondor" + - id: ssh-slurm + configs: [basic] + markers: "scheduler and slurm" + exec_in_container: false + - id: local-slurm + stack: ssh-slurm # reuses the ssh-slurm compose stack + configs: [basic] + markers: "scheduler and slurm" + exec_in_container: true + ``` + + `versions` names the entries of the backend's support window to run (§8); a bare or absent + `versions` means the single leading-edge pin. Adding a stack, a configuration, or a supported + version is one manifest entry plus files — no workflow edits. + +### 2. One contract test suite, capability-dispatched + +There is a **single** integration test suite, not one per backend. Tests are written against the +interCEde protocols and parameterized by the stack's client configuration. Capability variance is +handled exactly the way the library itself handles it — protocol narrowing: + +```python +async def test_output_refetch(backend, submitted_job, tmp_path): + if not isinstance(backend, OutputRetriever): + pytest.skip("backend has no output retrieval") + first = await backend.fetch_output([submitted_job], dest=tmp_path) + ... +``` + +This symmetry is deliberate: the integration suite is the executable form of the contract. If a +capability check works in the test harness, it works for DiracX; if a backend's structural typing +lies (the `runtime_checkable` presence-only problem from IC-ADR-001), the integration suite is +where the lie is caught against a real daemon. Backend-specific quirks (e.g. the HTCondor +one-shot retrieval semantics) get dedicated marker-selected tests rather than conditionals inside +generic ones. + +Markers (`remote`, `scheduler`, `arc`, `htcondor`, `slurm`, `ssh`, `destructive_fetch`) select +the applicable subset per stack via the manifest's `markers` expression. + +### 3. Prebuilt stack images on GHCR + +ARC and Slurm images build from RPMs; rebuilding them on every PR wastes 5–10 minutes per job. A +separate `build-images` workflow builds and pushes +`ghcr.io/diracgrid/intercede-testenv/{arc,slurm,htcondor-ce}:` whenever the Dockerfiles +change (including when Renovate bumps a pinned `ARG`), plus a weekly rebuild for base-image +security updates. PR CI **pulls** by digest. + +Daemons run in the foreground under a minimal supervisor (or a plain entrypoint script), **never +systemd** — privileged systemd containers are fragile on GitHub runners and hide daemon logs from +`docker logs`. + +### 4. Ephemeral credentials, generated per run + +No credential is ever committed: + +- **ARC stacks**: the container entrypoint runs `arcctl test-ca init` etc., generating the CA, a host + certificate, and a client credential; client credentials are written to a shared + `credentials` volume that the test harness reads. Token auth is added later as a second + configuration (`config/token/`), not baked into `basic`. +- **HTCondor / HTCondor-CE**: `IDTOKENS` auth; the CE entrypoint mints a token into the shared + volume. No GSI, no VOMS in `basic`. +- **SSH stacks**: `ssh-keygen` in a job step, public key mounted into the container's + `authorized_keys`, known-hosts pinned from `ssh-keyscan`. + +The shared credentials volume *is* the interface between stack and harness; its layout +(`credentials//…`) is part of the stack contract. + +### 5. Manifest-driven GitHub Actions matrix + +``` +.github/workflows/integration.yml +├── job: matrix — reads stacks.yml, emits JSON via fromJSON() +└── job: integration — matrix: {stack × config × version}, fail-fast: false + ├── docker compose up --wait (healthchecks gate readiness; version → image tag) + ├── pixi run pytest tests/integration -m "" --stack= --config= + │ (or `docker compose exec` for exec_in_container stacks) + └── on failure: `docker compose logs` + backend log dirs uploaded as artifacts +``` + +- Each stack runs in its **own job**: parallel wall-clock, isolated failure blast radius, and each + stays comfortably inside a 4 vCPU runner. There is no combined all-backends job. +- Healthchecks are mandatory in every compose service: ARC (`curl -k` the REST endpoint's info + URL), HTCondor (`condor_status -limit 1` / `condor_ce_status`), Slurm (`sinfo` reporting the + node up), sshd (TCP probe). `up --wait` then replaces every hand-rolled sleep-and-retry loop. +- The integration workflow is `workflow_call`-reusable, invoked from `ci.yml` after unit tests, + and required for merge. Log artifacts on failure are non-negotiable: a red integration job + without daemon logs is a re-run generator, not a signal. + +### 6. Renovate as the version radar — two lanes + +**Lane 1 — pinned, blocking (PRs).** Everything CI runs against is pinned: + +- image tags in `compose.yml` files → `docker-compose` manager (enable in `renovate.json`); +- base images and pinned package versions in Dockerfiles → `dockerfile` manager plus + `customManagers` regex on annotated `ARG`s: + + ```dockerfile + # renovate: datasource=repology depName=fedora_epel_9/nordugrid-arc + ARG ARC_VERSION=7.1.1 + ``` + +- Python dependencies → existing pep621 manager, unchanged. + +For a backend with a support window (§8), Renovate tracks **only the leading-edge ARG**; the older +anchor ARGs are pinned and constrained (`packageRule` `allowedVersions`/`matchCurrentVersion`, or +`enabled: false`) so they accept only patch bumps within their major and never jump it. + +A Renovate bump PR (e.g. HTCondor 25.7 → 25.8) triggers image rebuild + full integration matrix. +**A red Renovate PR is the feature**: it is the earliest, cheapest, fully-reproducible signal +that an upstream release broke an interCEde contract — the 25.8 destructive-fetch change would +have surfaced as exactly this. Backend bumps are grouped per backend (`arc`, `htcondor`, +`slurm`), never mixed, so a red PR names its culprit. + +**Lane 2 — unpinned canary (scheduled, non-blocking).** A weekly scheduled workflow runs the same +matrix against `latest`/nightly backend tags. It cannot block merges; it exists to catch +upstream changes *before* they reach the stable tags Renovate tracks. Failures open a +deduplicated issue via workflow automation. + +### 7. Configuration growth path + +The full matrix is `(stack × config × version)`, starting as `(6 × basic × leading-edge)`. Planned +configuration axes, added as `config//` directories and manifest entries — each an additive +change: + +- auth variants (token vs. proxy for ARC; token lifetimes for HTCondor-CE); +- filesystem variants (shared vs. non-shared session directories — the `stages_own_files` axis); +- queue topology (multiple queues, per-queue limits); +- resource constraints (memory/CPU limits surfacing scheduler translation bugs); +- a batch-host variant with **no usable Python interpreter**, asserting IC-ADR-001 §3's + no-interpreter-on-the-remote-host property (DIRAC's SSH CE shipped a Python driver to the host; + interCEde must never need one). + +PR CI runs `basic` only; the scheduled workflow runs the full configuration set. This keeps PR +latency flat as the configuration matrix grows. + +### 8. Backend version support window + +interCEde is a *client* that must interoperate with backend daemons whose version it does not +control. Two kinds of "version" therefore live in this repo and must not be conflated: + +- **Versions interCEde ships** — Python dependencies, base images. We own these; "bump to latest" + is correct and Renovate does exactly that (§6). +- **Backend server versions interCEde interoperates with** — ARC, HTCondor(-CE), Slurm daemons. + Here a pinned version is a *test fixture standing in for a version some site runs*, not a + dependency we upgrade. Real sites run a spread — HTCondor LTS years after release, older ARC 7 + where the latest hasn't rolled out — so testing only the newest daemon answers the wrong + question. + +Each backend therefore declares a **support window**: the set of upstream versions interCEde +claims to work against, expressed as an *oldest/LTS anchor* plus the *leading edge*. Version is a +matrix axis alongside `config` (the `versions:` key in `stacks.yml`, §1). + +- **Renovate tracks only the leading edge.** The leading-edge `_VERSION` ARG bumps + normally — this is Lane 1's radar unchanged: a new upstream release arrives as a red-or-green PR + that names its culprit. +- **Older anchors are human-managed.** Each anchor ARG is pinned and constrained (§6) so Renovate + offers only patch bumps within the anchor's major. **Moving the window — dropping an old major, + adopting a new one — is a deliberate human PR**, because dropping support for a version sites + still run is a decision, not an auto-merge. +- **PR latency stays flat.** As with `config`, PR CI runs a representative subset (leading edge, + plus one anchor for the backend most exposed to version skew); the scheduled workflow runs the + full version cross-product. + +Multi-version testing is also what *justifies* version-conditional client code: the HTCondor +≥ 25.8 one-shot-retrieval shim is exactly such a case, and the window is where it earns its +regression coverage. + +**ARC is supported from version 7 only.** ARC 6 (pre-REST era: GridFTP/EMI-ES data staging, LDAP +infosystem) and ARC 7 (REST-first, `arcctl`) are close to different backends, and interCEde's AREX +path is REST-native. ARC 6 is explicitly out of scope, so the ARC window's floor is 7.x, with no +v6 anchor. + +## Rationale + +The design follows directly from the drivers above: real daemons over fakes (to catch what +structural typing cannot), per-stack isolation over a shared environment (attributable failures +inside a small runner), pinned-plus-Renovate over unpinned tracking (every version change attached +to a reviewable PR), and manifest-plus-files growth over workflow surgery (so the configuration +matrix scales without CI edits). The resulting benefits and the trade-offs deliberately accepted +in exchange: + +**Benefits** + +- Contract violations against real daemons are caught at PR time, per backend, with logs. +- Upstream version compatibility becomes a reviewed, bisectable git history of Renovate PRs + instead of tribal knowledge. +- Stacks double as reproducible local development environments — `compose up`, point your client + config at localhost, develop against a real ARC. +- Adding backends (future: Cloud via a mock EC2/OpenStack endpoint) or configurations is + manifest-plus-files, no CI surgery. +- The destructive-fetch semantics, the sorest point of IC-ADR-001, get a permanent regression test + (`fetch twice, assert contract`) on every backend. + +**Trade-offs and accepted costs** + +- We own three Dockerfiles for daemons whose packaging we don't control; EL9 repo layout changes + will occasionally break image builds (contained to the `build-images` workflow). +- Integration jobs add ~5–8 minutes wall-clock per PR (parallel across stacks). Accepted; unit + tests remain the fast inner loop. +- Single-node Slurm/HTCondor pools do not exercise multi-node scheduling behaviour. Out of scope: + interCEde's contract ends at the CE/scheduler interface; scheduling fidelity beyond it is the + backend's problem. +- The version support window (§8) multiplies images and CI jobs per backend. Contained by running + only the leading edge (plus one anchor) on PRs and the full cross-product on the schedule, and by + keeping the window to a small anchor set rather than every historical release. +- The canary lane will produce noise when upstream nightlies are broken through no fault of ours. + Mitigated by non-blocking status and issue deduplication. +- GitHub-hosted runners only (linux/amd64). No arm64, no macOS backend testing. Revisit only if a + consumer materializes there. + +## Rejected Ideas + +**Kubernetes (kind/k3s) as the orchestration layer.** Buys multi-node realism and Helm-chart +reuse (diracx-charts) at the price of cluster boot time, YAML volume, and debuggability inside a +CI job. Nothing in the CE contract requires more than "daemon reachable on a port"; compose +delivers that in one file per stack. Revisit if interCEde ever tests in-cluster deployment +concerns, which today it explicitly doesn't have. + +**testcontainers-python / per-test containers.** Excellent for a Postgres; wrong for daemons with +30–60 s startup (ARC, slurmctld+munge). Session-scoped stacks amortize startup once per job. +Per-test isolation is recovered logically (unique job tags/working dirs per test), not by +container churn. + +**One mega-compose with every backend.** Single job, shared fate: one flaky daemon reds the +world, resource ceilings are shared, and log spelunking spans six daemons. Parallel per-stack +jobs are faster and attribute failures for free. + +**GitHub Actions `services:`.** No compose semantics, no build, no startup ordering beyond +health of individual containers, awkward log retrieval. Fine for a Redis sidecar; not for a CE +and its LRMS. + +**Installing backends directly on the runner (apt/yum in the job).** Fast to prototype, +impossible to pin properly (Ubuntu's Slurm is whatever the distro ships), nothing is reusable +locally, and ARC on Ubuntu is not a supported combination. Containers or nothing. + +**Reusing DIRAC's certification/integration environment.** It exists, but it drags in the full +DIRAC server stack and its configuration system — the exact coupling interCEde was extracted to +escape. interCEde must be testable by someone who has never installed DIRAC. + +**Tracking upstream only via the canary (no pinning).** Green-today-red-tomorrow CI with no +diff to point at. Pinning plus Renovate keeps every version change attached to a reviewable PR; +the canary is an early-warning supplement, not the mechanism of record. + +**Nightly-only integration tests (nothing on PR).** Decouples breakage from the change that +caused it and lets contract violations merge. The whole value proposition is the red X *on the +PR that introduces the problem* — including Renovate's version-bump PRs. + +## Open Issues + +- Exact Renovate datasource for EL9 RPM pins (`repology` vs. a custom endpoint for the NorduGrid + repo) — to be settled when the ARC Dockerfile lands. +- Whether `htcondor` stack `basic` uses `htcondor/mini` behind the CE or a cm/execute/submit + trio; `mini` is preferred until a test needs role separation. +- Whether Cloud backend testing (mock OpenStack/EC2) becomes a seventh stack or stays unit-level; + deferred until the Cloud backend is scheduled. diff --git a/docs/adr/IC-ADR-003_credentials.md b/docs/adr/IC-ADR-003_credentials.md new file mode 100644 index 0000000..624d377 --- /dev/null +++ b/docs/adr/IC-ADR-003_credentials.md @@ -0,0 +1,301 @@ +# IC-ADR-003: Backend credentials — typed, declared, provider-supplied + +## Metadata + +- **Created By:** Alexandre Boyer +- **Date:** 2026-07-03 +- **Status:** Draft +- **Decision Maker(s):** Federico Stagni, Christophe Haen, Chris Burr +- **Stakeholders:** interCEde backend implementers; DIRAC/DiracX SiteDirector and PushJobAgent maintainers; DiracX token-issuance maintainers; site/VO operators +- **Depends on:** IC-ADR-001 (contract shape, lifecycle, Tier A/B/C surface) + +> **Scope and altitude.** Direction-setting, like IC-ADR-001. This ADR decides how *backend* +> authentication credentials — bearer tokens and X.509 proxies — are **typed, declared, supplied, +> refreshed, scoped and materialised**. It does not decide token *issuance* (DiracX's), *payload* +> credential renewal (pilot-side, DIRAC's `_monitorProxy` lineage), or the exact dataclass fields +> (implementation). Long-lived transport secrets (SSH keys) are configuration, not credentials, and +> are only delimited here. + +## Abstract + +Backend auth is on the critical path for the ARC and HTCondor-CE backends, and DIRAC's incumbent +model does not survive the move to a stateless, cached, async library: credentials are **mutated +onto** long-lived CE objects (`setProxy`/`setToken`), freshness policy is **duplicated across +callers**, token opt-in is a **stringly-typed CS tag**, and audience is a bare attribute. This ADR +replaces that with three pieces: **typed, immutable credential values** (`BearerToken`, +`X509Proxy`, grouped in a `CredentialSet`); **backend-declared `CredentialRequirements`** (which +kinds it accepts, for which audience/scopes — data computed from the backend's resolved config, +replacing `Tag: Token[:vo]` and `audienceName`); and **provider-based supply** — backends pull a +fresh `CredentialSet` from a caller-supplied `CredentialProvider` when theirs is missing or near +expiry, so freshness *mechanics* live in one place while issuance/renewal *policy* stays entirely +in the consumer (DiracX token machinery, DIRAC's proxy manager, or a static file). Materialisation +(token/proxy files, env injection for CLI backends) is internal (Tier C). ARC's proxy delegation +remains backend-internal mechanics consuming a provider-supplied proxy. + +## Motivation + +What the DIRAC code actually does today (all verified against the current codebase): + +1. **Mutating setters on cached objects.** `ComputingElement.setProxy`/`setToken` mutate CE + instances that `QueueCECache` caches and reuses across agent cycles. Credential state and + connection state are entangled on the same long-lived object — exactly the hazard for a library + whose backends are cached and context-managed (IC-ADR-001 §2, lifecycle). +2. **Caller-side renewal policy, duplicated.** `SiteDirector._setCredentials` inspects + `ce.proxy.getRemainingSecs()` and re-supplies; `WMSUtilities.setPilotCredentials` reimplements + the same gate for the pilot-kill/log service path. Two copies of the same freshness loop is the + documented drift pattern IC-ADR-001 exists to end. +3. **Stringly-typed opt-in.** A CE accepts tokens iff `"Token"` or `f"Token:{vo}"` appears in its + CS `Tag` list. A *capability declaration* is encoded in a free-form tag shared with scheduling + metadata. +4. **Audience as a bare attribute.** Callers mint tokens with `audience=ce.audienceName` — an + untyped per-CE string (AREX: `https://:`; HTCondorCE: `:9619`) with no + accompanying scopes/kind information. +5. **Per-backend materialisation, ad hoc.** HTCondorCE writes the token to a temp file and injects + `_CONDOR_*` env vars around each CLI call; the base writes proxies to files and exports + `X509_USER_PROXY`; Cloud reads a `cloud.auth` ini with a magic `PROXY` secret that dumps the + pilot proxy. +6. **ARC needs more than a header.** AREX supports bearer tokens *and* X.509 proxy delegations + (create a delegation via CSR, sign with the proxy chain, upload, renew) — and can need **both at + once** (`AlwaysIncludeProxy`). +7. **HTCondorCE needs both at once — today, in production.** Even when SCITOKENS authenticates the + channel, DIRAC's submit description *unconditionally* contains `use_x509userproxy = true`, and + the code says why: *"For now, we still need to include a proxy in the submit file — HTCondor + extracts VOMS attribute from it for the sites"* — the proxy rides along for **site-side + consumption** (per-VO attribution in site accounting, i.e. the APEL pipeline), not for channel + auth. `_executeCondorCommand` even refuses to run with neither token nor proxy, and token mode + sets `_CONDOR_DELEGATE_JOB_GSI_CREDENTIALS=false` to work around a condor 24.4 delegation bug + (HTCONDOR-2904). Together with ARC's pairing, any model that assumes "one credential per + backend" is wrong on day one — twice. + +### Drivers + +- **Stateless and cache-safe.** Credentials must not be hidden mutable state on cached backends. +- **Policy with the consumer.** Where credentials *come from* (DiracX token service, proxy + download, a file) and *when they are renewed* is consumer policy; interCEde owns only the + mechanics of asking at the right time and using them correctly. +- **Typed declaration.** A consumer must be able to ask a backend "what do you need?" and get data + — not parse tags. +- **One supply path for one and many credentials.** Token-only, proxy-only, and token+proxy + backends use the same machinery. +- **Testable in containers.** The model must work with IC-ADR-002's ephemeral per-run credentials + (arcctl test-CA, IDTOKENS, ssh-keygen). + +## Specification + +### 1. Credential values (Tier A) + +Immutable, typed values with an expiry the machinery can read: + +```python +@dataclass(frozen=True) +class BearerToken: + value: str # opaque to interCEde; never logged + expires_at: datetime | None + +@dataclass(frozen=True) +class X509Proxy: + pem: bytes # full chain, PEM; opaque to interCEde beyond expiry + expires_at: datetime | None + +Credential = BearerToken | X509Proxy + +@dataclass(frozen=True) +class CredentialSet: # what a provider returns; may satisfy >1 kind + def get(self, kind: type[Credential]) -> Credential | None: ... +``` + +`CredentialSet` exists because of ARC's token+proxy case: requirements may name several kinds, and +one provider call returns everything needed, atomically. interCEde never inspects credential +*contents* (no VOMS parsing, no JWT decoding) beyond expiry bookkeeping. + +### 2. Requirements — the backend declares, as data (Tier A) + +```python +@dataclass(frozen=True) +class CredentialRequirements: + kinds: frozenset[type[Credential]] # the kinds needed TOGETHER (all-of, not a menu) + audience: str | None = None # token audience, if kinds include BearerToken + scopes: frozenset[str] = frozenset() # e.g. compute.create/compute.read (WLCG profile) +``` + +A backend computes its `CredentialRequirements` from its resolved configuration (it knows its +endpoint, hence its audience) and exposes them as a read-only property. This single piece of data +replaces both the `Tag: Token[:vo]` opt-in *and* `audienceName`: the consumer reads the +requirements and mints/fetches accordingly. Configuration may still *narrow* a backend that +accepts several kinds ("this site: proxy only"); it can never widen beyond what the backend +declares. + +Two semantic rules, both forced by the grid CEs: + +- **`kinds` means "all of these, together".** Requirements are a concrete ask for the backend's + *active configuration*, never a menu of alternatives — both first-party grid CEs need a *set* + (ARC `AlwaysIncludeProxy`; HTCondorCE token + proxy, Motivation 7). A backend that supports + alternative modes (token-only *or* proxy-only) exposes the mode as configuration, and its + requirements reflect the configured mode. +- **A required kind need not authenticate the channel.** HTCondorCE's proxy is materialised into + the submit description (`use_x509userproxy`) for *site-side* consumption — VOMS attributes + feeding the site's accounting (APEL) — while the token authenticates the channel. The provider + sees no difference (it supplies the set); what each member is *for* is backend mechanics + (Tier C). + +### 3. Supply — a provider the backend pulls from (Tier A) + +```python +@runtime_checkable +class CredentialProvider(Protocol): + async def get(self, requirements: CredentialRequirements) -> CredentialSet: ... +``` + +- The provider is part of the backend's construction config (registry request). A trivial + `StaticCredentials(CredentialSet)` provider covers files/fixed tokens — and the integration + stacks. +- **The backend pulls; the consumer implements.** Before an operation, if the backend's current + set is missing or within a freshness margin of expiry, it awaits `provider.get(...)` — once, in + one place, inside the library. What the provider *does* (call DiracX's token issuer, download a + proxy, read a refreshed file) is consumer code. This keeps the mechanism/policy split of + IC-ADR-001 §9 and deletes the SiteDirector/WMSUtilities duplication by construction. +- Provider calls are per-backend, not per-job: one credential authenticates the *channel/CE*, not + each submission. +- Failures raise the typed auth branch of `InterCEdeError` as whole-operation failures (IC-ADR-001 + §2 partial-failure rule: auth failure is never a per-id outcome). + +### 4. Materialisation (Tier C) + +Backends that drive CLIs or need files get internal helpers, never a public contract: secure temp +files (0600, private dir), env injection (`X509_USER_PROXY`, HTCondor `_CONDOR_*`/SciTokens env), +scoped to the operation and cleaned up deterministically (the backend's context-manager lifecycle +from IC-ADR-001 §2 is the natural cleanup boundary). Credential values never appear in logs or +exception messages. + +### 5. Delegation is backend mechanics (Tier C) + +ARC proxy delegation (CSR → sign with the provider-supplied `X509Proxy` → upload; renew before +expiry; reuse across submissions where valid) is `ARCBackend`-internal. The provider supplies the +proxy; everything downstream — delegation ids, renewal, `AlwaysIncludeProxy`-style pairing with a +token — is invisible to the consumer. + +### 6. Boundaries + +- **SSH keys are transport configuration**, not rotating credentials: long-lived, file/agent-based, + no audience. They stay in `SSHTransport` config; revisit only if key rotation becomes a real + requirement. +- **Payload credentials are pilot-side.** Renewing the proxy *of the running payload* (DIRAC's + `_monitorProxy`, `GENERIC_PILOT` branch) belongs to the severed pilot/runner domain, not here. +- **Issuance is the consumer's.** interCEde never holds refresh tokens, client secrets, or CA + material; it asks a provider and uses what it gets. +- **Cloud provider auth** (Libcloud driver keys, the `cloud.auth` ini) is `CloudBackend` internal + config; whether it adopts the provider model is deferred with the Cloud backend itself. + +### 7. Usage sketches (informative) + +**(a) The consumer implements the provider once; every backend pulls from it.** A DiracX +submission task wires its token/proxy machinery into one object and never polices freshness +again (this is the code that today exists twice, in `SiteDirector._setCredentials` and +`WMSUtilities.setPilotCredentials`): + +```python +class DiracXPilotCredentials: # consumer-side; satisfies CredentialProvider + async def get(self, req: CredentialRequirements) -> CredentialSet: + creds = [] + if BearerToken in req.kinds: # mint scoped to what the backend declared + tok = await token_service.mint(audience=req.audience, scopes=req.scopes) + creds.append(BearerToken(tok.value, tok.expires_at)) + if X509Proxy in req.kinds: # gProxyManager lineage, behind the provider + pem = await proxy_store.download(pilot_dn, pilot_group, lifetime=86400) + creds.append(X509Proxy(pem, expires_at=proxy_expiry(pem))) + return CredentialSet(creds) + +backend = registry.backend( + {"type": "htcondor-ce", "endpoint": "ce01.example.org:9619", ...}, + credentials=DiracXPilotCredentials(), +) +async with backend: # lifecycle from IC-ADR-001 §2 + sub = await backend.submit(spec, count=50) + # before the operation the backend compared its cached CredentialSet against + # backend.credential_requirements and awaited provider.get(...) only if stale +``` + +**(b) What the HTCondorCE backend declares, and what it does with the set (Tier C).** The +requirements make the Motivation-7 pairing explicit and typed; the materialisation reproduces +DIRAC's mechanics without the caller knowing any of it: + +```python +backend.credential_requirements == CredentialRequirements( + kinds=frozenset({BearerToken, X509Proxy}), # all-of: token for the channel, + audience="ce01.example.org:9619", # proxy for site-side VOMS/APEL accounting +) +# internally, per operation: +# BearerToken -> 0600 temp file; _CONDOR_SEC_CLIENT_AUTHENTICATION_METHODS=SCITOKENS, +# _CONDOR_SCITOKENS_FILE= +# X509Proxy -> 0600 temp file; X509_USER_PROXY=, referenced by the submit +# description's `use_x509userproxy = true` +# both cleaned up at the operation/lifecycle boundary; values never logged +``` + +**(c) Integration stacks and standalone use — static files, no issuance machinery.** The +IC-ADR-002 stacks mint ephemeral credentials into the shared volume; the test harness (or any +non-DiracX user) wraps them statically: + +```python +provider = StaticCredentials(CredentialSet([ + BearerToken(Path("credentials/htcondor/idtoken").read_text().strip(), expires_at=None), +])) +backend = registry.backend({"type": "htcondor-ce", ...}, credentials=provider) +``` + +## Rationale + +- **Provider-pull over mutating setters.** Setters put hidden state on cached objects and force + every caller to police freshness (two DIRAC copies prove the cost). A pull model puts the + *check* in one library-side place while leaving the *source and policy* in consumer code — the + same mechanism/policy line IC-ADR-001 draws for throttling. +- **Declared requirements over tag sniffing.** `Tag: Token` conflates scheduling metadata with an + auth capability and is invisible to type checkers and tooling. A typed declaration is + enumerable, testable, and lets the conformance suite assert that a backend's declared kinds are + the ones it actually uses. +- **`CredentialSet` over single credential.** ARC's token+proxy pairing is a first-party + requirement, not an edge case; modelling it from day one avoids a v2 of the provider protocol. +- **Immutability.** Frozen values make "refresh" a *replacement*, never an in-place mutation — + cache-safe and race-free under concurrent bulk operations. + +## Rejected Ideas + +- **DIRAC-style `set_credential()` mutators.** Hidden state on cached backends; freshness policy + smeared across callers; the incumbent model this ADR exists to replace. +- **Backend-driven issuance/refresh** (backend holds a refresh token or talks to the IdP). + Couples the library to DiracX/IdP specifics, embeds secrets in the library, and moves policy + inside — the opposite of the IC-ADR-001 §9 mechanism/policy split. +- **Credentials in the `SubmissionSpec`** (name provisional — IC-ADR-001 Open Issues). Auth is + per-backend/channel, not per-job; putting it on + the spec would force every task (status, fetch, kill) to re-thread it and would leak payload vs + backend credential confusion back in. +- **A single `Credential` per backend (no set).** Breaks on ARC's token+proxy pairing. +- **Keeping the `Tag: Token[:vo]` opt-in.** Stringly-typed, CS-coupled, invisible to types and + tooling. +- **An interCEde-owned credential store/daemon.** interCEde is stateless (IC-ADR-001 §1); caching + beyond the in-memory current set is the consumer's business. + +## Open Issues + +- **Freshness margin.** Fixed library default vs per-backend/per-provider configuration; ARC + delegation renewal wants a larger margin than a bearer-token header. +- **Requirements granularity.** Whether `CredentialRequirements` stays per-backend or needs + per-operation variance (e.g. a backend whose *fetch* endpoint needs a different scope than + *submit*). Start per-backend; split only on evidence. +- **Proxy representation.** Raw PEM bytes (current spec) vs a `cryptography` object; and whether + delegation needs key material the consumer must supply alongside the chain (DIRAC signs the + delegation CSR with the proxy's own key — implies the provider hands over key+chain, which + `X509Proxy.pem` as "full chain" must be explicit about). +- **Multi-VO backends.** One backend instance serving several VOs would need per-VO credential + sets; today DIRAC instantiates per-queue/VO CEs, and interCEde's per-backend provider assumes + the same. Confirm with DiracX's multi-VO design. +- **DiracX alignment.** The provider implementation on the DiracX side (token service, scopes, + pilot-credential flows) — tracked with the DiracX transition ADRs, not here. +- **Proxy-for-accounting lifetime.** HTCondorCE's proxy requirement is a WLCG-transition artifact + (sites attribute usage via the proxy's VOMS attributes, the APEL pipeline). When sites account + on token claims instead, the backend's declared requirements shrink to token-only — and because + requirements are *data*, that is a backend-version/configuration change, not an API break. + Track the WLCG token-transition timeline before hard-coding the pairing as permanent. +- **Conformance coverage.** IC-ADR-002 already plans token-vs-proxy configuration axes for the + ARC stack; extend the conformance suite with a "requirements honesty" check (backend declares X, + suite verifies it authenticates with exactly X). diff --git a/docs/adr/IC-ADR-XXX_template.md b/docs/adr/IC-ADR-XXX_template.md new file mode 100644 index 0000000..fa37760 --- /dev/null +++ b/docs/adr/IC-ADR-XXX_template.md @@ -0,0 +1,45 @@ +# IC-ADR-\[NUMBER\]: [Title of Decision] + +## Metadata + +- **Created By:** [Name] +- **Date:** [YYYY-MM-DD] +- **Status:** [Draft | Accepted | Rejected | Deprecated by IC-ADR-YYY | Supersedes IC-ADR-XXX] +- **Decision Maker(s):** [Name(s)] +- **Stakeholders:** [Name(s) / Role(s), only used for decisions which affect a subset of communities] +- **Depends on:** [IC-ADR-YYY — optional; upstream ADRs this decision builds on] + +> **Scope and altitude.** [Optional but recommended, especially for long ADRs. Two to four +> sentences: what kind of decision this is (direction-setting vs detailed), what is deliberately +> deferred and to where, and which sections a reviewer must read versus may skim.] + +## Abstract + +A short (~200 word) summary of the decision being made and why it matters. Prefer one +plain-language sentence per decision over a dense paragraph — this is the only part many +stakeholders will read. + +## Motivation + +Why is this decision needed now? What problem or limitation in the current system does it address? What are the functional and non-functional drivers? + +## Specification + +Describe the chosen solution in concrete detail — APIs, interfaces, configuration, behaviour. This is the "what we're building" section. + +## Rationale + +Explain *why* the chosen design looks the way it does. Why these trade-offs? Why this level of abstraction? Connect specific design choices back to the drivers in Motivation. + +## Evolution + +Optional. How the decision accommodates future change: what can be added without breaking +(additive extensions), and what would require superseding this ADR. + +## Rejected Ideas + +Why were the non-chosen options ultimately set aside? This is distinct from the pros/cons listing above — it's the narrative of what tipped the scales. Include any ideas that came up in discussion but weren't even promoted to full options, and why. + +## Open Issues + +Any points still being decided or discussed. Remove this section once the status moves to Accepted. diff --git a/docs/adr/index.md b/docs/adr/index.md new file mode 100644 index 0000000..2a53f1b --- /dev/null +++ b/docs/adr/index.md @@ -0,0 +1,13 @@ +# Architecture Decision Records + +Architecture Decision Records (ADRs) document significant design decisions made in the interCEde project. + +| ADR | Title | Status | +| --- | ----- | ------ | +| [IC-ADR-001](IC-ADR-001_computing_elements.md) | Computing Element interfaces and the DIRAC to interCEde migration | Draft | +| [IC-ADR-002](IC-ADR-002_integration_tests.md) | Integration testing against containerized backends | Draft | +| [IC-ADR-003](IC-ADR-003_credentials.md) | Backend credentials — typed, declared, provider-supplied | Draft | + +## Template + +New ADRs should follow the [ADR template](IC-ADR-XXX_template.md). diff --git a/docs/notes/reuse-survey.md b/docs/notes/reuse-survey.md new file mode 100644 index 0000000..a7a9504 --- /dev/null +++ b/docs/notes/reuse-survey.md @@ -0,0 +1,68 @@ +# Prior-art survey: existing job-submission abstractions + +> Companion note to [IC-ADR-001](../adr/IC-ADR-001_computing_elements.md) (Rejected Ideas, +> *"Reuse an existing job-submission abstraction instead of defining our own"*). This records the +> survey evidence behind that rejection; the ADR keeps only the conclusion. Surveyed 2026-06/07 — +> per-tool facts here go stale independently of the decision. + +## What interCEde needs + +A modern async, typed Python library covering, together: SSH + local batch (Slurm, HTCondor, +LSF, …), ARC-CE over its REST interface, HTCondor-CE, and cloud (Libcloud-style providers). No +surveyed tool covers this set. + +## HTCondor GAHP / blahp + +Not a library but a *protocol* — no C/Python binding; you spawn a helper binary and speak ASCII +over stdin/stdout — and a *family* of binaries: `blahp` covers only **local** batch +(Slurm/PBS/LSF/SGE/Condor), while ARC and cloud are separate `arc_gahp`/`ec2_gahp`/`gce_gahp`/ +`azure_gahp`. blahp has **no SSH** — remote-cluster access is a `condor_gridmanager` + +`condor_remote_cluster`/BOSCO feature — so "SSH + Slurm" would drag in the whole HTCondor grid +stack. The protocol is internal, self-described as "SECOND DRAFT", with no back-compat guarantee. + +Apache-2.0. Its per-LRMS submit/status/cancel scripts and its async request-id/`RESULTS`-poll +model are worth *learning from* — they are cited in IC-ADR-001's Rationale as prior art for the +`Scheduler` protocol and the async, bulk, poll-based contract. + +The reverse direction — HTCondor reusing interCEde — is a non-goal: its gridmanager is C++ and +speaks a process protocol, and HTCondor-CE is a backend interCEde *targets*, so the layering runs +the other way. + +## ATLAS PanDA / Harvester + +Python, Apache-2.0, and has the exact submitter/monitor/sweeper plugin shape interCEde wants +(HTCondor, ARC via **aCT**, Slurm/LSF/PBS, cloud, k8s, HPC) — but its communicator talks only to +the PanDA server and its data model is PanDA specs, so it is not an importable standalone +library. Notably, ATLAS built Harvester *because* aCT was too tied to ARC-CE for US HPCs — the +same "generic beats CE-specific" lesson IC-ADR-001 applies. + +## ALICE JAliEn + +Its `alien.site.batchqueue` layer (ARC, HTCondor, direct Slurm/PBS, NERSC SuperFacility) is the +closest HEP analogue and does *both* grid-CE and direct-batch — but it is Java, never released as +an artifact (CVMFS-only), coupled to JAliEn's LDAP/central/token machinery, and of unconfirmed +licence. + +## Generic libraries + +RADICAL-SAGA, PSI/J, DRMAA, Parsl, AiiDA, Dask-Jobqueue are batch/HPC-scheduler oriented: the +ARC-CE REST **+** HTCondor-CE **+** cloud triple never co-occurs in any one of them. In fact +**none of the six speaks ARC at all**, in either the modern ARC-CE REST or the classic +ARC0/EMI-ES interface (RADICAL-SAGA never shipped a NorduGrid adaptor in any release — the Python +`radical.saga` is distinct from the old C++/Java SAGA; AiiDA has no `aiida-arc` plugin in core or +its registry). Where they touch "HTCondor" it is vanilla HTCondor/Condor-G, not the HTCondor-CE +grid gateway. + +ARC-CE REST is itself well-supported — HTCondor's `arc_gahp` speaks it in C++ (libcurl against +`/arex/rest/1.0`), and interCEde uses the Python `pyarcrest` — but only through ARC-specific +clients, never through one of these generic scheduler abstractions. + +Concurrency is *not* the differentiator: most of the six are async (futures/callbacks or a +non-blocking submit + poll/wait); DRMAA is the lone synchronous-style one, and it is unmaintained. + +## Conclusion + +Every HEP experiment builds its own CE abstraction and none is a reusable standalone library. The +genuinely reusable layer is the WLCG grid-CE ecosystem itself (HTCondor-CE, ARC-CE) plus +first-party per-backend clients (`pyarcrest`, the `htcondor` bindings, Apache Libcloud). +interCEde composes those behind a typed, async contract rather than adopting any monolith. From 793c4a9b17cfffd7350c17c09bc8e4599256a21c Mon Sep 17 00:00:00 2001 From: aldbr Date: Mon, 6 Jul 2026 09:29:20 +0200 Subject: [PATCH 2/7] feat: provide integration tests scaffolding --- .github/actions/pixi-setup/action.yml | 29 +++++ .github/workflows/ci.yml | 14 +-- .github/workflows/integration.yml | 102 +++++++++++++++ .gitignore | 3 + RENOVATE.md | 52 -------- docs/dev/explanations/index.md | 3 + docs/dev/explanations/integration-testing.md | 38 ++++++ docs/dev/how-to/add-an-integration-stack.md | 52 ++++++++ docs/dev/how-to/index.md | 4 + .../how-to/run-integration-tests-locally.md | 56 +++++++++ docs/dev/index.md | 11 ++ docs/dev/reference/index.md | 5 + docs/dev/reference/integration-images.md | 48 +++++++ docs/dev/reference/integration-stacks.md | 107 ++++++++++++++++ docs/dev/reference/renovate.md | 75 +++++++++++ docs/dev/tutorials/index.md | 6 + pyproject.toml | 9 ++ renovate.json | 39 +++++- tests/conftest.py | 29 +++++ tests/integration/README.md | 20 +++ tests/integration/__init__.py | 0 tests/integration/conftest.py | 71 +++++++++++ tests/integration/generate_matrix.py | 117 ++++++++++++++++++ tests/integration/stacks.yml | 31 +++++ tests/integration/stacks/_images/README.md | 7 ++ .../integration/stacks/_template/compose.yml | 62 ++++++++++ .../config/basic/intercede-client.toml | 20 +++ tests/integration/test_manifest.py | 84 +++++++++++++ 28 files changed, 1028 insertions(+), 66 deletions(-) create mode 100644 .github/actions/pixi-setup/action.yml create mode 100644 .github/workflows/integration.yml delete mode 100644 RENOVATE.md create mode 100644 docs/dev/explanations/index.md create mode 100644 docs/dev/explanations/integration-testing.md create mode 100644 docs/dev/how-to/add-an-integration-stack.md create mode 100644 docs/dev/how-to/index.md create mode 100644 docs/dev/how-to/run-integration-tests-locally.md create mode 100644 docs/dev/index.md create mode 100644 docs/dev/reference/index.md create mode 100644 docs/dev/reference/integration-images.md create mode 100644 docs/dev/reference/integration-stacks.md create mode 100644 docs/dev/reference/renovate.md create mode 100644 docs/dev/tutorials/index.md create mode 100644 tests/conftest.py create mode 100644 tests/integration/README.md create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/generate_matrix.py create mode 100644 tests/integration/stacks.yml create mode 100644 tests/integration/stacks/_images/README.md create mode 100644 tests/integration/stacks/_template/compose.yml create mode 100644 tests/integration/stacks/_template/config/basic/intercede-client.toml create mode 100644 tests/integration/test_manifest.py diff --git a/.github/actions/pixi-setup/action.yml b/.github/actions/pixi-setup/action.yml new file mode 100644 index 0000000..d4accdb --- /dev/null +++ b/.github/actions/pixi-setup/action.yml @@ -0,0 +1,29 @@ +name: Set up pixi +description: > + Install pixi and the requested environment(s), applying the editable-install workaround for + prefix-dev/pixi#3762 before the environment is solved. Shared by ci.yml and integration.yml so + the workaround lives in one place. + +inputs: + environments: + description: Space-separated pixi environment name(s) to install (e.g. "py313"). + required: true + +runs: + using: composite + steps: + - uses: prefix-dev/setup-pixi@v0.9.3 + with: + run-install: false + post-cleanup: false + - name: Workaround for prefix-dev/pixi#3762 + shell: bash + run: | + # Editable installs break under pixi in CI; pin to non-editable before the env is solved. + sed -i.bak 's@editable = true@editable = false@g' pyproject.toml + rm pyproject.toml.bak + git diff --stat # show what the workaround changed + - uses: prefix-dev/setup-pixi@v0.9.3 + with: + cache: false + environments: ${{ inputs.environments }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de4534b..04b1af7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,20 +38,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v6 - - uses: prefix-dev/setup-pixi@v0.9.3 + - uses: ./.github/actions/pixi-setup with: - run-install: false - post-cleanup: false - - name: Apply workarounds - run: | - # Workaround for https://github.com/prefix-dev/pixi/issues/3762 - sed -i.bak 's@editable = true@editable = false@g' pyproject.toml - rm pyproject.toml.bak - # Show any changes - git diff - - uses: prefix-dev/setup-pixi@v0.9.3 - with: - cache: false environments: ${{ matrix.python-version }} - name: Run pytest run: pixi run -e ${{ matrix.python-version }} pytest diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..cd8313a --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,102 @@ +# Skeleton of the manifest-driven integration workflow (IC-ADR-002 §5; scaffold issue #9). +# +# The matrix is generated from tests/integration/stacks.yml — adding a stack, configuration or +# version is a manifest entry plus files, never a workflow edit. generate_matrix.py emits jobs only +# for *harmonized* stacks (those with a stacks//compose.yml); stacks that are seeded in the +# manifest but not yet harmonized are announced with a `::notice` in the matrix job and produce no +# job here (so this workflow has zero jobs until the first stack lands). +# +# Triggers on any PR that touches the integration suite (so a stack PR self-tests the moment it +# lands files — no workflow edit), plus manual/reusable dispatch. It is NOT yet gating: until issue +# #14 wires it into ci.yml, has CI pull prebuilt images from GHCR (instead of building on the +# runner), and makes it a required check, a red run here does not block merges. +name: Integration tests + +on: + workflow_dispatch: + workflow_call: + pull_request: + paths: + - "tests/integration/**" + - ".github/workflows/integration.yml" + - ".github/actions/pixi-setup/**" + +permissions: + contents: read + +jobs: + matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v6 + - name: Generate {stack x config x version} matrix from stacks.yml + id: gen + # Same script developers run locally (without --github-output) to preview the matrix. It + # also filters to harmonized stacks and prints a notice for the seeded-only ones. + # PyYAML ships with the runner's system python; `pip install pyyaml` here if that changes. + run: python3 tests/integration/generate_matrix.py --github-output + + integration: + needs: matrix + if: needs.matrix.outputs.matrix != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.matrix.outputs.matrix) }} + name: ${{ matrix.stack }} / ${{ matrix.config }} / ${{ matrix.version }} + env: + COMPOSE_FILE: tests/integration/stacks/${{ matrix.compose_stack }}/compose.yml + BACKEND_VERSION: ${{ matrix.version }} + steps: + - uses: actions/checkout@v6 + + # Every job in the matrix is a harmonized stack (the matrix job filtered out the rest), so + # compose.yml is guaranteed present — no per-step "has this stack landed?" guard is needed. + - name: Start the stack (healthchecks gate readiness) + run: docker compose -f "$COMPOSE_FILE" up --wait + + - uses: ./.github/actions/pixi-setup + if: matrix.exec_in_container == false + with: + environments: py313 + + - name: Run the contract suite + run: | + if [ "${{ matrix.exec_in_container }}" = "true" ]; then + # local-* stacks: run pytest inside the batch container (IC-ADR-002 §1). The batch + # scheduler typically refuses to run as root (HTCondor, Slurm), so the image must ship + # an unprivileged test user and the suite + pytest must be mounted or baked in. + # TODO(#13/#14): mount/install the suite and pin the exec user (compose `user:` or + # `exec --user `); see docs/dev/reference/integration-stacks.md. + runner=(docker compose -f "$COMPOSE_FILE" exec -T backend pytest) + else + runner=(pixi run -e py313 pytest) + fi + set +e # keep going so exit 5 (no tests selected) can be handled below + "${runner[@]}" tests/integration -m "${{ matrix.markers }}" \ + --stack="${{ matrix.stack }}" --config="${{ matrix.config }}" + rc=$? + # pytest exits 5 when the stack's markers select no tests. Phase 0 ships only the + # manifest tests, so the first harmonized stack would otherwise go red for "no tests + # ran"; tolerate exit 5 with a notice until the Phase-2 contract suite lands. + if [ "$rc" -eq 5 ]; then + echo "::notice::stack '${{ matrix.stack }}': no tests matched markers '${{ matrix.markers }}' (pytest exit 5) - passing until the contract suite lands" + rc=0 + fi + exit "$rc" + + - name: Collect daemon logs + if: failure() + # Daemons run in the foreground and log to stdout/stderr (IC-ADR-002 §3), so compose logs + # are the whole failure story — a file-logging daemon redirects its log to /dev/stdout. + run: docker compose -f "$COMPOSE_FILE" logs --no-color > compose-logs.txt || true + + - name: Upload logs as artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.stack }}-${{ matrix.config }}-${{ matrix.version }} + path: compose-logs.txt diff --git a/.gitignore b/.gitignore index dee8f1b..89c486e 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,9 @@ htmlcov/ #remove in case we want a specific LHCb one .pylintrc +# Integration-stack credentials: minted per run into a bind mount, never committed (IC-ADR-002 §4) +tests/integration/stacks/*/credentials/ + # docs (auto-generated build output only; docs/assets is tracked) docs/_build/ site/ diff --git a/RENOVATE.md b/RENOVATE.md deleted file mode 100644 index a7075f1..0000000 --- a/RENOVATE.md +++ /dev/null @@ -1,52 +0,0 @@ -# Renovate Configuration - -This repository uses [Renovate](https://www.mend.io/renovate/) to automatically keep dependencies up to date. - -## Configuration - -The Renovate configuration is defined in `renovate.json` and includes: - -### Dependency Groups - -1. **Python dependencies** - Core Python packages from `pyproject.toml` -2. **Testing dependencies** - Pytest and related testing tools -3. **Build dependencies** - Hatchling and build tools -4. **GitHub Actions** - GitHub Actions workflow dependencies - -### Update Strategy - -- **Minimum release age**: Updates are only proposed once a release is at least 7 days old (30 days for major Python updates) -- **Update types**: Minor, patch, pin, and digest updates -- **Grouping**: Related dependencies are grouped together -- **Major versions**: Handled separately for better control -- **Automerge**: Disabled (requires manual review) - -### Security - -- Vulnerability alerts are enabled -- Security updates are labeled and prioritized - -## How It Works - -1. Renovate checks for dependency updates on schedule -2. Creates pull requests for available updates -3. Groups related dependencies together -4. Applies labels and follows the project's update strategy -5. Requires manual review and approval before merging - -## Customization - -To modify the Renovate behavior: - -1. Edit `renovate.json` -2. Adjust package rules, schedules, or grouping strategies -3. Refer to the [Renovate documentation](https://docs.renovatebot.com/) for details - -## Ignored Paths - -Renovate ignores: -- `.pixi/` environments -- `__pycache__/` directories -- Virtual environments - -This ensures Renovate only updates the actual dependency definitions in `pyproject.toml` and GitHub Actions workflows. diff --git a/docs/dev/explanations/index.md b/docs/dev/explanations/index.md new file mode 100644 index 0000000..0f02a04 --- /dev/null +++ b/docs/dev/explanations/index.md @@ -0,0 +1,3 @@ +# Explanations + +- [Why integration stacks — testing against real backend daemons](integration-testing.md) diff --git a/docs/dev/explanations/integration-testing.md b/docs/dev/explanations/integration-testing.md new file mode 100644 index 0000000..f8210de --- /dev/null +++ b/docs/dev/explanations/integration-testing.md @@ -0,0 +1,38 @@ +# Why integration stacks — testing against real backend daemons + +The full reasoning is [IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md); this page is the +short version plus the phasing plan. + +## Why real daemons + +interCEde's architecture is deliberately mock-friendly — everything is a `Protocol`, and the unit +suite exercises contracts against fakes. That is also its greatest testing risk: a structural +protocol and a fake will happily agree with each other forever while the real ARC REST endpoint +or the real `condor_submit -spool` handshake drift away underneath. The bugs interCEde exists to +absorb (version-specific daemon behaviour, destructive output retrieval, status-mapping quirks) +are exactly the bugs unit tests structurally cannot see — HTCondor ≥ 25.8 silently turning +spooled-output retrieval into a one-shot operation is the canonical example. + +So the integration suite runs **real backend daemons** in containerised **stacks**, on every pull +request, and every backend version is pinned and tracked by Renovate so an upstream release that +breaks a contract surfaces as a red, bisectable PR. + +## Capability symmetry + +The integration suite is the executable form of the contract: it narrows to optional capabilities +exactly the way the library's consumers do (`isinstance` against a `runtime_checkable` protocol) +and skips where a backend genuinely lacks a capability. If a capability check works in the test +harness, it works for DiracX; if a backend's structural typing lies, a real daemon is where the lie +is caught. The skip helper that does this lands with the Phase-2 contract suite that calls it — +its signature would otherwise be guessed ahead of its callers. + +## Phase 0 vs Phase 2 + +**Phase 0 (now):** the `intercede` package's protocols don't exist yet, so stack PRs validate +their stacks with **native client tools** (`arcsub`/`arcstat`/`arcget`, `condor_submit`, +`sbatch`) in marker-selected test modules. + +**Phase 2:** a single backend-agnostic **contract suite** (`test_submit.py`, `test_status.py`, +`test_output.py`, `test_kill.py`) written against the interCEde protocols replaces the +native-tool tests. Backend-specific semantics (e.g. HTCondor's destructive fetch) get dedicated +marker-selected tests (`destructive_fetch`) rather than conditionals inside generic ones. diff --git a/docs/dev/how-to/add-an-integration-stack.md b/docs/dev/how-to/add-an-integration-stack.md new file mode 100644 index 0000000..f1128a9 --- /dev/null +++ b/docs/dev/how-to/add-an-integration-stack.md @@ -0,0 +1,52 @@ +# Add an integration stack + +A *stack* is a named, self-contained backend environment (one `docker compose` file plus +versioned configuration) that the integration suite runs against. Adding one requires **no +workflow edits** — the CI matrix is generated from the manifest. + +Before starting, read the [stack contract](../reference/integration-stacks.md) — the rules below +reference it. + +## Steps + +1. **Copy the template**: `tests/integration/stacks/_template/` → + `tests/integration/stacks//`. Fill in `compose.yml` (real healthcheck, host-readable + `./credentials` bind mount) and `config/basic/` (backend config + `intercede-client.toml`, + which change together as one atomic configuration). An **aliased** stack (`stack:` reusing + another's compose) ships only `config/` under its own id — no `compose.yml` of its own — and its + `markers` exclude the borrowed transport (e.g. `and not ssh`). +2. **Add the image** under `tests/integration/stacks/_images//` following the + [image conventions](../reference/integration-images.md) (EL9, pinned Renovate-annotated + `ARG`s, foreground daemons, ephemeral credentials at start). +3. **Add the manifest entry** in `tests/integration/stacks.yml` (`id`, `configs`, `markers`, + optionally `versions` / `stack:` alias / `exec_in_container`). +4. **Verify locally**: + + ```sh + docker compose -f tests/integration/stacks//compose.yml up --wait + pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic + pixi run -e py313 pytest tests/integration/test_manifest.py # layout checks + ``` + +## Harmonization checklist + +For a stack PR (including the in-flight prototype stacks converging onto this layout): + +- [ ] Files under `stacks//` following `_template/` (compose.yml + `config/basic/`) +- [ ] `docker compose … up --wait` succeeds locally; healthchecks probe real readiness +- [ ] Daemons foregrounded (no systemd), logging to stdout/stderr; Docker (not Podman) on + EL9-based images +- [ ] Dockerfile under `stacks/_images//` with pinned, `# renovate:`-annotated `ARG`s +- [ ] Credentials generated at start, world-readable, into the host-readable `./credentials` bind + mount under `credentials//…` (gitignored) +- [ ] `intercede-client.toml` filled in (own `id`, endpoint, credential type/path, queue); a + host-run stack publishes its port on localhost with a matching Test-CA SAN +- [ ] `stacks.yml` entry present; `pytest tests/integration/test_manifest.py` green +- [ ] At least one marker-selected Phase-0 smoke test ships with the stack (so its job isn't just + "no tests ran" — the workflow tolerates that, but a smoke test makes the job meaningful) +- [ ] Bespoke per-PR workflow removed: the manifest-driven `integration.yml` now runs + automatically on any PR touching `tests/integration/**`, so the stack self-tests. It stays + **non-blocking** until issue #14 (GHCR image pulls + `ci.yml` invocation + required check), + so keep a bespoke workflow only if you need a *gating* check in the interim +- [ ] `docker compose logs` captures the daemons' output on failure (daemons log to stdout; a + file-logging daemon redirects to `/dev/stdout`) diff --git a/docs/dev/how-to/index.md b/docs/dev/how-to/index.md new file mode 100644 index 0000000..f039145 --- /dev/null +++ b/docs/dev/how-to/index.md @@ -0,0 +1,4 @@ +# How-to guides + +- [Run the integration tests locally](run-integration-tests-locally.md) +- [Add an integration stack](add-an-integration-stack.md) diff --git a/docs/dev/how-to/run-integration-tests-locally.md b/docs/dev/how-to/run-integration-tests-locally.md new file mode 100644 index 0000000..9225a9b --- /dev/null +++ b/docs/dev/how-to/run-integration-tests-locally.md @@ -0,0 +1,56 @@ +# Run the integration tests locally + +Everything CI does is reproducible on a laptop — a design rule of +[IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md): CI does nothing a developer cannot do +locally. + +## Run one stack + +1. Start the stack; healthchecks gate readiness, so when `up --wait` returns the backend is + usable: + + ```sh + docker compose -f tests/integration/stacks//compose.yml up --wait + ``` + +2. Run the applicable test subset — the stack's `markers:` expression is in + `tests/integration/stacks.yml`: + + ```sh + pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic + ``` + +For `local-*` stacks (`exec_in_container: true`) the tests run *inside* the batch container +instead: + +```sh +docker compose -f tests/integration/stacks//compose.yml exec --user backend \ + pytest tests/integration -m "" --stack= --config=basic +``` + +Two conditions this recipe depends on (both open TODOs, #13/#14): the image must ship the suite +and pytest (mounted or baked in), and it must run as an **unprivileged** user — HTCondor and Slurm +refuse to run jobs as root, hence `--user ` (or a compose `user:`). Note `--stack` +still names the aliased entry (e.g. `local-slurm`) so its own `intercede-client.toml` is used. + +Stacks double as development environments: leave the stack up and point your client config at +`localhost` to develop against a real backend. + +## Preview the CI matrix + +The CI matrix is generated from `stacks.yml` by a plain script — run it locally to see exactly +which `{stack × config × version}` jobs CI would create: + +```sh +pixi run -e py313 python tests/integration/generate_matrix.py +``` + +(The workflow calls the same script with `--github-output`.) + +## Manifest-only checks (no Docker needed) + +The manifest schema and stack-layout checks run as ordinary unit tests: + +```sh +pixi run -e py313 pytest tests/integration/test_manifest.py +``` diff --git a/docs/dev/index.md b/docs/dev/index.md new file mode 100644 index 0000000..ae92204 --- /dev/null +++ b/docs/dev/index.md @@ -0,0 +1,11 @@ +# Developer documentation + +Organised following [Diátaxis](https://diataxis.fr/), as in the other DiracX-family projects: + +- [Tutorials](tutorials/index.md) — learning-oriented, step-by-step lessons +- [How-to guides](how-to/index.md) — task-oriented recipes +- [Reference](reference/index.md) — information-oriented technical description +- [Explanations](explanations/index.md) — understanding-oriented discussion + +Architecture decisions are recorded separately in the [ADRs](../adr/index.md). User-facing +documentation (`docs/user/`) is deferred until the library has a public API to document. diff --git a/docs/dev/reference/index.md b/docs/dev/reference/index.md new file mode 100644 index 0000000..c1fc846 --- /dev/null +++ b/docs/dev/reference/index.md @@ -0,0 +1,5 @@ +# Reference + +- [Integration stacks — layout, manifest schema, stack contract](integration-stacks.md) +- [Integration images — Dockerfile conventions](integration-images.md) +- [Renovate — dependency and version tracking](renovate.md) diff --git a/docs/dev/reference/integration-images.md b/docs/dev/reference/integration-images.md new file mode 100644 index 0000000..8082fe4 --- /dev/null +++ b/docs/dev/reference/integration-images.md @@ -0,0 +1,48 @@ +# Integration images — Dockerfile conventions + +Dockerfiles under `tests/integration/stacks/_images/` are shared across stacks (`arc/`, `slurm/`, +`htcondor-ce/`, …) and follow [IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md) §3 and §6: + +1. **EL9 base** (`almalinux:9`), pinned by tag; backend daemons installed from **pinned RPMs**. +2. **Every version is a Renovate-annotated `ARG`** so upstream releases arrive as red-or-green + Renovate PRs (Lane 1): + + ```dockerfile + FROM almalinux:9 + + # renovate: datasource=repology depName=fedora_epel_9/nordugrid-arc + ARG ARC_VERSION=7.1.1 + + RUN dnf install -y epel-release \ + && dnf install -y "nordugrid-arc-arex-${ARC_VERSION}" \ + && dnf clean all + ``` + + The `dockerfile` + `custom.regex` managers that read these annotations are already enabled in + [`renovate.json`](../../../renovate.json) (per-backend grouping and the leading-edge/anchor + convention are documented in [the Renovate reference](renovate.md)) — **adding a stack needs no + `renovate.json` edit**, only the annotated `ARG`. The annotation carries its own `datasource`, + so the exact datasource for the NorduGrid/EL9 repos (an open point of IC-ADR-002) is chosen when + the ARC Dockerfile lands, without touching the manager config. +3. **Foreground entrypoint** — daemons run under a plain entrypoint script or minimal supervisor, + never systemd, and **log to stdout/stderr** so `docker compose logs` captures everything (a + file-logging daemon redirects its log to `/dev/stdout`). The CI failure artifact is compose + logs only; there is no mounted log-dir upload. +4. **Ephemeral credentials at start**: the entrypoint mints whatever the stack needs (ARC: + `arcctl test-ca` CA + host cert + client credential; HTCondor(-CE): IDTOKENS) and writes + client-side material world-readable into the host-readable `./credentials` bind mount + (`credentials//…`, IC-ADR-002 §4). A TLS host certificate carries a SAN for the name the + client dials (the compose hostname *and* `localhost` for host-run stacks); ARC on EL9 needs the + crypto-policy set to `LEGACY` for the proxy chain. +5. **Version windows**: where a backend declares an anchor + leading edge (IC-ADR-002 §8), the + image accepts the version as `ARG`/build-arg so the same Dockerfile serves every entry of the + `versions:` axis; only the leading-edge `ARG` default is Renovate-tracked. +6. **Roles and intra-stack secrets** (multi-service stacks). A CE + LRMS or a Slurm + controller + worker is one parameterized image, not several: a `case "$ROLE"` entrypoint + selects the daemon per service rather than duplicating Dockerfiles. Secrets shared *between* + services (a munge key, an internal CA) obey the same rule as client credentials — minted at + container start into a stack-internal volume, **never baked into the image** (a GHCR-published + image with a baked munge key is a leaked credential). + +Images are built and pushed to `ghcr.io/diracgrid/intercede-testenv/:` by the +`build-images` workflow; until that lands, stacks may `build:` locally from these directories. diff --git a/docs/dev/reference/integration-stacks.md b/docs/dev/reference/integration-stacks.md new file mode 100644 index 0000000..d1cebca --- /dev/null +++ b/docs/dev/reference/integration-stacks.md @@ -0,0 +1,107 @@ +# Integration stacks — layout, manifest schema, stack contract + +Reference for the integration-test infrastructure decided in +[IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md). + +## Layout + +``` +tests/integration/ +├── stacks.yml # THE manifest: single source of truth for the CI matrix +├── conftest.py # --stack/--config options, stack fixtures, capability skip +├── generate_matrix.py # manifest → CI matrix expansion (runnable locally) +├── test_manifest.py # manifest schema + layout checks (run in plain unit CI) +├── test_*.py # the backend-agnostic contract suite (Phase 2) +└── stacks/ + ├── _template/ # copy me: compose.yml + config/basic/intercede-client.toml + ├── _images/ # shared Dockerfiles (see integration-images.md) + └── / + ├── compose.yml + └── config/ + └── basic/ # backend config + intercede-client.toml, changed together +``` + +## Manifest schema (`stacks.yml`) + +| Field | Required | Meaning | +| --- | --- | --- | +| `id` | yes | Stack name; also the directory under `stacks/` unless `stack:` is set | +| `stack` | no | Alias: reuse another stack's compose environment (e.g. `local-slurm` → `ssh-slurm`) | +| `configs` | yes | Configuration names under `stacks//config/`; start with `[basic]` | +| `versions` | no | Backend support-window entries to run (IC-ADR-002 §8); absent = single leading-edge pin | +| `markers` | yes | `pytest -m` expression selecting the applicable contract-suite subset | +| `exec_in_container` | no | Run pytest inside the backend container instead of against it (default `false`) | + +The CI matrix is the expansion `{stack × config × version}` produced by +`tests/integration/generate_matrix.py`; each matrix entry becomes its own CI job. Only +*harmonized* stacks — those with a `stacks//compose.yml` — become jobs; a +seeded-only entry is announced with a `::notice` by `generate_matrix.py` and skipped. + +**Aliases resolve two ways.** `stack:` reuses another stack's *compose environment* (so +`local-slurm` starts `ssh-slurm`'s containers), but the *client configuration* is always resolved +from the entry's **own** id: `local-slurm` ships its own `stacks/local-slurm/config/basic/ +intercede-client.toml` (a different `kind`/endpoint — Local vs SSH transport) and its `markers` +exclude the borrowed transport's tests (`scheduler and slurm and not ssh` — the pattern every +future `local-*` alias needs so an `ssh`-marked test does not run in both the ssh and local jobs). + +**`exec_in_container: true` runs pytest inside the batch container** (Local transport, no port 22). +The image must ship the suite + pytest — mounted or baked in (open TODO #13/#14) — and an +unprivileged test user, since HTCondor and Slurm refuse to run jobs as root; the workflow (and the +local `compose exec` recipe) pin that user with compose `user:` / `exec --user`. + +## The stack contract + +A stack that lands in `stacks//` commits to all of the following (enforced partly by +`test_manifest.py`, partly by review): + +1. **One `compose.yml`, runnable locally.** CI does nothing a developer cannot do on a laptop + (see [the how-to](../how-to/run-integration-tests-locally.md)). +2. **Healthchecks on every service.** `up --wait` gates on them; hand-rolled sleep-and-retry + loops are not accepted. +3. **Foreground daemons, never systemd.** A plain entrypoint or minimal supervisor — privileged + systemd containers are fragile on GitHub runners and hide daemon logs from `docker logs` + (IC-ADR-002 §3). Daemons **log to stdout/stderr** so `docker compose logs` is the whole failure + story; a file-logging daemon redirects its log to `/dev/stdout` rather than expecting a bespoke + log-directory upload (the CI failure artifact is compose logs only). +4. **Images from `stacks/_images/`,** EL9 base, pinned Renovate-annotated `ARG`s (see + [image conventions](integration-images.md)). +5. **Ephemeral credentials, minted at container start** into the shared, host-readable + `./credentials` bind mount (gitignored) — never committed, and never a Docker *named* volume + (host pytest cannot read a root-owned named volume for `exec_in_container: false` stacks). The + layout `credentials//…` is the interface between stack and harness (IC-ADR-002 §4). Mint + throwaway Test-CA material world-readable; the harness copies anything needing private modes + (0400/0600) into a private dir itself. +6. **Configurations are atomic directories.** `config//` holds the backend config *and* + `intercede-client.toml` together; a new auth/filesystem/queue variant is a new directory plus + a `configs:` entry in `stacks.yml`, never a mutation of `basic` (IC-ADR-002 §7). +7. **One `stacks.yml` entry.** That is how CI finds the stack — adding a stack requires no + workflow edits. +8. **Host reachability + TLS naming** (host-run stacks). A stack the harness drives from the host + (`exec_in_container: false`) publishes its backend port on localhost (compose `ports:`) and + records the same `host:port` in `intercede-client.toml`. A TLS backend's Test-CA host + certificate must carry a SAN for the name the client actually dials — mint it for both the + compose hostname and `localhost`. ARC on EL9 also needs the system crypto-policy set to `LEGACY` + for the proxy chain to validate. + +## Test harness + +Options and fixtures provided by `tests/integration/conftest.py`: + +| Name | Kind | Meaning | +| --- | --- | --- | +| `--stack` | option | Stack id from `stacks.yml`; stack-bound tests self-skip when absent (registered in `tests/conftest.py`, so every `pytest` invocation form accepts it) | +| `--config` | option | Configuration name (default `basic`) | +| `stack` | fixture | The selected manifest entry | +| `stack_dir` | fixture | Directory with the stack's `compose.yml` (follows the `stack:` alias) | +| `client_config` | fixture | Parsed `intercede-client.toml`, resolved under the entry's *own* id (not the alias) | + +Markers, registered in `pyproject.toml`: `remote`, `scheduler`, `arc`, `htcondor`, `slurm`, +`ssh`, `destructive_fetch`. Each stack's `markers:` expression selects its subset. The +capability-narrowing skip helper (`isinstance` against a `runtime_checkable` protocol) lands with +the Phase-2 contract suite that calls it, alongside the interCEde protocols — it is intentionally +not part of this scaffold, whose signature would otherwise be guessed ahead of its callers. + +A *selected* stack whose `intercede-client.toml` is missing **fails** under CI (`$CI` set) rather +than skipping: an all-skipped run would report a green integration cell that verified nothing +(IC-ADR-002 §5). Locally the same case skips, which stays convenient while a stack is being +brought up. diff --git a/docs/dev/reference/renovate.md b/docs/dev/reference/renovate.md new file mode 100644 index 0000000..0b13613 --- /dev/null +++ b/docs/dev/reference/renovate.md @@ -0,0 +1,75 @@ +# Renovate — dependency and version tracking + +This repository uses [Renovate](https://www.mend.io/renovate/) to keep dependencies up to date. +The configuration is defined in [`renovate.json`](../../../renovate.json). + +## Dependency groups + +1. **Python dependencies** — core Python packages from `pyproject.toml` +2. **Testing dependencies** — Pytest and related testing tools +3. **Build dependencies** — Hatchling and build tools +4. **GitHub Actions** — GitHub Actions workflow dependencies +5. **Integration-stack backends** — ARC, HTCondor(-CE), and Slurm daemon versions, grouped per + backend so a red bump PR names its culprit (see below) + +## Update strategy + +- **Minimum release age**: updates are proposed only once a release is at least 7 days old (30 days + for major Python updates) +- **Update types**: minor, patch, pin, and digest updates +- **Grouping**: related dependencies are grouped together +- **Major versions**: handled separately for better control +- **Automerge**: disabled (requires manual review) +- **Security**: vulnerability alerts are enabled; security updates are labeled and prioritized + +## Integration stack backends + +The integration stacks ([IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md) §6) pin the backend +daemon versions they test against, and Renovate tracks those pins so a new upstream release arrives +as a reviewable red-or-green PR rather than tribal knowledge. Three managers are enabled for this: + +- **`docker-compose`** — upstream image tags referenced directly in a stack's `compose.yml` + (e.g. `htcondor/mini:`). +- **`dockerfile`** — base images in `tests/integration/stacks/_images/*/Dockerfile`. +- **`custom.regex`** — pinned RPM/package versions carried as `# renovate:`-annotated `ARG`s in + those Dockerfiles. The annotation itself declares the datasource, depName and (optionally) + versioning: + + ```dockerfile + # renovate: datasource=repology depName=fedora_epel_9/nordugrid-arc + ARG ARC_VERSION=7.1.1 + ``` + +**Adding a stack needs no `renovate.json` edit.** A stack author writes the annotated `ARG`; the +manager and per-backend grouping already configured here pick it up. Bumps are grouped per backend +(`arc`, `htcondor`, `slurm`) and never mixed. + +**Version windows (§8) are half-automatic.** Renovate tracks the *leading-edge* `ARG` normally. +The *older anchor* versions (e.g. an HTCondor LTS line) are human-managed: pin them and add a +`packageRule` with `allowedVersions`/`matchCurrentVersion` (or `enabled: false`) — landed alongside +the stack that introduces the anchor — so Renovate offers only in-major patch bumps and never +silently moves the support window. + +## How it works + +1. Renovate checks for dependency updates on schedule +2. Creates pull requests for available updates +3. Groups related dependencies together +4. Applies labels and follows the project's update strategy +5. Requires manual review and approval before merging + +## Changing the configuration + +Edit [`renovate.json`](../../../renovate.json) and adjust package rules, schedules, or grouping +strategies (see the [Renovate documentation](https://docs.renovatebot.com/)). Validate before +committing: + +```sh +npx --package renovate renovate-config-validator renovate.json +``` + +## Ignored paths + +Renovate ignores `.pixi/` environments, `__pycache__/` directories, and virtual environments, so it +only updates the actual dependency definitions in `pyproject.toml`, GitHub Actions workflows, and +the integration stacks' image / `ARG` version pins. diff --git a/docs/dev/tutorials/index.md b/docs/dev/tutorials/index.md new file mode 100644 index 0000000..9b9d039 --- /dev/null +++ b/docs/dev/tutorials/index.md @@ -0,0 +1,6 @@ +# Tutorials + +None yet — the first candidate, once a stack has landed, is *"Run your first integration stack +locally"* (boot a stack, submit a job through the native client tools, watch it complete). + +For task-oriented material in the meantime, see the [how-to guides](../how-to/index.md). diff --git a/pyproject.toml b/pyproject.toml index 6e6f077..ca0b2fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ testing = [ "pytest", "pytest-asyncio", "pytest-xdist", + "pyyaml", "ty", ] @@ -49,6 +50,14 @@ asyncio_mode = "auto" markers = [ "backend_only: restrict a test to the named backend fixture(s)", "xfail_backend: strict xfail a test on the named backend fixture(s)", + # Integration-suite markers (IC-ADR-002 §2): stacks.yml selects per-stack subsets with -m. + "remote: drives a remote CE backend (grid gateway: ARC, HTCondor-CE)", + "scheduler: drives a Transport x Scheduler composition (BatchBackend)", + "arc: requires an ARC CE", + "htcondor: requires an HTCondor schedd or HTCondor-CE", + "slurm: requires a Slurm cluster", + "ssh: requires an sshd-reachable batch host", + "destructive_fetch: asserts one-shot/destructive output-retrieval semantics", ] diff --git a/renovate.json b/renovate.json index 437d827..5ec1516 100644 --- a/renovate.json +++ b/renovate.json @@ -7,7 +7,10 @@ "enabledManagers": [ "pip_requirements", "pep621", - "github-actions" + "github-actions", + "dockerfile", + "docker-compose", + "custom.regex" ], "python": { "rangeStrategy": "auto" @@ -18,6 +21,19 @@ "**/venv/**", "**/env/**" ], + "customManagers": [ + { + "customType": "regex", + "description": "Track pinned backend versions in stack image Dockerfiles via `# renovate:`-annotated ARGs (IC-ADR-002 §6). The datasource/depName/versioning ride in the annotation, so a new stack image needs no renovate.json edit.", + "fileMatch": [ + "(^|/)tests/integration/stacks/_images/.+/Dockerfile$" + ], + "matchStrings": [ + "# renovate: datasource=(?[a-z-]+?) depName=(?[^\\s]+?)(?: versioning=(?[^\\s]+?))?\\s+ARG [A-Za-z0-9_]+?_VERSION=\"?(?[^\"\\s]+)\"?" + ], + "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}loose{{/if}}" + } + ], "packageRules": [ { "description": "Group all GitHub Actions updates together", @@ -63,6 +79,27 @@ "matchPackagePatterns": ["pytest.*", "ty"], "groupName": "Testing dependencies", "labels": ["dependencies", "testing"] + }, + { + "description": "Group ARC backend bumps, never mixed with other backends so a red PR names its culprit (IC-ADR-002 §6). Renovate tracks the leading-edge ARG; anchors are human-managed (§8).", + "matchManagers": ["dockerfile", "docker-compose", "custom.regex"], + "matchPackageNames": ["/nordugrid-arc/"], + "groupName": "ARC backend (integration stack)", + "labels": ["dependencies", "integration-stack", "arc"] + }, + { + "description": "Group HTCondor / HTCondor-CE backend bumps (IC-ADR-002 §6, §8)", + "matchManagers": ["dockerfile", "docker-compose", "custom.regex"], + "matchPackageNames": ["/htcondor/"], + "groupName": "HTCondor backend (integration stack)", + "labels": ["dependencies", "integration-stack", "htcondor"] + }, + { + "description": "Group Slurm backend bumps (IC-ADR-002 §6, §8)", + "matchManagers": ["dockerfile", "docker-compose", "custom.regex"], + "matchPackageNames": ["/slurm/"], + "groupName": "Slurm backend (integration stack)", + "labels": ["dependencies", "integration-stack", "slurm"] } ] } diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6f86f89 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,29 @@ +"""Session-wide pytest options for interCEde (must live at the tests/ root). + +``--stack``/``--config`` are consumed by the integration suite (tests/integration/conftest.py), but +pytest only honours ``pytest_addoption`` from *initial* conftest files — those in the rootdir and +the invocation's target directories. Registering the options here (the root of ``testpaths``) means +every invocation form accepts them: bare ``pytest``, ``pytest tests`` and +``pytest tests/integration`` all work, instead of only the last. The fixtures that read the options +stay in tests/integration/conftest.py. +""" + +from __future__ import annotations + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + group = parser.getgroup("intercede-integration") + group.addoption( + "--stack", + action="store", + default=None, + help="Stack id from tests/integration/stacks.yml to run against (e.g. arc-slurm)", + ) + group.addoption( + "--config", + action="store", + default="basic", + help="Configuration name under stacks//config/ (default: basic)", + ) diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000..8fc27da --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,20 @@ +# Integration tests + +Backend-agnostic tests that run against real, containerised backend daemons ("stacks"), per +[IC-ADR-002](../../docs/adr/IC-ADR-002_integration_tests.md). + +Quickstart: + +```sh +docker compose -f tests/integration/stacks//compose.yml up --wait +pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic +``` + +Documentation lives in [`docs/dev/`](../../docs/dev/index.md): + +- How-to: [run the tests locally](../../docs/dev/how-to/run-integration-tests-locally.md) · + [add a stack](../../docs/dev/how-to/add-an-integration-stack.md) (incl. the harmonization + checklist) +- Reference: [layout, manifest schema, stack contract](../../docs/dev/reference/integration-stacks.md) · + [image conventions](../../docs/dev/reference/integration-images.md) +- Explanation: [why integration stacks](../../docs/dev/explanations/integration-testing.md) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..deeccf9 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,71 @@ +"""Stack-aware fixtures for the integration suite (IC-ADR-002). + +Tests are written once, against the interCEde contract, and parameterized by a *stack* — a named, +self-contained backend environment described in ``stacks.yml`` and started with +``docker compose -f stacks//compose.yml up --wait``. Select the stack with ``--stack`` and the +configuration with ``--config`` (registered in tests/conftest.py; see +docs/dev/how-to/run-integration-tests-locally.md). + +Stack-bound tests skip themselves when no ``--stack`` is given, so the unit-test run +(``pixi run pytest``) stays green without any compose environment. +""" + +from __future__ import annotations + +import os +import tomllib +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.generate_matrix import STACKS_DIR, compose_stack, load_manifest + + +@pytest.fixture(scope="session") +def stack(request: pytest.FixtureRequest) -> dict[str, Any]: + """Return the stacks.yml entry selected by --stack; skip the test when no stack is given.""" + stack_id = request.config.getoption("--stack") + if stack_id is None: + pytest.skip("no --stack given (this test runs against a compose stack)") + entries = {entry["id"]: entry for entry in load_manifest()} + if stack_id not in entries: + # A bad --stack is a usage error for the whole session, not a per-test failure: raising + # UsageError from a fixture yields per-test tracebacks (repeated per xdist worker) and no + # exit-4; pytest.exit stops the run once, cleanly. + pytest.exit( + f"unknown stack {stack_id!r}; known stacks: {', '.join(sorted(entries))}", + returncode=pytest.ExitCode.USAGE_ERROR, + ) + return entries[stack_id] + + +@pytest.fixture(scope="session") +def stack_dir(stack: dict[str, Any]) -> Path: + """Directory holding the stack's compose.yml, following the `stack:` alias if set.""" + return STACKS_DIR / compose_stack(stack) + + +@pytest.fixture(scope="session") +def client_config( + request: pytest.FixtureRequest, stack: dict[str, Any] +) -> dict[str, Any]: + """Load intercede-client.toml for --stack/--config: endpoint, credential type, queue names. + + The client config is resolved from the entry's *own* id, never the ``stack:`` alias: an aliased + entry (e.g. ``local-slurm`` reusing ``ssh-slurm``'s compose) drives the shared backend over a + different transport/endpoint and so keeps its own client half. Backend config and client config + change together under one configuration name (IC-ADR-002 §1); this fixture is the client half. + """ + config_name = request.config.getoption("--config") + path = STACKS_DIR / stack["id"] / "config" / config_name / "intercede-client.toml" + if not path.is_file(): + msg = f"stack {stack['id']!r} has no {config_name!r} configuration ({path} is missing)" + # In CI a *selected* stack with no config is a real failure, not a skip: an all-skipped run + # would report a green integration cell that verified nothing (IC-ADR-002 §5). Locally a + # skip stays convenient while a stack is still being brought up. + if os.environ.get("CI"): + pytest.fail(msg) + pytest.skip(f"{msg} - skipping (set CI=1 to make this a failure)") + with path.open("rb") as f: + return tomllib.load(f) diff --git a/tests/integration/generate_matrix.py b/tests/integration/generate_matrix.py new file mode 100644 index 0000000..96b423d --- /dev/null +++ b/tests/integration/generate_matrix.py @@ -0,0 +1,117 @@ +"""Generate the integration CI matrix from stacks.yml (IC-ADR-002 §5). + +Run it locally to see exactly which {stack x config x version} jobs CI would run: + + pixi run -e py313 python tests/integration/generate_matrix.py + +The integration workflow (.github/workflows/integration.yml) calls it with --github-output to +append the matrix to $GITHUB_OUTPUT. Only *harmonized* stacks — those whose stacks//compose.yml +has landed — become jobs; stacks that are merely seeded in the manifest are announced and skipped +here (rather than by a per-job step), so an unharmonized stack never turns a matrix job red. + +This module is also the single home of the manifest helpers (``load_manifest``, ``compose_stack``): +conftest.py and test_manifest.py import them from here. It must therefore stay import-light — the +workflow runs it with the runner's *system* python3, so it may not import pytest. See +docs/dev/reference/integration-stacks.md for the manifest schema. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +import yaml + +STACKS_FILE = Path(__file__).parent / "stacks.yml" +STACKS_DIR = Path(__file__).parent / "stacks" + +REQUIRED_KEYS = ("id", "configs", "markers") + + +def load_manifest() -> list[dict[str, Any]]: + """Return the list of stack entries from stacks.yml (the CI-matrix source of truth).""" + with STACKS_FILE.open() as f: + data = yaml.safe_load(f) + if not isinstance(data, dict) or "stacks" not in data: + raise SystemExit(f"{STACKS_FILE}: expected a top-level `stacks:` list") + return data["stacks"] + + +def compose_stack(entry: dict[str, Any]) -> str: + """Return the compose environment a stack runs in: its own id, or the `stack:` alias.""" + return entry.get("stack", entry["id"]) + + +def is_harmonized(entry: dict[str, Any]) -> bool: + """Report whether the stack's compose.yml has landed (vs. merely seeded in the manifest).""" + return (STACKS_DIR / compose_stack(entry) / "compose.yml").is_file() + + +def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Expand manifest entries into the flat {stack x config x version} job list.""" + include = [] + for entry in stacks: + stack_id = entry.get("id") + for key in REQUIRED_KEYS: + if key not in entry: + # Name the offending stack instead of dying with a bare KeyError deep in a loop. + raise SystemExit( + f"stack {stack_id or entry!r}: missing required key {key!r}" + ) + for config in entry["configs"]: + for version in entry.get("versions", ["latest"]): + include.append( + { + "stack": stack_id, + "compose_stack": compose_stack(entry), + "config": config, + "version": version, + "markers": entry["markers"], + "exec_in_container": entry.get("exec_in_container", False), + } + ) + return include + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--github-output", + action="store_true", + help="append matrix= to $GITHUB_OUTPUT instead of pretty-printing", + ) + args = parser.parse_args() + + harmonized: list[dict[str, Any]] = [] + seeded: list[dict[str, Any]] = [] + for entry in load_manifest(): + (harmonized if is_harmonized(entry) else seeded).append(entry) + matrix = build_matrix(harmonized) + + # Seeded-but-not-yet-harmonized stacks are not jobs. Announce them — as a CI annotation under + # --github-output, else on stderr so the JSON on stdout stays clean — so the manifest's intent + # stays visible without a skipped-job placeholder in the check UI. + for entry in seeded: + note = ( + f"stack {entry.get('id')!r} is seeded in stacks.yml but has no compose.yml yet" + " - skipping" + ) + if args.github_output: + print(f"::notice::{note}") + else: + print(note, file=sys.stderr) + + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + out.write(f"matrix={json.dumps(matrix)}\n") + else: + json.dump(matrix, sys.stdout, indent=2) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/tests/integration/stacks.yml b/tests/integration/stacks.yml new file mode 100644 index 0000000..488e0d4 --- /dev/null +++ b/tests/integration/stacks.yml @@ -0,0 +1,31 @@ +# THE manifest: single source of truth for the integration CI matrix (IC-ADR-002 §1, §5). +# +# The field reference and the stack contract live in docs/dev/reference/integration-stacks.md — +# this file is data, not documentation. Adding a stack, a configuration, or a supported version is +# one entry here plus files under stacks//; no workflow edits. +# +# The entries below are seeded for the three in-flight stack PRs (issues #5, #12, #13); a stack +# becomes a CI job only once its stacks//compose.yml lands ("harmonized"). +stacks: + - id: arc-slurm + configs: [basic] + versions: [latest] # ARC 7.x leading edge; floor is 7.x (IC-ADR-002 §8) + markers: "remote and arc" + + - id: htcondor + configs: [basic] + versions: [lts, latest] # HTCondor LTS anchor + leading edge (IC-ADR-002 §8) + markers: "remote and htcondor" + + - id: ssh-slurm + configs: [basic] + markers: "scheduler and slurm" + + - id: local-slurm + stack: ssh-slurm # reuse ssh-slurm's compose environment... + configs: [basic] + # ...but drive Slurm over the Local transport, so exclude the SSH-transport tests. Every + # future local-* alias needs the same `and not ssh` guard: the `ssh` marker is otherwise + # byte-identical here and in ssh-slurm, so an ssh test would run in both jobs. + markers: "scheduler and slurm and not ssh" + exec_in_container: true diff --git a/tests/integration/stacks/_images/README.md b/tests/integration/stacks/_images/README.md new file mode 100644 index 0000000..ba8694c --- /dev/null +++ b/tests/integration/stacks/_images/README.md @@ -0,0 +1,7 @@ +# Shared stack images + +Dockerfiles shared across stacks (`arc/`, `slurm/`, `htcondor-ce/`, …). + +Conventions (EL9 base, pinned Renovate-annotated `ARG`s, foreground daemons, ephemeral +credentials, GHCR publishing) are documented in +[docs/dev/reference/integration-images.md](../../../../docs/dev/reference/integration-images.md). diff --git a/tests/integration/stacks/_template/compose.yml b/tests/integration/stacks/_template/compose.yml new file mode 100644 index 0000000..cde717a --- /dev/null +++ b/tests/integration/stacks/_template/compose.yml @@ -0,0 +1,62 @@ +# Template stack — copy to stacks//compose.yml and adapt. +# Stack contract: docs/dev/reference/integration-stacks.md +# How-to: docs/dev/how-to/add-an-integration-stack.md +# +# Contract highlights: +# - `docker compose -f compose.yml up --wait` must yield a ready backend: healthchecks are +# mandatory on every service (`up --wait` gates on them — no sleep-and-retry loops). +# - Daemons run in the foreground under a plain entrypoint or minimal supervisor — never +# systemd (IC-ADR-002 §3) — and log to stdout/stderr so `docker compose logs` captures them +# (a file-logging daemon redirects its log to /dev/stdout; the CI failure artifact is compose +# logs only). +# - Credentials are minted ephemerally at container start into the shared, host-readable +# `./credentials` bind mount (gitignored) — a Docker *named* volume is root-owned under +# /var/lib/docker and unreadable by the host pytest that drives `exec_in_container: false` +# stacks. The layout `credentials//…` is the interface between stack and harness +# (IC-ADR-002 §4). Mint throwaway Test-CA material world-readable; the harness copies anything +# needing private modes (0400/0600) into a private dir itself. +# - Images come from stacks/_images/ Dockerfiles with pinned, Renovate-annotated versions; +# CI pulls prebuilt images from GHCR once issue #14 lands (build: is the local fallback). +# +# `backend` is the service the harness drives. A multi-service stack (a CE + its LRMS, or a Slurm +# controller + worker) adds more services here and names one of them `backend`; parameterize a +# per-role image rather than duplicating it (integration-images.md). Intra-stack secrets (e.g. a +# munge key shared between Slurm services) follow the same rule as credentials — minted at start +# into a stack-internal volume, never baked into an image (a published image with a baked key is a +# leaked credential). +services: + backend: + # image: ghcr.io/diracgrid/intercede-testenv/:${BACKEND_VERSION:-latest} + build: + context: ../_images/CHANGEME + args: + BACKEND_VERSION: ${BACKEND_VERSION:-latest} + hostname: backend + networks: + - stack + # For a host-run harness (exec_in_container: false) publish the backend's port on localhost and + # record the same host:port in config//intercede-client.toml. A TLS backend's Test-CA host + # certificate must carry a SAN for the name the client actually dials — mint it for both the + # compose hostname and localhost (ARC on EL9 additionally needs crypto-policies set to LEGACY + # for the proxy chain). + # ports: + # - "127.0.0.1:2222:22" # e.g. sshd + volumes: + # host-readable bind mount, NOT a named volume (see the credentials note above); gitignored. + - ./credentials:/shared/credentials + # - ./config/basic/backend.conf:/etc/…:ro # backend half of the configuration (§1) + healthcheck: + # Real readiness probe, e.g.: + # Slurm: ["CMD-SHELL", "sinfo | grep -q idle"] + # HTCondor: ["CMD", "condor_status", "-limit", "1"] + # HTCondor-CE: ["CMD", "condor_ce_status"] + # ARC REST: ["CMD-SHELL", "curl -ksf https://localhost:443/arex/rest/1.0 -o /dev/null"] + # sshd: ["CMD-SHELL", "true < /dev/tcp/localhost/22"] + test: ["CMD", "false"] # CHANGEME — a stack without a real healthcheck must not pass review + interval: 5s + timeout: 10s + retries: 24 + start_period: 30s + +networks: + stack: {} diff --git a/tests/integration/stacks/_template/config/basic/intercede-client.toml b/tests/integration/stacks/_template/config/basic/intercede-client.toml new file mode 100644 index 0000000..174fad8 --- /dev/null +++ b/tests/integration/stacks/_template/config/basic/intercede-client.toml @@ -0,0 +1,20 @@ +# Client-side half of the `basic` configuration for this stack, consumed by the test harness +# (the `client_config` fixture in tests/integration/conftest.py). +# +# Backend configuration (arc.conf, condor config, slurm.conf, mounted by compose.yml) and this +# file change together, atomically, under one configuration name (IC-ADR-002 §1). A new +# configuration (e.g. token vs proxy auth) is a new config// directory plus a stacks.yml +# `configs:` entry — never a mutation of `basic`. + +[stack] +id = "CHANGEME" # must match THIS entry's own id in stacks.yml — an aliased entry (reusing another + # stack's compose) still keeps its own client config under its own id + +[backend] +kind = "CHANGEME" # arc | htcondor-ce | htcondor | ssh | local — which client path the tests drive +endpoint = "CHANGEME" # host:port the harness dials, e.g. "https://localhost:443/arex" or "localhost:2222" +queue = "CHANGEME" # target queue/partition name inside the container + +[credentials] +type = "CHANGEME" # token | proxy | ssh-key +path = "credentials/CHANGEME/" # under the stack's host-readable ./credentials bind mount (IC-ADR-002 §4) diff --git a/tests/integration/test_manifest.py b/tests/integration/test_manifest.py new file mode 100644 index 0000000..ec74df9 --- /dev/null +++ b/tests/integration/test_manifest.py @@ -0,0 +1,84 @@ +"""Validate stacks.yml — the single source of truth for the CI matrix (IC-ADR-002 §1). + +These tests need no running stack: they keep the manifest schema honest and enforce the stack +contract layout (docs/dev/reference/integration-stacks.md) for every stack that has landed. They +run in the plain unit-test job. The manifest helpers come from generate_matrix (kept pytest-free so +the workflow can run it with the system python3). +""" + +from __future__ import annotations + +from typing import Any + +from tests.integration.generate_matrix import ( + STACKS_DIR, + build_matrix, + compose_stack, + load_manifest, +) + + +def test_entries_have_required_keys(): + for entry in load_manifest(): + assert "id" in entry, f"manifest entry without an id: {entry}" + assert entry.get("configs"), ( + f"stack {entry.get('id')}: `configs` must be a non-empty list" + ) + assert entry.get("markers"), ( + f"stack {entry.get('id')}: `markers` expression is required" + ) + + +def test_stack_ids_are_unique(): + ids = [entry["id"] for entry in load_manifest()] + assert len(ids) == len(set(ids)), f"duplicate stack ids in stacks.yml: {ids}" + + +def test_stack_aliases_resolve(): + ids = {entry["id"] for entry in load_manifest()} + for entry in load_manifest(): + alias = entry.get("stack") + if alias is not None: + assert alias in ids, ( + f"stack {entry['id']}: alias {alias!r} is not a known stack id" + ) + + +def test_matrix_expansion(): + """generate_matrix expands the versions axis and resolves stack aliases.""" + by_stack: dict[str, list[dict[str, Any]]] = {} + for job in build_matrix(load_manifest()): + by_stack.setdefault(job["stack"], []).append(job) + assert { + "stack", + "compose_stack", + "config", + "version", + "markers", + "exec_in_container", + } <= set(job) + assert {job["version"] for job in by_stack["htcondor"]} == {"lts", "latest"} + assert all(job["compose_stack"] == "ssh-slurm" for job in by_stack["local-slurm"]) + + +def test_harmonized_stacks_follow_the_layout(): + """A stack whose directory has landed must follow the stack contract. + + ``compose.yml`` lives with the *compose* stack (the ``stack:`` alias target when set); the + client config lives with the entry's *own* id, so an aliased entry (``local-slurm``) still ships + its own ``intercede-client.toml`` even though it borrows another stack's compose environment. + The contract is documented in docs/dev/reference/integration-stacks.md. + """ + for entry in load_manifest(): + own_dir = STACKS_DIR / entry["id"] + if not own_dir.is_dir(): + continue # seeded in the manifest, not harmonized yet — files land with the stack PR + compose_dir = STACKS_DIR / compose_stack(entry) + assert (compose_dir / "compose.yml").is_file(), ( + f"{compose_dir} is missing compose.yml" + ) + for config in entry["configs"]: + client_toml = own_dir / "config" / config / "intercede-client.toml" + assert client_toml.is_file(), ( + f"{own_dir} is missing config/{config}/intercede-client.toml" + ) From 01cd1d4d4f36c56c2d8b4ab9554aa016d5ba2941 Mon Sep 17 00:00:00 2001 From: aldbr Date: Fri, 23 Oct 2026 10:37:38 +0200 Subject: [PATCH 3/7] docs: drop the ADRs from the integration scaffold ADRs will come once a first implementation exists. Remove docs/adr and the reuse-survey note, and rewrite the IC-ADR-002 links and section references in the dev docs, workflow, manifest, template and renovate config so nothing points at the removed files. Assisted-by: Claude:claude-opus-5-5 --- .github/workflows/integration.yml | 6 +- .gitignore | 2 +- docs/adr/IC-ADR-001_computing_elements.md | 964 ------------------ docs/adr/IC-ADR-002_integration_tests.md | 398 -------- docs/adr/IC-ADR-003_credentials.md | 301 ------ docs/adr/IC-ADR-XXX_template.md | 45 - docs/adr/index.md | 13 - docs/dev/explanations/integration-testing.md | 3 +- .../how-to/run-integration-tests-locally.md | 4 +- docs/dev/index.md | 4 +- docs/dev/reference/integration-images.md | 10 +- docs/dev/reference/integration-stacks.md | 21 +- docs/dev/reference/renovate.md | 8 +- docs/notes/reuse-survey.md | 68 -- pyproject.toml | 2 +- renovate.json | 8 +- tests/integration/README.md | 3 +- tests/integration/conftest.py | 6 +- tests/integration/generate_matrix.py | 2 +- tests/integration/stacks.yml | 6 +- .../integration/stacks/_template/compose.yml | 10 +- .../config/basic/intercede-client.toml | 4 +- tests/integration/test_manifest.py | 2 +- 23 files changed, 48 insertions(+), 1842 deletions(-) delete mode 100644 docs/adr/IC-ADR-001_computing_elements.md delete mode 100644 docs/adr/IC-ADR-002_integration_tests.md delete mode 100644 docs/adr/IC-ADR-003_credentials.md delete mode 100644 docs/adr/IC-ADR-XXX_template.md delete mode 100644 docs/adr/index.md delete mode 100644 docs/notes/reuse-survey.md diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index cd8313a..4e37124 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,4 +1,4 @@ -# Skeleton of the manifest-driven integration workflow (IC-ADR-002 §5; scaffold issue #9). +# Skeleton of the manifest-driven integration workflow (scaffold issue #9). # # The matrix is generated from tests/integration/stacks.yml — adding a stack, configuration or # version is a manifest entry plus files, never a workflow edit. generate_matrix.py emits jobs only @@ -66,7 +66,7 @@ jobs: - name: Run the contract suite run: | if [ "${{ matrix.exec_in_container }}" = "true" ]; then - # local-* stacks: run pytest inside the batch container (IC-ADR-002 §1). The batch + # local-* stacks: run pytest inside the batch container. The batch # scheduler typically refuses to run as root (HTCondor, Slurm), so the image must ship # an unprivileged test user and the suite + pytest must be mounted or baked in. # TODO(#13/#14): mount/install the suite and pin the exec user (compose `user:` or @@ -90,7 +90,7 @@ jobs: - name: Collect daemon logs if: failure() - # Daemons run in the foreground and log to stdout/stderr (IC-ADR-002 §3), so compose logs + # Daemons run in the foreground and log to stdout/stderr, so compose logs # are the whole failure story — a file-logging daemon redirects its log to /dev/stdout. run: docker compose -f "$COMPOSE_FILE" logs --no-color > compose-logs.txt || true diff --git a/.gitignore b/.gitignore index 89c486e..a80dc8a 100644 --- a/.gitignore +++ b/.gitignore @@ -76,7 +76,7 @@ htmlcov/ #remove in case we want a specific LHCb one .pylintrc -# Integration-stack credentials: minted per run into a bind mount, never committed (IC-ADR-002 §4) +# Integration-stack credentials: minted per run into a bind mount, never committed tests/integration/stacks/*/credentials/ # docs (auto-generated build output only; docs/assets is tracked) diff --git a/docs/adr/IC-ADR-001_computing_elements.md b/docs/adr/IC-ADR-001_computing_elements.md deleted file mode 100644 index b7535cd..0000000 --- a/docs/adr/IC-ADR-001_computing_elements.md +++ /dev/null @@ -1,964 +0,0 @@ -# IC-ADR-001: Computing Element interfaces and the DIRAC to interCEde migration - -## Metadata - -- **Created By:** Alexandre Boyer -- **Date:** 2026-06-30 -- **Status:** Draft -- **Decision Maker(s):** Federico Stagni, Christophe Haen, Chris Burr -- **Stakeholders:** DIRAC/DiracX SiteDirector, PushJobAgent and Pilot maintainers; extension - communities operating Computing Elements and batch systems; interCEde contributors - -> **Scope and altitude.** This is a *direction-setting* ADR. It commits to the **shape** of -> interCEde's Computing Element interfaces and records what is ported from DIRAC's -> `Resources/Computing` and what is deliberately left out. Illustrative code sketches pin down the -> shape; the exhaustive method-by-method signatures, the wire/serialisation details, and the -> per-backend porting mechanics are deferred to follow-up ADRs and to the implementation itself. -> interCEde is a standalone library in the [DIRACGrid](https://github.com/DIRACGrid) ecosystem -> designed to back the Computing Element layer used by -> [DiracX](https://github.com/DIRACGrid/diracx); this ADR is intended to be read alongside the -> DiracX transition ADRs but governs interCEde's own API. (Why a standalone library rather than a -> `diracx` subpackage is recorded in Rejected Ideas.) -> -> **How to read this.** To review the decision, read the Abstract, Motivation, §1–§4 and §7. -> The rest is supporting reference and can be skimmed: §8 (migration map), §9 (consumer sketches), -> the Rejected-Ideas entries you don't want to contest, and the Open Issues backlog. - -## Abstract - -interCEde gives the workload management layer one payload-agnostic way to **submit, monitor and -retrieve** units of work on heterogeneous Computing Elements (CEs) and batch systems, validated -against containerised backends. This ADR fixes the shape of that interface with five decisions: - -1. **The contract is a set of small typed interfaces (`typing.Protocol`s), not a base class.** - Backends are defined by structural conformance, implement only the capabilities they have, and - can be mocked without inheriting interCEde types. -2. **Access and scheduling compose instead of multiplying.** A `Transport` (how you reach the - resource) and a `Scheduler` (how work is queued) combine in one generic `BatchBackend`, so N - transports × M schedulers costs N+M classes; genuinely monolithic backends (ARC, HTCondor-CE, - Cloud) implement the contract directly. -3. **DIRAC's in-process CEs (`InProcess`, `Singularity`, `Pool`) leave the library.** They belong - to the pilot/worker-node domain and get their own `Runner`/`ExecutorPool` subsystem, - disconnected from this contract. -4. **The data plane handles arbitrary input/output sandboxes**, not only stdout/stderr — the - direct consequence of never interpreting the payload. -5. **The extension surface is explicit and three-tiered:** stable contracts third parties - implement, provided implementations they may use but not subclass, and internals with no - stability guarantee. - -## Motivation - -### What DIRAC has today - -DIRAC's CE machinery lives in `DIRAC/Resources/Computing/`. A base class `ComputingElement` is -subclassed by nine concrete CEs (`AREX`, `HTCondorCE`, `SSH`, `SSHBatch`, `Local`, `Cloud`, -`InProcess`, `Singularity`, `Pool`), instantiated through a `ComputingElementFactory` that -string-loads modules via `ObjectLoader`. A parallel `BatchSystems/` package (`SLURM`, `Condor`, -`LSF`, `GE`, `OAR`, `Torque`, `Host`) provides scheduler plugins that the `SSH`/`Local` CEs -compose via `loadBatchSystem()`. - -### Why it needs to change - -1. **The base class is a fat class that conflates contract with policy.** `ComputingElement` - mixes the job-lifecycle contract (submit/status/output/kill) with cross-cutting policy: - CPU-slot accounting (`available()`), proxy renewal (`_monitorProxy`), and a layered - configuration hierarchy. Every CE drags this along whether it needs it or not. - -2. **The contract is "abstract by convention", and unenforced.** There is no `abc.ABC`, no - `@abstractmethod`, no Protocol — it is a plain `class ComputingElement:`. What stubs exist - (`submitJob`, `getCEStatus`) merely `return S_ERROR("…should be implemented in a subclass")`; - the rest of the lifecycle (`getJobStatus`, `getJobOutput`, `killJob`) is not declared on the - base **at all**, so nothing — not even a stub — stops a subclass from silently omitting - `getJobOutput`. - -3. **The interface has drifted across implementations.** `submitJob` alone appears as - `(executableFile, proxy, numberOfJobs, inputs, outputs)` (AREX), `(…, numberOfJobs=1)` - (HTCondorCE/SSH), `(…, proxy=None, inputs=None, **kwargs)` (InProcess/Pool), and - `(…, proxy=None, **kwargs)` (Singularity); `proxy` is required in some and optional in others. - `getJobOutput` appears with four different signatures. Not all CEs implement the same methods. - -4. **Three incompatible semantics are crammed under one base.** The base docstring itself - distinguishes **Remote** (async, pollable, multi-job), **Inner** (synchronous, blocking, one - job), and **Inner Pool** CEs. They serve *different actors* — Remote CEs are driven by the - central SiteDirector/PushJobAgent, Inner CEs by the JobAgent inside a pilot — and are never - substituted for one another. Their *mechanics* don't even overlap (`CGroups2.systemCall` + - periodic proxy refresh on the inner side vs batch submit/poll/REST on the remote side). The - shared supertype is a false abstraction. - -5. **The good idea — composition — is applied inconsistently.** `SSH`/`Local` compose a transport - with a `BatchSystems/*` plugin, but `AREX`/`HTCondorCE`/`Cloud` hard-code their scheduler - interaction, and the `BatchSystems/*` plugins are a separate hierarchy with their own ad-hoc, - string-generating interface. - -6. **The data plane is too narrow.** Only `AREXComputingElement` retrieves arbitrary output files - (it lists the ARC session directory and streams every file to disk — though even it still - *returns* only stdout/stderr in memory, `S_OK((output, error))`); every other CE handles only - stdout/stderr, returning their contents in memory (SSH/Local can instead write them to a - directory and return paths, but nothing generalises to an arbitrary sandbox). Input staging is - similarly limited. A payload-agnostic library cannot assume payloads emit only stdout/stderr. - -7. **Untyped results and discovery.** Everything is `S_OK`/`S_ERROR` dicts; backends are - discovered by string-to-module loading. Neither is checkable or pluggable in a typed, - third-party-friendly way. - -### Drivers - -- **Payload-agnostic.** interCEde must not care whether the payload is a DIRAC job, a DIRAC - pilot, or anything else. It submits an opaque executable + sandbox and lets the caller - poll/fetch/kill. -- **Orchestration-agnostic.** Whether work is *pulled* (SiteDirector) or *pushed* (PushJobAgent) - is a workload management system concern above interCEde. -- **One contract, many backends**, with *combinations* (`SSH + Slurm`, `SSH + HTCondor`, - `Local + HTCondor`) as first-class. -- **Tested against the real thing** — backends verified against containerised schedulers, so the - contract must be trivially implementable and mockable. -- **Extensible by third parties** — VOs must be able to add their own transports, schedulers and - CEs without forking interCEde. -- **Typed and async** — DiracX is async; results and errors should be typed. - -## Specification - -### 1. Scope — what interCEde is and is not - -interCEde is the **delegate-and-poll** side of running work on a resource: *submit a payload to a -scheduler/resource, get a handle, then poll it, fetch its sandbox, or kill it.* The boundary with -the rest of the system, stated without any pilot/job vocabulary: - -- **In scope:** reaching a resource (local shell, SSH, REST/ARC, HTCondor-CE, cloud API); - queueing work on a scheduler; tracking a remote handle; staging input/output sandboxes. -- **Out of scope — execute-here-and-now:** running a payload *in this process* on a worker node - (DIRAC's `InProcess`/`Singularity`/`Pool`). This is the pilot/worker-node domain (see §6). -- **Out of scope — orchestration:** pull vs push, matching, pilot lifecycle. interCEde exposes - mechanism; DiracX/DIRAC decides policy. -- **Out of scope — payload interpretation:** interCEde never inspects payload contents. -- **Out of scope — persistence:** interCEde keeps no store of jobs or handles; the caller - persists handles and owns idempotency/reconciliation (see §9). -- **Out of scope — other resource types:** Storage Elements, File Catalogs, and the like are - *different contracts* (data movement; namespace/metadata) with different backends, testing, and - consumers. They are **sibling libraries, not part of interCEde** — see Rejected Ideas. - -The submitted thing is a **`SubmissionSpec`** — the description of an opaque payload (name -provisional; see Open Issues). Submitting a spec, optionally `count` identical times, returns one -**`Submission`**, which bundles one **`JobHandle`** per created job plus any per-copy failures. - -**Throughout this ADR, "job" means the *scheduler's* job (a Slurm/Condor job) — explicitly *not* -a DIRAC Job.** The naming deliberately reserves `Job*` for the backend handle and avoids -`Payload`, which collides with the pilot-side payload concept (see Open Issues). - -A `JobHandle` is the durable, serialisable identity of one job: the caller persists it and later -replays it into status/output/kill calls. It must therefore be stable across processes and carry -whatever *routing* the backend needs to find the job again (e.g. the target host that DIRAC's -`sshcondor://…` scheme used to encode — see §8). - -### 2. The contract — capability-segmented Protocols - -The caller-facing contract is a set of small `typing.Protocol`s, and **every operation works on -many jobs at once**: submit fans a single spec into `count` identical jobs; status/output/kill/ -purge take a sequence of ids and return a mapping keyed by `JobID`, so every per-item outcome is -addressable (see the partial-failure rule below). - -The contract is sized by its consumers. DiracX splits the old monolithic SiteDirector into -independent, separately-scheduled **tasks** — one submits, one polls status, one fetches outputs -(§9 sketches them). The **essential** capabilities of a CE are two — *submit a payload (with -inputs)* and *get the jobs' status* — each driven by its own task. *Retrieving outputs* is a -third task and its own protocol, but it is **optional, not essential**: some backends genuinely -cannot serve a pull-style sandbox retrieval — the canonical case is **Cloud**, where a booted VM -has no server-side filesystem interCEde controls, so exporting outputs is the payload's own -responsibility (e.g. pushing them to an object store) — and the ADR's rule is *never force a -backend to stub a capability it lacks*. So `OutputRetriever` sits with the optional capabilities, -and the output task narrows to it structurally. - -```python -StatusMap = Mapping[JobID, JobStatus] # named alias, stable (Tier A) - -@runtime_checkable -class Submitter(Protocol): # the submission task - async def submit(self, spec: SubmissionSpec, count: int = 1) -> Submission: ... - # submit the SAME spec `count` times (DIRAC's numberOfJobs); returns one Submission - # bundling one JobHandle per successful copy, plus per-copy failures - -@runtime_checkable -class Monitor(Protocol): # the status task - async def status(self, ids: Sequence[JobID]) -> StatusMap: ... - # ids the backend no longer knows come back as JobStatus.UNKNOWN — never dropped - -@runtime_checkable -class JobBackend(Submitter, Monitor, Protocol): ... -``` - -`OutputRetriever` and the other capabilities are additive and optional (below). `JobBackend` is -the composition of the two essential protocols; it names "a complete backend" and is what the -registry returns and validates. But **consumers depend on the narrow protocol they use, not on -`JobBackend`** — the submission task takes a `Submitter`, the status task a `Monitor`, the output -task an `OutputRetriever` (which it must confirm structurally, since a backend may lack it). A -backend implements the slices it can, and each task is handed the same object but sees only its -slice. - -> **Naming — why `JobBackend`, not `ComputingElement`.** interCEde types name *what interCEde -> provides* — a backend the WMS drives — never the external resource. The composed contract is -> `JobBackend` (it pairs with `JobHandle`/`JobID`/`JobStatus`); a Slurm-over-SSH `BatchBackend` -> is emphatically *not* a Computing Element in the WLCG sense, and the concrete drivers -> (`ARCBackend`, `HTCondorCEBackend`, `CloudBackend`) are clients *of* resources, not the -> resources themselves. The domain term "Computing Element / CE" stays in prose and docstrings. -> The one retained "CE" is `HTCondorCEBackend`: there "CE" names the external product -> (HTCondor-CE, a grid gateway distinct from the local `HTCondorScheduler`), not a claim that the -> class *is* a CE. - -> **The registry gate is a smoke-test, not the drift guard.** `isinstance` against a -> `@runtime_checkable` Protocol checks method **presence only, never signatures** — a backend -> whose `submit` has an extra required argument, or that forgot `async`, passes the gate and -> fails only at the first call. The real defence against DIRAC-style signature drift is (a) -> static typing for first-party code and (b) the **container conformance suite**, which is -> therefore a *requirement* of registration, not optional (see §5, Evolution). (Two PEP 544 -> mechanics for implementers: keep `Protocol` in the base list when composing protocols, or the -> subclass silently becomes a concrete class; and `@runtime_checkable` is not inherited — the -> composed `JobBackend` must carry the decorator itself.) - -**Everything else is optional** — additive, independent protocols, *not* part of `JobBackend`, -each justified by a real caller and by backends that have it and don't. All are bulk and return a -per-`JobID` outcome map (`kill`/`purge`/`fetch_output` never return bare `None`, so a -partial-batch failure is always reportable): - -```python -@runtime_checkable -class OutputRetriever(Protocol): # the (on-demand) output task — OPTIONAL - # Bulk: materialise each job's whole output sandbox (incl. the CE/scheduler log) into - # `dest`. No separate log fetch — the log is a manifest member. Retrieval is IDEMPOTENT - # by default (ARC/SSH/Local keep remote state and can be re-fetched); a backend whose - # retrieval is physically one-shot DECLARES it via `destructive` — e.g. HTCondor - # (>= 25.8), where a completed spooled job leaves the queue after `condor_transfer_data` - # (overridable via `leave_in_queue`). interCEde never *forces* destructiveness on a - # re-fetchable backend; the consumer reads `destructive` to know whether `dest` is a - # hard commit point or a re-fetchable cache. - destructive: bool # False for ARC/SSH/Local; True for HTCondor (>= 25.8) - async def fetch_output(self, ids: Sequence[JobID], dest: Path) -> Mapping[JobID, JobOutput]: ... - -@runtime_checkable -class Cancellable(Protocol): - async def kill(self, ids: Sequence[JobID]) -> Mapping[JobID, OpOutcome]: ... - -@runtime_checkable -class Purgeable(Protocol): - # Delete a job's outputs/remote state WITHOUT fetching them — the counterpart to the - # destructive fetch. Needed on its own where a site mandates scratch/spool cleanup - # even for outputs no one will retrieve. - async def purge(self, ids: Sequence[JobID]) -> Mapping[JobID, OpOutcome]: ... - -@runtime_checkable -class LoadReporter(Protocol): - # Aggregate counts of *our* submitted jobs — NOT a capacity number, NOT "available - # slots" (that is the caller's throttling policy). HTCondor-CE cannot provide this at - # all (`getCEStatus()` returns an explicit `S_ERROR("… not supported")`). - async def counts(self) -> JobCounts: ... # {running, waiting} - -@runtime_checkable -class SupportsLiveDiagnostics(Protocol): - # An independent, repeatable diagnostics fetch *before* completion — ARC is the - # clear case (its `diagnose/errors` endpoint); whether HTCondorCE's spool log-peek - # qualifies is an Open Issue. The *final* log is always a member of the - # fetch_output manifest, so there is no standalone "get the final log" method. - async def diagnostics(self, ids: Sequence[JobID]) -> Mapping[JobID, Diagnostics]: ... -``` - -The comment on `OutputRetriever.destructive` above is the **single normative statement** of the -destructive-retrieval model; every other mention in this ADR cross-references it. - -**Partial failure is per item, not per batch.** A bulk call raises a typed exception only for a -whole-operation failure (transport down, auth rejected); anything that can succeed for some ids -and fail for others reports it in the returned map — `status` yields `JobStatus.UNKNOWN` for -unknown/expired ids, and `kill`/`purge`/`fetch_output` yield a per-id `OpOutcome` (ok / error). -Callers never have to guess which ids a partial batch touched. - -**Backends have an explicit lifecycle, because consumers cache them.** DIRAC already proves both -halves: `QueueCECache` caches CE objects keyed on a hash of their parameters, reuses them across -agent cycles, and calls `shutdown()` on eviction (with an explicit "eviction must never fail" -guard), and the SSH CE's `shutdown()` closes per-host connections *and the ssh jump-gateway* — -yet the base class declares no lifecycle contract at all, so eviction relies on `shutdown()` -being "polymorphic by convention". interCEde backends and transports hold the same kind of state -(SSH connections, HTTP sessions), so the lifecycle is part of the Tier-A contract: **backends and -transports are async context managers** — construction is cheap and side-effect-free (no I/O -before first use), close is idempotent, and a consumer that caches backends owns closing them on -eviction. This is also what makes the statelessness rule (§9) honest: the *only* resources a -backend holds are connections, released deterministically, never job state. - -**Decision rule for segmentation:** one protocol per capability a *specific caller wants in -isolation*. The DiracX-task split makes that concrete — submit and status are two separate tasks -(so two essential protocols), and output-retrieval is a third task narrowed to an *optional* -`OutputRetriever` (an earlier draft bundled status+fetch into one "operator"; the task topology -shows that was too coarse). `kill` and `purge` are management actions some flows never use; the -submission task's throttling wants `LoadReporter`; the WMS matcher wants none of this. Do not -segment further than a caller exists for — that discipline is what prevents drift back into a -god-interface. - -```mermaid -classDiagram - direction LR - class Submitter { - <> - +submit(spec, count) Submission - } - class Monitor { - <> - +status(ids) StatusMap - } - class OutputRetriever { - <> - +fetch_output(ids, dest) OutputMap - } - class JobBackend { - <> - } - Submitter <|-- JobBackend - Monitor <|-- JobBackend - class Cancellable { - <> - +kill(ids) OutcomeMap - } - class Purgeable { - <> - +purge(ids) OutcomeMap - } - class LoadReporter { - <> - +counts() JobCounts - } - class SupportsLiveDiagnostics { - <> - +diagnostics(ids) DiagnosticsMap - } - class Transport { - <> - +run(argv) CommandResult - +put(local, remote) - +get(remote, local) - } - class Scheduler { - <> - +stages_own_files bool - +submit_cmd(spec) Command - +parse_status(raw) StatusMap - +kill_cmd(ids) Command - } - class BatchBackend - JobBackend <|.. BatchBackend - OutputRetriever <|.. BatchBackend - Cancellable <|.. BatchBackend - Purgeable <|.. BatchBackend - LoadReporter <|.. BatchBackend - BatchBackend o-- Transport : uses - BatchBackend o-- Scheduler : uses - Transport <|.. SSHTransport - Transport <|.. LocalTransport - Scheduler <|.. Slurm - Scheduler <|.. HTCondorScheduler - class ARCBackend - class HTCondorCEBackend - class CloudBackend - JobBackend <|.. ARCBackend - JobBackend <|.. HTCondorCEBackend - JobBackend <|.. CloudBackend - OutputRetriever <|.. ARCBackend - OutputRetriever <|.. HTCondorCEBackend - Cancellable <|.. ARCBackend - Purgeable <|.. ARCBackend - LoadReporter <|.. ARCBackend - SupportsLiveDiagnostics <|.. ARCBackend - Cancellable <|.. HTCondorCEBackend -``` - -(The two essential protocols compose into `JobBackend`; output-retrieval and the rest are -optional and picked up à la carte. `ARCBackend` adds `OutputRetriever` plus all four optional -capabilities. `HTCondorCEBackend` adds `OutputRetriever` and `Cancellable` only — it reports no -counts, offers no independent diagnostics, and its retrieval is destructive (§2). -`CloudBackend` implements only the essential `Submitter`+`Monitor` — it deliberately does *not* -claim `OutputRetriever` (§2).) - -### 3. Composition — `Transport` × `Scheduler` - -Most backends are really two orthogonal choices: **access/transport** (how you reach the -resource) and **scheduler** (what queues the work). `Transport` and `Scheduler` are -*collaborator* protocols — they are **not** sub-protocols of `JobBackend`. Their relationship to -the contract is realised by composition: - -```python -class BatchBackend: # satisfies JobBackend - def __init__(self, transport: Transport, scheduler: Scheduler): ... -``` - -This one class covers `SSH + Slurm`, `SSH + HTCondor`, `Local + Slurm`, … — the matrix collapses -from *transports × schedulers* to *transports + schedulers*. Backends that are not -"transport + scheduler" (ARC is a REST service fronting an opaque batch system; Cloud boots a VM -and has no scheduler) implement `JobBackend` directly. The Protocol is what lets composed and -monolithic implementations sit behind one caller-facing type. - -Cloud is the clearest case of *why* composition does not always apply. Batch has **two -independent axes** — any `Scheduler` behind any `Transport` — which is what makes the N×M → N+M -collapse worthwhile. Cloud has **one** axis, the provider (OpenStack/EC2/OpenNebula/…), and "how -you reach it" and "what it is" are *fused* into a single Apache Libcloud driver — there is no -matrix to collapse. So `CloudBackend` is **not** `BatchBackend(libcloud, opennebula)`: Libcloud -is not a `Transport` (no shell to `run` argv, no filesystem to `put`/`get`) and a provider like -OpenNebula is not a `Scheduler` (no queue, no submit/parse-status commands). The provider axis -*is* a composition, but Libcloud already owns it (its `get_driver`/`set_driver` registry); -`CloudBackend` is a monolithic `JobBackend` that composes Libcloud **internally** (Tier C, §8), -which is why the provider drivers do not appear as collaborators in the class diagram. - -**Staging is a Transport × Scheduler interaction, and the `Scheduler` carries it.** Who moves the -sandbox is not purely a transport choice: HTCondor declares its file transfer *in the submit -description at submit time* (`should_transfer_files`, `transfer_output_files`) and stages -worker→schedd itself, whereas a Slurm-over-SSH combination declares nothing to the scheduler and -interCEde stages the sandbox over the transport (`put`/`get`) at fetch time. `BatchBackend` must -not hard-code this per scheduler, so the `Scheduler` protocol exposes it as data — a -`stages_own_files` flag (and, where needed, `stage_inputs`/`collect_outputs` hooks). -`BatchBackend.submit()` consults the flag instead of branching on scheduler identity, which is -what keeps "add a scheduler by implementing the protocol, no core change" true even for a -scheduler whose transfer model differs from the reference one. - -**Shared batch-system knowledge.** A backend family's command construction and output parsing — -building a `condor_submit` description, parsing `condor_q`/`condor_history` ClassAds, mapping -native states to `JobStatus` — lives in *one* internal module (`_htcondor`, `_slurm`, …) consumed -by **both** the `Scheduler` adapter and any related monolithic CE. `HTCondorScheduler` (used by -`BatchBackend` for SSH/Local + HTCondor) and `HTCondorCEBackend` (a remote schedd reached over a -grid/CE transport, with tokens and the grid universe) share that core and differ only in -transport/auth and submit-description routing. That overlap is itself a hint that -`HTCondorCEBackend` might eventually be expressed as -`BatchBackend(HTCondorCETransport, HTCondorScheduler)` rather than a monolith (see Open Issues). -The rule is the same one from §6: **share implementation as functions/internal modules, never by -making one a subtype of the other.** - -**No interpreter on the remote host.** DIRAC's SSH/Local CEs work by *shipping* a Python driver -to the resource (`BatchSystems/.py` concatenated with `executeBatch.py`, uploaded and executed -by whatever Python the host provides — the file itself warns "support for py2 and py3 is -necessary"), which is why those drivers are stdlib-only, DIRAC-free, and stuck on 2014-era -conventions. interCEde deliberately drops the ship-and-exec model: `Scheduler`s build commands -*locally* and only argv (plus staged sandbox files) crosses the `Transport`. The remote host -needs the scheduler CLI and a shell — no Python, no shipped code, no py2 constraint. This is a -portability gain worth advertising and a behavioural change worth testing: the integration suite -should assert it with a batch-host variant that has no usable interpreter (IC-ADR-002 §7). - -### 4. Data plane — payload and sandbox - -DIRAC's stdout/stderr-only model is generalised to **arbitrary input/output sandboxes**. This is -not a feature bolt-on; it is the *same requirement* as payload-agnosticism — if interCEde does -not interpret the payload, it cannot assume the payload emits only stdout/stderr. stdout/stderr -become two well-known **members** of the output sandbox; the executable becomes one distinguished -member of the input sandbox. - -```python -@dataclass -class SubmissionSpec: # the opaque payload (name provisional — see Open Issues) - executable: str - arguments: list[str] - input_sandbox: list[FileRef] # files/dirs (or streams) staged in before the run - output_sandbox: OutputSpec # explicit names | globs | "everything produced" - resources: Resources - environment: Mapping[str, str] - tag: str | None = None # consumer-set token, reserved for post-crash - # reconciliation (list-by-tag — see Open Issues) -``` - -Rules: - -- **Declared at submit, materialised at fetch.** HTCondor needs `transfer_output_files` *in the - submit description*, so the output-sandbox spec lives in `SubmissionSpec`, not only as a - fetch-time argument. Permissive backends (ARC lists the session dir; SSH globs the remote - workdir) may also discover outputs at fetch time, so `OutputSpec` expresses both explicit lists - and globs/"all". -- **Caller-owned inputs are consumed at submit.** `submit()` stages (or copies) the executable - and every `input_sandbox` member *during the call* and retains no reference to caller-owned - paths after it returns — the caller may delete its temp files immediately. This rule removes a - DIRAC wart by construction: today the SiteDirector deletes the submitted executable *unless* - the CE returns `ExecutableToKeep` (HTCondorCE does, because it still needs the file on disk - afterwards), a hidden ownership handshake no other CE participates in. -- **Return paths, not contents — and stream.** `fetch_output(ids, dest)` streams each member into - `dest` and returns, per job, a **manifest** (`JobOutput` with `.stdout`, `.stderr`, - `.files[...]`, and `.log`); it never loads file contents into memory. This is the one model - that survives multi-GB outputs. -- **Staging is delegated or synthesised.** "Stage a sandbox" is either delegated to the backend's - native mechanism (ARC session dir, HTCondor file transfer) or synthesised by interCEde over the - transport (`Transport.put`/`get`), selected by the scheduler's `stages_own_files` flag (§3). - The composed CEs get arbitrary sandboxes *for free* because the transport already provides - `put`/`get`. -- **One manifest, one retrieval.** `fetch_output` materialises stdout, stderr, output files - **and** the CE/scheduler log together into `dest`, as members of a single manifest. The - manifest separates **CE/scheduler artifacts** (the log — infra-level, interCEde's to expose) - from **payload artifacts** (fetched, never interpreted). Whether the fetch also *releases* the - remote state is the backend's declared `destructive` property (§2) — when it is declared, the - consumer must treat the local `dest` as the commit point (§9). There is no standalone "get the - final log" call; a *while-running* diagnostics fetch exists only where it is genuinely - independent and non-destructive (`SupportsLiveDiagnostics`, §2). `purge(ids)` on its own still - deletes a job's outputs/remote state *without* fetching — the path a site takes when it - mandates scratch cleanup for outputs no one will retrieve (`Purgeable`, §2). -- **Bounded materialisation, enforced where interCEde does the copy.** "Everything produced" plus - arbitrary filenames from a payload interCEde never inspects means `fetch_output` needs a size - ceiling, a file-count ceiling, a per-transfer timeout, and path containment (every member lands - **under** `dest`; `..`/absolute members are rejected). How much is enforceable inline depends - on who does the copy: where interCEde streams file-by-file itself (ARC, SSH) all four limits - apply mid-stream; where the backend's own tool does the transfer (HTCondor) enforcement is - coarser — a submit-time allow-list, a timeout on the transfer command, and a post-transfer - size/count check. The limits are stated per backend rather than assumed uniform. - -Results are typed dataclasses/models; whole-operation failures raise a **typed exception -hierarchy** (replacing `S_OK`/`S_ERROR`) while per-item outcomes are returned in the bulk map -(§2). The contract is **`async`, and async is the single source of truth**: the backends are all -I/O-bound (SSH, ARC/cloud REST, subprocess) and the contract is bulk, so async is the natural -model for polling/fetching many jobs concurrently. A sync-only backend library (the `htcondor` -bindings, Libcloud, a batch CLI) is wrapped in a thread internally (`asyncio.to_thread`), never -surfaced as a second contract — backends implement only the async protocols. Sync *callers* -(DIRAC during the transition) are served by a thin sync **facade** that runs the async calls to -completion, **not** by a parallel sync API (see Open Issues); a sync caller still benefits from -the bulk fan-out, which runs concurrently *inside* each run-to-completion call. - -### 5. Discovery — registry and entry points - -`ComputingElementFactory` + `ObjectLoader` string-to-module loading is replaced by a typed -**registry** populated through Python **entry points**. Three groups, one per pluggable kind: - -```toml -# a backend package's pyproject.toml -[project.entry-points."intercede.backends"] -arc = "intercede.backend.arc:ARCBackend" -htcondor-ce = "intercede.backend.htcondor:HTCondorCEBackend" - -[project.entry-points."intercede.transports"] -ssh = "intercede.transport.ssh:SSHTransport" -local = "intercede.transport.local:LocalTransport" - -[project.entry-points."intercede.schedulers"] -slurm = "intercede.scheduler.slurm:Slurm" -htcondor = "intercede.scheduler.htcondor:HTCondorScheduler" -``` - -How it resolves: - -- **Lazy, typed lookup.** The registry reads the entry-point groups on first use but imports only - the *one* target a request names — no eager import of every backend. Each resolved object is - checked for structural conformance (`isinstance` against the relevant `@runtime_checkable` - protocol) at the boundary, so a misregistered class fails fast with a clear error instead of at - the first method call. -- **A CE is described by data, not a class name.** A request is a small typed config — - `{"type": "arc", ...}` for a monolithic backend, or `{"transport": "ssh", "scheduler": "slurm", - ...}` for a composed one. For the composed form the registry instantiates the named transport - and scheduler and wraps them in `BatchBackend`; the *combination* never needs its own - registered class, which is what keeps the matrix at N+M. -- **Third parties register without forking.** A VO ships a package advertising any of the three - groups (an in-house scheduler, a site-specific transport, a bespoke CE); it is discovered - automatically once installed in the same environment. Nothing in interCEde is edited, and the - package depends only on Tier-A protocols. -- **Discoverable and versioned.** Unlike `ObjectLoader`, the mapping from a short stable name - (`"slurm"`) to an implementation is owned by the providing package's metadata, versioned with - it, and enumerable (`importlib.metadata.entry_points`) for tooling and diagnostics ("what - backends does this environment offer?"). -- **Explicit override precedence.** Name collisions resolve by a documented rule (e.g. a - configured allow-list / last-installed-wins) so a site can shadow a built-in scheduler with its - own without patching interCEde. - -### 6. The severed "inner" subsystem (out of scope, recorded for completeness) - -`InProcess`, `Singularity` and `Pool` move to the **pilot/worker-node domain** with their own -contract, completely disconnected from `JobBackend`, and deliberately **not** modelled as -backends at all (in DIRAC they were lumped under the same `ComputingElement` base, and that -shared name is what created the false unity). Their relationship is a **depth-1 aggregator** — -*not* the full (recursive) Composite pattern, because pools-of-pools are unwanted: - -- `Runner`: execute one payload. `HostRunner` (was `InProcess`) runs it directly; - `ContainerRunner` (was `Singularity`) is a **decorator** that wraps another `Runner`'s command - in a container. Decoration is a *fixed-depth* `Runner → Runner` edge (a container around a host - runner), not aggregation — it composes exactly one runner, never a collection. -- `ExecutorPool` (was `Pool`): aggregates a collection of `Runner`s and adds bounded concurrency. - The restriction that matters is that **`ExecutorPool` is not itself a `Runner`** (the child - edge is typed `ExecutorPool → Runner`); since a pool is not a runner, it cannot be a child of - another pool. That is what makes pools-of-pools unrepresentable — without forbidding the - bounded, non-recursive `ContainerRunner`-over-`HostRunner` decoration. - -```mermaid -classDiagram - direction LR - class Runner { - <> - +run(payload) PayloadResult - } - class HostRunner - class ContainerRunner - class ExecutorPool { - -limit: int - } - Runner <|.. HostRunner - Runner <|.. ContainerRunner - ContainerRunner o-- Runner : wraps (fixed depth) - ExecutorPool o-- Runner : aggregates (pool is not a Runner) -``` - -This subsystem is tracked as separate work (likely against the Pilot repo); it is named here only -so the migration map is complete. One scope warning for that work, from the consumer analysis: -the JobAgent-side contract is richer than `run(payload)`. It also covers -*description-for-matching* (`getDescription()` — where Pool returns a **list** of CE dicts -implementing the MultiProcessor tag strategy), *filling-mode accounting* (`setCPUTimeLeft()`), -*asynchronous result harvesting* (the mutable `taskResults` dict plus the `AsyncSubmission` -flag), and Pool's per-job writes to `pilot.cfg`. The sibling ADR must scope all four, not just -execution. - -### 7. Extension surface — what is extendable, and what is not - -interCEde is a library implemented and mocked by others, so its public surface is defined -**explicitly** and governed by SemVer. Every public module declares `__all__`; anything not in a -public module's `__all__`, and any module or name prefixed with `_`, is internal and may change -without notice. Three tiers: - -**Tier A — Stable contracts (implement these).** The extension points third parties build -against. Breaking changes are SemVer-major. - -- The protocols: the essential `Submitter`, `Monitor` and their composition `JobBackend`; the - collaborators `Transport`, `Scheduler`; and the optional capability protocols - (`OutputRetriever`, `Cancellable`, `Purgeable`, `LoadReporter`, `SupportsLiveDiagnostics`, …). -- The data types and enums: `SubmissionSpec` (name provisional — see Open Issues), `Submission` - (bundles `.handles` and `.failures`), `JobHandle` (durable, serialisable, routing-carrying) and - its identity `JobID`, `JobStatus` (incl. `UNKNOWN`), `StatusMap`, `JobOutput`, `OpOutcome`, - `JobCounts`, `Diagnostics`, `Resources`, `FileRef`, `OutputSpec`. -- The exception hierarchy (a single rootable `InterCEdeError`). -- The registry API and entry-point group names. -- The lifecycle: backends and transports are async context managers (§2) — cheap, - side-effect-free construction; idempotent close; consumers that cache them own eviction-time - closing. - -**Tier B — Provided implementations (instantiate and register; do not subclass).** Public to -*use*, not a subclassing contract. Extend by **implementing a protocol or composing**, never by -subclassing these. - -- `BatchBackend`; the concrete transports (`SSHTransport`, `LocalTransport`, …); the concrete - schedulers (`Slurm`, `HTCondorScheduler`, `LSF`, `SGE`, `OAR`, `Torque`, `Direct`); the - monolithic CEs (`ARCBackend`, `HTCondorCEBackend`, `CloudBackend`). -- An **optional** convenience base, `BaseBackend(ABC)`, may carry *shared mechanics only* (config - normalisation, logging, retry/timeout helpers) and exposes a small, documented set of - overridable hooks. It is **not** the contract: a fully-conformant CE can be written without it, - and policy (slot accounting, credential renewal) is **not** placed in it. Because "no policy, - by convention" is exactly the DIRAC weakness this ADR indicts, two guardrails keep it from - re-accreting into a fat base: (1) the helpers it may contain are a **closed, enumerated set**, - and a unit test asserts it exposes *no* lifecycle method (`submit`/`status`/`fetch_output`/ - `kill`) and *no* policy method (anything like DIRAC's `available()`/`_monitorProxy`); (2) - interCEde's **own** monolithic CEs do **not** subclass it — so it can never become load-bearing - inside interCEde the way DIRAC's base became load-bearing in nine places. It exists for - third-party convenience and is tested against the same conformance suite as any other backend. - -**Tier C — Internal (do not import).** Command builders, output parsers, transport internals, -materialisation/retry machinery, anything under a `_`-prefixed module. No stability guarantee. - -The guiding rule, and the single most important lesson from DIRAC's fat base: **the preferred -extension mechanism is composition + structural typing, not inheritance.** You add a backend by -implementing a Tier-A protocol and registering it — not by subclassing a Tier-B class. - -### 8. DIRAC → interCEde migration map - -The map records *where each DIRAC piece lands*. Method-by-method porting mechanics (DIRAC's -job-ref formats, `S_OK` payload keys, CS parameter names) belong to the migration mapping -document owned by the sync-facade work (see Open Issues), not to this table. - -| DIRAC (`Resources/Computing`) | interCEde destination | Notes | -| --- | --- | --- | -| `AREXComputingElement` | `ARCBackend` (`JobBackend` + all four optional capabilities + `OutputRetriever`) | REST; reference implementation of the full sandbox model; `killJob` → `kill`, `cleanJob` → `purge`; the `diagnose` endpoint → `SupportsLiveDiagnostics` | -| `HTCondorCEComputingElement` | `HTCondorCEBackend` (monolithic; `JobBackend` + `OutputRetriever` + `Cancellable`) | native file transfer; destructive fetch (§2); no `LoadReporter` (reports no counts); shares the internal `_htcondor` core with `HTCondorScheduler` — composition candidate (Open Issues) | -| `SSHComputingElement` | `BatchBackend(SSHTransport, )` | composition; sandbox via `Transport.put`/`get` | -| `SSHBatchComputingElement` | `BatchBackend(SSHMultiHostTransport, Direct)` | host routing moves into `JobHandle`; host-spreading is placement *policy* (Open Issues) | -| `LocalComputingElement` | `BatchBackend(LocalTransport, )` | remote-family despite the name; its spoofed `ssh://` job-ID hack is dropped — routing lives in `JobHandle` (§1) | -| `CloudComputingElement` | `CloudBackend` (monolithic; essential protocols only) | no `OutputRetriever` (§2); `cleanupPilots` → `purge` | -| `CloudProviders/{OpenNebula,…}` | custom Apache Libcloud `NodeDriver`s — Tier-C internals of `CloudBackend` | provider selection stays delegated to Libcloud (§3, Rejected Ideas); VO-pluggability open | -| `BatchSystems/{SLURM,Condor,LSF,GE,OAR,Torque,Host}` | `Scheduler` implementations (`Slurm`, `HTCondorScheduler`, `LSF`, `SGE`, `OAR`, `Torque`, `Direct`) | promoted from ad-hoc string plugins to the typed protocol; ship-a-python-driver model dropped (§3) | -| `BatchSystems/TimeLeft/*` | **out of scope as an interface** → pilot/worker-node side | queried from *inside* an allocation — a different vantage than the submission-side `Scheduler` (Open Issues) | -| `ComputingElementFactory` (`ObjectLoader`) | typed registry + entry points (§5) | | -| `submitJob` (all remote CEs) | essential `Submitter.submit` | inputs consumed at submit — the `ExecutableToKeep` handshake disappears (§4) | -| `getJobStatus` (all remote CEs) | essential `Monitor.status` | | -| `getJobOutput` (all remote CEs) | **optional** `OutputRetriever.fetch_output` | optional because some backends cannot serve pull-retrieval (§2) | -| `killJob` / `cleanJob` (remote CEs) | optional `Cancellable.kill` / `Purgeable.purge` | not essential — some flows never use them | -| `ComputingElement` base — `getCEStatus()` counts | optional `LoadReporter` (counts only) | the *availability* computation (counts vs `Max*Jobs`) is **caller policy**, removed from interCEde | -| `ComputingElement` base — `shutdown()` (+ `QueueCECache` eviction) | async context-manager lifecycle (§2) | | -| `ComputingElement` base — `setProxy`/`setToken`/`_monitorProxy` | **split**: backend auth → [IC-ADR-003](IC-ADR-003_credentials.md); *payload* proxy renewal → pilot/runner side | | -| `ComputingElement` base — config hierarchy | a loader utility (Tier C), not a base-class responsibility | | -| `InProcessComputingElement` | **out of scope** → `HostRunner` (pilot/runner subsystem, §6) | severed | -| `SingularityComputingElement` | **out of scope** → `ContainerRunner` (decorator over a runner, §6) | severed | -| `PoolComputingElement` | **out of scope** → `ExecutorPool` (depth-1 aggregator, §6) | severed | - -### 9. Consumer interface — how DiracX uses interCEde - -The consumer surface is deliberately small: the Tier-A protocols plus the registry factory. -DiracX splits the old monolithic SiteDirector into **separate tasks**, and each one is typed to -the *narrowest* protocol it needs — never to `JobBackend`, never to a concrete class. The -registry resolves the same backend for all of them; each task sees only its slice. One sketch — -the submission task; the status and output tasks follow the same resolve-narrow-drive pattern: - -```python -from intercede import registry, Submitter, LoadReporter - -async def submission_task(resource, want, make_payload): - ce: Submitter = registry.backend(resource) # isinstance-checked at the boundary - if isinstance(ce, LoadReporter): # optional -> structural narrowing - want = min(want, policy.slots(await ce.counts())) # throttling policy is DiracX's - spec = make_payload() # -> one SubmissionSpec - sub = await ce.submit(spec, count=want) # same spec, `want` identical copies - await store.record(sub.handles) # DiracX owns this store - # sub.failures carries any copies the backend rejected, addressable per copy -``` - -- **The status task** depends on `Monitor` only: it polls and records (`store.update(await - ce.status(handles))`), with unknown ids coming back as `JobStatus.UNKNOWN` so nothing is - silently lost. It literally *cannot* call `fetch_output` — which both documents intent and - keeps output transfers out of the polling loop. -- **The output task** first confirms the backend has the *optional* `OutputRetriever` at all - (`isinstance` narrowing; Cloud lacks it, §2) and fetches into a durable `dest`. When the - backend declares `destructive` (§2), the remote copy is gone once the call returns, so `dest` - is the commit point — upload from there, and on a crash resume from `dest`, never re-fetch. - -What this fixes on the contract side: - -- **Restart re-drives handles; durability is DiracX's.** Tasks re-resolve the backend and drive - their slice on handles **reloaded from the DiracX store** — they never submitted anything in - this process, and interCEde never stored anything. This is why `JobHandle` must be serialisable - and self-contained (§1). -- **The submit→record window is the consumer's to close (interCEde is stateless).** A crash after - `submit` returns but before `store.record` commits leaves jobs on the backend that DiracX has - no handle for. This is not new — DIRAC's SiteDirector has the identical window today — and - interCEde cannot fix it, having no store; the candidate closing mechanism (tag-at-submit + - list-by-tag, using the reserved `tag` field from §4) is an Open Issue. -- **Pull vs push is invisible to interCEde.** A pull SiteDirector and a push PushJobAgent run the - *same* submission task; only *where the payload comes from and when* differs, and that lives in - the consumer. -- **Policy is the consumer's, mechanism is interCEde's.** Every `policy.*` call (throttling, - expiry, retries, "what is a pilot") is consumer-side; interCEde supplies only the verbs. - Optional capabilities are reached by structural narrowing, never assumed. - -## Rationale - -- **Protocols over a fat base class.** The thing callers depend on is a *contract*; a Protocol - expresses it without coupling implementations to interCEde's base, which is exactly what a - library implemented by third parties and mocked against containers needs. DIRAC's pain came - from the opposite: a fat base plus "abstract by convention" stubs with no enforcement and - drifting signatures. `@runtime_checkable` keeps an `isinstance` gate at the registry boundary; - a real `@abstractmethod` ABC is available where forcing an override is genuinely wanted. -- **Composition over inheritance for access × scheduler.** Inheritance forces N×M leaf classes or - fragile multiple inheritance; composition makes it N+M and is the literal expression of the - project's "backends combine" promise. DIRAC already proved the bone is sound (`SSH` + - `BatchSystems`); interCEde just applies it uniformly and gives the scheduler side a real typed - contract. This is well-trodden ground: HTCondor's **blahp** translates one protocol into - per-LRMS submit/status/cancel scripts, and ALICE's JAliEn has per-batch `BatchQueue` drivers — - the `Scheduler` protocol is the same idea, in typed Python. Their async - request-id/`RESULTS`-poll model is also why interCEde's contract is async, bulk, and poll-based - (§2) rather than blocking (see Rejected Ideas for why we reimplement rather than reuse them). -- **Capability segmentation, sized by real callers.** Interface Segregation, with the DiracX-task - split as the evidence: a status task wants `Monitor` and an output task wants `OutputRetriever`, - each *without* `submit` and without each other — so these are separate protocols, not one - "operator". `submit` and `status` are the two *essential* ones (every usable backend has them); - output-retrieval is a real task but optional, because a backend that cannot serve - pull-retrieval must not be forced to stub it — the same reasoning that makes `LoadReporter` and - independent diagnostics optional. Segment to match real callers, no further — and keep *policy* - (the counts → "is there room?" decision) in the caller, not the backend. (Counts-only was - checked against DIRAC: no remote-family caller consumes more than `{running, waiting}` — the - processors-cap math in the base `available()` is fed only by the severed inner CEs, so it moves - with the pilot-side subsystem.) -- **Severing the inner CEs.** Liskov is the test: no caller ever holds a `JobBackend` and uses it - without knowing whether it is remote or inner, because remote and inner serve *different - actors* with disjoint mechanics. The shared base bought only the factory and some config - boilerplate — both replaceable. Removing it deletes a false abstraction at near-zero cost. -- **Sandbox generalisation.** Payload-agnostic ⟹ sandbox-general. Returning a streamed manifest - of paths (not in-memory contents) is the only model that scales to real output, and it unifies - stdout/stderr/output-files/CE-log behind one retrieval. -- **Explicit extension surface.** Uncontrolled subclassing of concrete classes is how the DIRAC - fat base became load-bearing in nine places. A defined three-tier surface lets interCEde evolve - Tier-B/C freely while third parties depend only on Tier-A — which is what makes SemVer - meaningful for a plugin ecosystem. - -## Evolution & non-conforming backends - -No abstraction is eternal; the design's job is to make change cheap and localised, not to predict -the future. Three cases: - -1. **A new backend that fits the shape — trivial, by construction.** A scheduler/transport/CE - that is still "delegate-and-poll" (most new batch systems and CEs are) is added by - implementing the relevant protocol, registering an entry point, and shipping a container - conformance test. No core change — this is the designed-for case, and the reason discovery is - data-driven. - -2. **A backend that does not fit the shape — branch, don't contort.** A future resource may be - fundamentally different: serverless/FaaS with no persistent handle, a callback/push resource - that calls the consumer, a long-lived interactive session, or something with no output-sandbox - notion. The rule — the same one applied when this ADR severed the inner CEs — is **when the - semantics differ, add a sibling abstraction; never overload `JobBackend`.** Additive - differences become optional capability protocols; a genuinely different lifecycle gets its own - protocol and its own registry group. The monolithic-CE escape hatch already lets a - weird-but-pollable backend (e.g. Kubernetes: submit a Job object, poll, fetch logs, delete) - implement the contract directly with whatever internals it needs. - -3. **The contract itself stops making sense — manage it deliberately.** This is what ADRs, SemVer - and the narrow surface are for: - - Consumers depend only on small Tier-A protocols resolved by data, so implementations - (Tier B/C) can be rewritten freely — schedulers moving from CLI to REST changes only - `Transport`/`Scheduler` bodies, not the contract. - - A breaking change is a **new ADR superseding this one** plus a SemVer-major with a - deprecation window. - - Protocols are structural and versionable: introduce `JobBackendV2`, expose both through the - registry, bridge with adapters, and migrate backends one at a time — old and new run side by - side. - - **Prefer additive evolution** — a new optional capability protocol over a core change. The - core is kept deliberately small precisely to minimise what can break. - - The container conformance suite is the early-warning system: change an interface and the - per-backend tests show what breaks before any user does. - - The meta-principle: keep the core minimal, push variation into optional capabilities and - composition, and treat "this does not fit" as a signal to branch a new abstraction rather than - bend the old one. - -## Rejected Ideas - -- **Keep DIRAC's single fat `ComputingElement` base.** Conflates contract and policy, enforces - nothing, and is the documented source of the signature drift and the Remote/Inner/Pool - confusion. -- **One god-interface with every method mandatory.** Forces backends that lack a capability to - ship `S_ERROR`/`NotImplementedError` stubs — exactly what DIRAC's - `HTCondorCEComputingElement.getCEStatus()` already is. Additive optional protocols replace it — - even output-retrieval is optional (§2). -- **A separate `fetch_log` retrieval operation.** Breaks on destructive-fetch backends (§2), - where retrieval deletes the spool — the log must come out in the *same* operation as the - output. The log is therefore a manifest member of the single `fetch_output`; only ARC's - genuinely independent, non-destructive `diagnose` endpoint justifies a separate (optional) - diagnostics call. -- **Model access × scheduler with inheritance** (N×M classes, or a mixin lattice). Combinatorial - and fragile; composition is strictly better here. -- **Keep the inner CEs under `JobBackend`** (even as a sibling branch). They are never - substituted for remote CEs and share no mechanics; the common supertype is fictitious. -- **Use the full recursive Composite pattern for the pool.** Its defining feature — a composite - containing components, hence composites-of-composites — yields pools-of-pools, which we do not - want. A depth-1 aggregator with a leaf-typed child edge is the right restriction. -- **stdout/stderr-only output (DIRAC's default).** Incompatible with payload-agnosticism; ARC - already shows the general model is feasible, and HTCondor/SSH can do it natively or via the - transport. -- **Subclassing concrete CEs as the extension mechanism.** Re-creates the fat-base coupling; the - registry + protocol path keeps the provided implementations free to change. -- **`ObjectLoader` string-to-module discovery.** Untyped and undiscoverable; entry points give - typed, packageable, third-party-friendly registration. -- **Keep the CE layer inside DIRAC/diracx (no standalone library).** Rejected. A standalone - library must serve **both** DIRAC (sync, throughout the migration) and DiracX (async) from one - contract — an in-`diracx` subpackage cannot back the sync DIRAC side — and it lets - backend/CE-version support ship on its own cadence, without a DiracX release. It also isolates - the containerised test matrix from DiracX's CI, gives third-party VOs an entry-point extension - path that does not fork DiracX, and turns the Tier-A/B/C boundary into a packaging fact rather - than a convention. The cost — a second release train, SemVer ceremony, and diracx↔intercede - version skew — is accepted in exchange. (Were DiracX ever the sole consumer *and* the DIRAC - transition complete, folding it back in-tree would be worth revisiting.) -- **One repo for all DIRAC resources — or adding Storage/Catalog to interCEde.** Rejected: the - library boundary is a *coherent contract + testing domain*, so the repo count follows the - number of distinct contracts — contract-driven, not taxonomy-driven — and that count is small - and stable. - - *submit / monitor / retrieve* is one contract with one collaborator model - (`Transport` × `Scheduler`), one test rig (containerised schedulers), one consumer (the WMS). - Storage Elements (`put`/`get`/space, over XRootD/S3/gsiftp) and File Catalogs - (`register`/`query`/metadata, over DFC/Rucio) are *different contracts* with different - backends, test rigs and consumers (the Data Management System) — **sibling libraries**, not - tenants of interCEde. - - Merging them is the repo-level form of the fat-base false-unity this ADR rejects ("when - semantics differ, add a sibling abstraction, never overload the type" — §6, Evolution). DIRAC - itself never unified them: `Resources/{Computing,Storage,Catalog,…}` have distinct base - classes and factories, with no common `Resource` supertype. - - What siblings *may* share is **plumbing, never a contract** — the entry-point registry - mechanism, the typed-error base, and the async/sync-facade conventions could later factor - into a small `dirac-resources-core`. That factoring is a separate, deferrable decision, not a - reason to merge contracts now. -- **Reuse an existing job-submission abstraction instead of defining our own.** Surveyed and - rejected — nothing covers interCEde's target set (SSH+batch, ARC-CE REST, HTCondor-CE, Cloud) - as a modern async, typed Python library. HTCondor's blahp/GAHP is a process protocol, not a - library, and has no SSH story; PanDA/Harvester has the right plugin shape but is inseparable - from the PanDA server; JAliEn's batchqueue layer is Java, unreleased and licence-unclear; and - **none** of the generic scheduler libraries (RADICAL-SAGA, PSI/J, DRMAA, Parsl, AiiDA, - Dask-Jobqueue) speaks ARC at all, in any interface generation. The genuinely reusable layer is - the WLCG grid-CE ecosystem itself plus first-party per-backend clients (`pyarcrest`, the - `htcondor` bindings, Apache Libcloud), which interCEde composes behind a typed async contract. - The per-tool evidence is in the [reuse survey note](../notes/reuse-survey.md); blahp's per-LRMS - scripts and async poll model are worth learning from (see Rationale). - -## Open Issues - -Items tagged **(blocking)** need an answer before this ADR moves to Accepted; **(deferred)** -items are follow-up work that does not gate acceptance. - -- **Naming at the API boundary (blocking).** Keep `Job*` only for the *scheduler* handle - (`JobHandle`, `JobID`, `JobStatus` — accurate and namespaced). The submitted description is - provisionally `SubmissionSpec` — `JobSpec` clashes with DiracX Jobs, `Payload` with the - pilot-side payload. Confirm, or pick another neutral name. -- **Destructive-fetch confirmation (blocking).** Confirm the HTCondor ≥ 25.8 one-shot spool - behaviour (§2) against the release notes, whether setting `leave_in_queue` restores ARC-like - re-fetchability, and that every consumer reads `destructive` before assuming it can re-fetch. -- **HTCondorCE live pilot-log fetch (blocking).** DIRAC operators fetch running-pilot logs from - both AREX *and* HTCondorCE (`dirac-admin-get-pilot-logging-info`, via the CE's `getJobLog`). - Under this ADR the final log is a `fetch_output` manifest member and only ARC gets - `SupportsLiveDiagnostics` — which would regress the operator flow for running HTCondorCE - pilots, because HTCondorCE's log fetch rides the same spool-transfer machinery as output - retrieval. Decide: implement `SupportsLiveDiagnostics` on `HTCondorCEBackend` over that - machinery (documenting that a fetch against a *completed* spooled job shares the destructive - semantics, §2), or record the regression as accepted. -- **Submit idempotency / list-by-tag (deferred).** The submit→record crash window (§9) is the - consumer's to close; the realistic mechanism is **tag-at-submit + list-by-tag** — stamp a token - on the jobs at submit, list jobs carrying it after a crash, reconcile before re-submitting. The - `tag` field is already reserved on `SubmissionSpec` (§4, additive and non-breaking); the - deferrable part is the enumeration capability (a future optional protocol). `LoadReporter`'s - aggregate counts cannot substitute for it. -- **Cloud staging (deferred).** Whether `CloudBackend` later grows a limited `OutputRetriever` - (a transport *into* the VM), and whether "declared staging capability" deserves its own - optional protocol (§2 records why it has none today). -- **Cloud-provider drivers (deferred).** interCEde keeps *composing* Libcloud - (`get_driver`/`set_driver` stays the provider-selection mechanism) rather than defining its own - driver protocol (§3). Open: whether VO-pluggable clouds warrant a thin - `intercede.cloud_drivers` entry-point group over Libcloud's registry, and whether the - OpenNebula driver is maintained in-tree (Tier C) or upstreamed to Libcloud. -- **Home of the severed subsystem (deferred).** Where `Runner`/`ExecutorPool` live (Pilot repo vs - a new small library) — a separate ADR/tracking item (§6 lists the scope warnings for it). -- **`HTCondorCEBackend` as composition (deferred).** It shares the `_htcondor` core with - `HTCondorScheduler`; expressing it as `BatchBackend(HTCondorCETransport, HTCondorScheduler)` - would need a transport-level notion of a "destructive get", and the CE fronts a *different* - scheduler than a raw schedd (its own blahp hides the site's real batch system). Share the - internals now; revisit the shape later. -- **Multi-host SSH placement (deferred).** DIRAC's `SSHBatch` spreads a submission across hosts - by free slots and encodes the host in the job ref. The routing belongs in `JobHandle` (§8); the - spreading decision is placement **policy**. Decide when this backend is scheduled: caller - policy (consistent with the mechanism/policy split) vs a declared multi-host transport - capability. -- **`get_time_left()` placement (deferred — confirm).** It is queried from *inside* a running - allocation — a different vantage and consumer than the submission-side `Scheduler`. Kept - **off** the `Scheduler` protocol to preserve role cohesion; the batch-system parsing is shared - internal code, exposed via a separate pilot-facing capability if the pilot needs it. Confirm - this split (and payload-proxy renewal, likewise pilot-side). -- **DIRAC compatibility shim (deferred).** DIRAC is synchronous and `S_OK`/`S_ERROR`-based; the - resolution is a sync **facade** generated over the async core (run-to-completion + - `S_OK`/`S_ERROR` translation), usable only by sync callers outside an event loop — **not** a - hand-maintained parallel sync protocol, which would re-introduce the two-surfaces drift this - ADR fights. One design constraint is fixed now, because getting it wrong breaks cached - backends: the facade must own a **single long-lived background event loop** in a dedicated - thread and dispatch every call onto it (`run_coroutine_threadsafe`) — never per-call - `asyncio.run()`. Backend connection state (HTTP sessions, SSH connections) is bound to the - event loop it was created on, so a fresh loop per call breaks any backend reused across agent - cycles (the §2 lifecycle/caching model); the facade likewise exposes a sync `close()` over the - async close so cache eviction works. Sync callers inside DIRAC's Tornado-based services are - expected to be safe (handlers run in worker threads, with no running loop in the calling - thread) — verify once. Confirm the facade's lifetime (transition-only vs kept for sync - third-party tools). The facade work also owns the **migration mapping document** — the - translation table from DIRAC's load-bearing conventions (job-ref formats, `PilotStampDict`, - `S_OK` payload keys, CS parameter names, `Tag: Token[:vo]` vs IC-ADR-003 requirements) to - interCEde's. The mapping document lives in interCEde docs; the adapter code lives DIRAC-side. -- **Heterogeneous submit batches (deferred).** `submit(spec, count)` submits `count` *identical* - copies, which map to native array submission (HTCondor `queue N`, Slurm `--array`). A - payload-agnostic push consumer (PushJobAgent) may want a *heterogeneous* batch of distinct - specs. Decide whether to add a `submit(specs: Sequence[SubmissionSpec])` overload (which loses - native-array efficiency on most backends) or leave heterogeneous batches as N separate calls. -- **Staging as a third collaborator (deferred).** Staging currently rides on `Scheduler` - (`stages_own_files` + hooks, §3), and the HTCondorCE-as-composition idea would need a - "destructive get" on the *transport* — two cross-axis concerns before any code exists. - Evaluate promoting staging to its own collaborator protocol (a `Stager` strategy composed - alongside `Transport`/`Scheduler`). -- **Capability declaration vs structural sniffing (deferred).** Dispatch narrows by `isinstance` - against `@runtime_checkable` protocols, which check method *presence only* (§2). Evaluate an - explicit capability declaration (registry metadata, or a `capabilities()` set) as a sturdier - gate, keeping `isinstance` as the smoke-test. -- **Loud partial-failure results (deferred).** Bulk verbs return per-`JobID` outcome maps, which - a caller can silently ignore (unlike a raised exception). Consider a result type that makes - undrained failures loud (a `.raise_for_failures()` helper, or logging on drop) plus single-job - convenience wrappers over the bulk verbs. - -Note: the backend credential/auth model was an open issue here and is now its own ADR — -[IC-ADR-003](IC-ADR-003_credentials.md) (typed credentials, backend-declared requirements, -provider-based supply). Only *backend* auth is interCEde's; payload credential renewal stays -pilot-side. diff --git a/docs/adr/IC-ADR-002_integration_tests.md b/docs/adr/IC-ADR-002_integration_tests.md deleted file mode 100644 index 1599795..0000000 --- a/docs/adr/IC-ADR-002_integration_tests.md +++ /dev/null @@ -1,398 +0,0 @@ -# IC-ADR-002: Integration testing against containerized backends - -## Metadata - -- **Created By:** Alexandre Boyer -- **Date:** 2026-07-02 -- **Status:** Draft -- **Decision Maker(s):** Federico Stagni, Christophe Haen, Chris Burr -- **Stakeholders:** interCEde contributors and maintainers; DIRACGrid CI maintainers -- **Depends on:** IC-ADR-001 (core architecture: Protocols, Transport × Scheduler composition, capability segmentation) - -> **Scope and altitude.** Direction-setting for the *test architecture*: what runs (real backend -> daemons in per-stack compose environments), where (GitHub-hosted CI, on every PR), and how -> versions are managed (pins + Renovate + support windows). The contents of individual tests, the -> exact Dockerfiles, and the CI YAML are implementation, governed by the architecture decided -> here. - -## Abstract - -interCEde's contract is expressed as structural `Protocol`s exercised against fakes, which structurally cannot catch the bugs interCEde exists to absorb: version-specific daemon behaviour, destructive output retrieval, and status-mapping quirks. This ADR decides to add integration tests that run **real backend daemons** — in the combinations interCEde supports — in GitHub-hosted CI on every pull request, within a standard runner. The unit of everything is a **stack**: a named, self-contained backend environment defined by one `docker compose` file plus versioned configuration, declared in a single `stacks.yml` manifest from which the CI matrix is generated. A single, backend-agnostic contract suite is dispatched across stacks by capability (protocol narrowing), mirroring how the library itself handles capability variance. Backend images build from pinned RPMs, are prebuilt and pushed to GHCR, and every version — image tags, Dockerfile `ARG`s, Python dependencies — is pinned and tracked by Renovate, so an upstream release that breaks a contract surfaces as a red, bisectable Renovate PR rather than tribal knowledge. Credentials are generated ephemerally per run and never committed. A non-blocking weekly canary runs the same matrix against unpinned tags for early warning. Adding a backend or configuration is a manifest entry plus files, with no workflow edits. - -## Motivation - -interCEde's core architecture is deliberately mock-friendly: everything is a `Protocol`, backends -are composed, and the unit test suite exercises contracts against fakes. That is also its greatest -testing risk. A structural protocol plus a fake will happily agree with each other forever while -the real ARC REST endpoint, the real `condor_submit -spool` handshake, or the real -`sbatch`-over-SSH quoting rules drift away underneath. The bugs interCEde exists to absorb — -version-specific daemon behaviour, destructive output retrieval, status-mapping quirks — are -precisely the bugs that unit tests structurally cannot see. - -DIRAC's history makes the point concretely: HTCondor ≥ 25.8 silently changed output retrieval for -spooled jobs into a one-shot operation. No unit test anywhere would have caught that; only a test -that submits a real job to a real schedd and fetches its output twice would go red. - -We therefore need integration tests that: - -1. run real backend daemons, in the combinations interCEde actually supports; -2. run in GitHub-hosted CI, on every pull request, within the resources of a standard runner - (4 vCPU / 16 GB on public repositories); -3. surface **new upstream versions** (ARC, HTCondor, Slurm, OpenSSH, and our own Python - dependencies) as reviewable Renovate pull requests whose CI result *is* the compatibility - verdict; -4. support **multiple configurations per backend combination** eventually (token vs. proxy auth, - shared vs. non-shared filesystem, alternative queue setups) while starting with exactly one - basic configuration each. - -### Initial target combinations - -| Stack ID | Backend under test | interCEde components exercised | -|---|---|---| -| `arc-slurm` | ARC 7 CE (REST) → Slurm LRMS | `ARCBackend` (AREX REST), end-to-end through a real LRMS | -| `arc-condor` | ARC 7 CE (REST) → HTCondor LRMS | `ARCBackend` against the other major LRMS | -| `htcondor` | HTCondor-CE → HTCondor pool | `HTCondorCEBackend`; plain-schedd Condor scheduler tests reuse the same pool | -| `ssh-slurm` | sshd → Slurm | SSH transport × Slurm scheduler | -| `ssh-condor` | sshd → HTCondor (schedd) | SSH transport × Condor scheduler | -| `local-slurm` | Slurm, tests run *inside* the container | Local transport × Slurm scheduler | - -`ssh-*` and `local-*` share the same images — the local variant simply executes pytest inside the -batch container instead of connecting over port 22. Which of `ssh-slurm` / `local-slurm` lands -first is an implementation detail; the architecture treats them identically. Local × HTCondor -(first-class in IC-ADR-001's driver list) is deliberately not an initial stack: the Local -transport is exercised by `local-slurm` and the Condor scheduler by `ssh-condor`, so the -combination adds no new axis; it can be added later as a manifest entry reusing the `ssh-condor` -stack (`exec_in_container: true`). - -### Facts constraining the design - -- **HTCondor** publishes official role images on Docker Hub (`htcondor/mini`, `htcondor/cm`, - `htcondor/execute`, `htcondor/submit`), version-tagged (`-el9`, `lts-el9`). These are - Renovate-trackable out of the box. -- **HTCondor-CE** has no first-party image with clean semantic tags (the OSG images use date tags - and OSG-series semantics); an in-repo Dockerfile installing a pinned `htcondor-ce` RPM is more - Renovate-friendly than chasing OSG tags. -- **ARC** has no maintained official Docker image. The supported path is installing pinned - `nordugrid-arc-*` RPMs on EL9 in our own Dockerfile. interCEde targets **ARC 7 and later - only** — ARC 6 is out of scope (see §8). Crucially, ARC ships *zero configuration*: - a minimal working CE with a **Test-CA and host certificate generated at install time** - (`arcctl test-ca`), which removes the entire grid-PKI problem from CI. -- **Slurm** has no official image; community images are unmaintained to varying degrees. An - in-repo Dockerfile with a pinned Slurm package version (single node running `slurmctld` + - `slurmd` + `munged`) is small and fully under our control. -- **Renovate** is already configured in this repository (pip/pep621/github-actions managers). It - additionally supports `dockerfile` and `docker-compose` managers (image tags), and - `customManagers` (regex) for versions embedded in Dockerfiles as `ARG`s — the standard - `# renovate: datasource=... depName=...` comment convention. -- GitHub Actions **service containers** cannot express what we need (no compose, no build step, - no ordered startup, poor log access). Docker and `docker compose` are preinstalled on - `ubuntu-latest`, and compose v2 supports `up --wait` gating on healthchecks. - -## Specification - -### 1. A *stack* is the unit of everything - -A **stack** is a named, self-contained backend environment: - -``` -tests/integration/ -├── stacks.yml # the manifest: single source of truth for the CI matrix -├── conftest.py # stack-aware fixtures, capability-based skipping -├── test_submit.py # ONE backend-agnostic contract suite … -├── test_status.py -├── test_output.py # … including the fetch-twice destructiveness test -├── test_kill.py -└── stacks/ - ├── _images/ # Dockerfiles shared across stacks - │ ├── arc/Dockerfile # EL9 + pinned nordugrid-arc RPMs (ARG ARC_VERSION) - │ ├── slurm/Dockerfile # EL9 + pinned slurm packages (ARG SLURM_VERSION) - │ └── htcondor-ce/Dockerfile # EL9 + pinned htcondor-ce RPM (ARG HTCONDOR_CE_VERSION) - ├── arc-slurm/ - │ ├── compose.yml - │ └── config/ - │ └── basic/ # arc.conf, slurm.conf, intercede-client.toml - ├── arc-condor/… - ├── htcondor/… - └── ssh-slurm/… # sshd enabled; local-slurm reuses this stack -``` - -Rules: - -- One `compose.yml` per stack. Shared plumbing (network, credential volume, healthcheck blocks) - is factored with YAML anchors or compose `include:`, **not** with a generated mega-file. - A stack must be runnable locally with exactly - `docker compose -f tests/integration/stacks/arc-slurm/compose.yml up --wait` — CI does nothing - a developer cannot do on a laptop. -- Every configurable surface lives under `config//`. Today each stack has only - `config/basic/`. A *configuration* is mounted into the containers (arc.conf, condor config, - slurm.conf) **and** consumed by the test client (`intercede-client.toml` describing endpoint, - credential type, queue names). Backend config and client config change together, atomically, - under one name — this is what makes future configuration-matrix growth mechanical rather than - a refactor. -- `stacks.yml` is the manifest the CI matrix is generated from: - - ```yaml - stacks: - - id: arc-slurm - configs: [basic] - versions: [latest] # ARC 7.x leading edge (support window in §8) - markers: "remote and arc" - - id: htcondor - configs: [basic] - versions: [lts, latest] # HTCondor 24.0 LTS anchor + leading edge - markers: "remote and htcondor" - - id: ssh-slurm - configs: [basic] - markers: "scheduler and slurm" - exec_in_container: false - - id: local-slurm - stack: ssh-slurm # reuses the ssh-slurm compose stack - configs: [basic] - markers: "scheduler and slurm" - exec_in_container: true - ``` - - `versions` names the entries of the backend's support window to run (§8); a bare or absent - `versions` means the single leading-edge pin. Adding a stack, a configuration, or a supported - version is one manifest entry plus files — no workflow edits. - -### 2. One contract test suite, capability-dispatched - -There is a **single** integration test suite, not one per backend. Tests are written against the -interCEde protocols and parameterized by the stack's client configuration. Capability variance is -handled exactly the way the library itself handles it — protocol narrowing: - -```python -async def test_output_refetch(backend, submitted_job, tmp_path): - if not isinstance(backend, OutputRetriever): - pytest.skip("backend has no output retrieval") - first = await backend.fetch_output([submitted_job], dest=tmp_path) - ... -``` - -This symmetry is deliberate: the integration suite is the executable form of the contract. If a -capability check works in the test harness, it works for DiracX; if a backend's structural typing -lies (the `runtime_checkable` presence-only problem from IC-ADR-001), the integration suite is -where the lie is caught against a real daemon. Backend-specific quirks (e.g. the HTCondor -one-shot retrieval semantics) get dedicated marker-selected tests rather than conditionals inside -generic ones. - -Markers (`remote`, `scheduler`, `arc`, `htcondor`, `slurm`, `ssh`, `destructive_fetch`) select -the applicable subset per stack via the manifest's `markers` expression. - -### 3. Prebuilt stack images on GHCR - -ARC and Slurm images build from RPMs; rebuilding them on every PR wastes 5–10 minutes per job. A -separate `build-images` workflow builds and pushes -`ghcr.io/diracgrid/intercede-testenv/{arc,slurm,htcondor-ce}:` whenever the Dockerfiles -change (including when Renovate bumps a pinned `ARG`), plus a weekly rebuild for base-image -security updates. PR CI **pulls** by digest. - -Daemons run in the foreground under a minimal supervisor (or a plain entrypoint script), **never -systemd** — privileged systemd containers are fragile on GitHub runners and hide daemon logs from -`docker logs`. - -### 4. Ephemeral credentials, generated per run - -No credential is ever committed: - -- **ARC stacks**: the container entrypoint runs `arcctl test-ca init` etc., generating the CA, a host - certificate, and a client credential; client credentials are written to a shared - `credentials` volume that the test harness reads. Token auth is added later as a second - configuration (`config/token/`), not baked into `basic`. -- **HTCondor / HTCondor-CE**: `IDTOKENS` auth; the CE entrypoint mints a token into the shared - volume. No GSI, no VOMS in `basic`. -- **SSH stacks**: `ssh-keygen` in a job step, public key mounted into the container's - `authorized_keys`, known-hosts pinned from `ssh-keyscan`. - -The shared credentials volume *is* the interface between stack and harness; its layout -(`credentials//…`) is part of the stack contract. - -### 5. Manifest-driven GitHub Actions matrix - -``` -.github/workflows/integration.yml -├── job: matrix — reads stacks.yml, emits JSON via fromJSON() -└── job: integration — matrix: {stack × config × version}, fail-fast: false - ├── docker compose up --wait (healthchecks gate readiness; version → image tag) - ├── pixi run pytest tests/integration -m "" --stack= --config= - │ (or `docker compose exec` for exec_in_container stacks) - └── on failure: `docker compose logs` + backend log dirs uploaded as artifacts -``` - -- Each stack runs in its **own job**: parallel wall-clock, isolated failure blast radius, and each - stays comfortably inside a 4 vCPU runner. There is no combined all-backends job. -- Healthchecks are mandatory in every compose service: ARC (`curl -k` the REST endpoint's info - URL), HTCondor (`condor_status -limit 1` / `condor_ce_status`), Slurm (`sinfo` reporting the - node up), sshd (TCP probe). `up --wait` then replaces every hand-rolled sleep-and-retry loop. -- The integration workflow is `workflow_call`-reusable, invoked from `ci.yml` after unit tests, - and required for merge. Log artifacts on failure are non-negotiable: a red integration job - without daemon logs is a re-run generator, not a signal. - -### 6. Renovate as the version radar — two lanes - -**Lane 1 — pinned, blocking (PRs).** Everything CI runs against is pinned: - -- image tags in `compose.yml` files → `docker-compose` manager (enable in `renovate.json`); -- base images and pinned package versions in Dockerfiles → `dockerfile` manager plus - `customManagers` regex on annotated `ARG`s: - - ```dockerfile - # renovate: datasource=repology depName=fedora_epel_9/nordugrid-arc - ARG ARC_VERSION=7.1.1 - ``` - -- Python dependencies → existing pep621 manager, unchanged. - -For a backend with a support window (§8), Renovate tracks **only the leading-edge ARG**; the older -anchor ARGs are pinned and constrained (`packageRule` `allowedVersions`/`matchCurrentVersion`, or -`enabled: false`) so they accept only patch bumps within their major and never jump it. - -A Renovate bump PR (e.g. HTCondor 25.7 → 25.8) triggers image rebuild + full integration matrix. -**A red Renovate PR is the feature**: it is the earliest, cheapest, fully-reproducible signal -that an upstream release broke an interCEde contract — the 25.8 destructive-fetch change would -have surfaced as exactly this. Backend bumps are grouped per backend (`arc`, `htcondor`, -`slurm`), never mixed, so a red PR names its culprit. - -**Lane 2 — unpinned canary (scheduled, non-blocking).** A weekly scheduled workflow runs the same -matrix against `latest`/nightly backend tags. It cannot block merges; it exists to catch -upstream changes *before* they reach the stable tags Renovate tracks. Failures open a -deduplicated issue via workflow automation. - -### 7. Configuration growth path - -The full matrix is `(stack × config × version)`, starting as `(6 × basic × leading-edge)`. Planned -configuration axes, added as `config//` directories and manifest entries — each an additive -change: - -- auth variants (token vs. proxy for ARC; token lifetimes for HTCondor-CE); -- filesystem variants (shared vs. non-shared session directories — the `stages_own_files` axis); -- queue topology (multiple queues, per-queue limits); -- resource constraints (memory/CPU limits surfacing scheduler translation bugs); -- a batch-host variant with **no usable Python interpreter**, asserting IC-ADR-001 §3's - no-interpreter-on-the-remote-host property (DIRAC's SSH CE shipped a Python driver to the host; - interCEde must never need one). - -PR CI runs `basic` only; the scheduled workflow runs the full configuration set. This keeps PR -latency flat as the configuration matrix grows. - -### 8. Backend version support window - -interCEde is a *client* that must interoperate with backend daemons whose version it does not -control. Two kinds of "version" therefore live in this repo and must not be conflated: - -- **Versions interCEde ships** — Python dependencies, base images. We own these; "bump to latest" - is correct and Renovate does exactly that (§6). -- **Backend server versions interCEde interoperates with** — ARC, HTCondor(-CE), Slurm daemons. - Here a pinned version is a *test fixture standing in for a version some site runs*, not a - dependency we upgrade. Real sites run a spread — HTCondor LTS years after release, older ARC 7 - where the latest hasn't rolled out — so testing only the newest daemon answers the wrong - question. - -Each backend therefore declares a **support window**: the set of upstream versions interCEde -claims to work against, expressed as an *oldest/LTS anchor* plus the *leading edge*. Version is a -matrix axis alongside `config` (the `versions:` key in `stacks.yml`, §1). - -- **Renovate tracks only the leading edge.** The leading-edge `_VERSION` ARG bumps - normally — this is Lane 1's radar unchanged: a new upstream release arrives as a red-or-green PR - that names its culprit. -- **Older anchors are human-managed.** Each anchor ARG is pinned and constrained (§6) so Renovate - offers only patch bumps within the anchor's major. **Moving the window — dropping an old major, - adopting a new one — is a deliberate human PR**, because dropping support for a version sites - still run is a decision, not an auto-merge. -- **PR latency stays flat.** As with `config`, PR CI runs a representative subset (leading edge, - plus one anchor for the backend most exposed to version skew); the scheduled workflow runs the - full version cross-product. - -Multi-version testing is also what *justifies* version-conditional client code: the HTCondor -≥ 25.8 one-shot-retrieval shim is exactly such a case, and the window is where it earns its -regression coverage. - -**ARC is supported from version 7 only.** ARC 6 (pre-REST era: GridFTP/EMI-ES data staging, LDAP -infosystem) and ARC 7 (REST-first, `arcctl`) are close to different backends, and interCEde's AREX -path is REST-native. ARC 6 is explicitly out of scope, so the ARC window's floor is 7.x, with no -v6 anchor. - -## Rationale - -The design follows directly from the drivers above: real daemons over fakes (to catch what -structural typing cannot), per-stack isolation over a shared environment (attributable failures -inside a small runner), pinned-plus-Renovate over unpinned tracking (every version change attached -to a reviewable PR), and manifest-plus-files growth over workflow surgery (so the configuration -matrix scales without CI edits). The resulting benefits and the trade-offs deliberately accepted -in exchange: - -**Benefits** - -- Contract violations against real daemons are caught at PR time, per backend, with logs. -- Upstream version compatibility becomes a reviewed, bisectable git history of Renovate PRs - instead of tribal knowledge. -- Stacks double as reproducible local development environments — `compose up`, point your client - config at localhost, develop against a real ARC. -- Adding backends (future: Cloud via a mock EC2/OpenStack endpoint) or configurations is - manifest-plus-files, no CI surgery. -- The destructive-fetch semantics, the sorest point of IC-ADR-001, get a permanent regression test - (`fetch twice, assert contract`) on every backend. - -**Trade-offs and accepted costs** - -- We own three Dockerfiles for daemons whose packaging we don't control; EL9 repo layout changes - will occasionally break image builds (contained to the `build-images` workflow). -- Integration jobs add ~5–8 minutes wall-clock per PR (parallel across stacks). Accepted; unit - tests remain the fast inner loop. -- Single-node Slurm/HTCondor pools do not exercise multi-node scheduling behaviour. Out of scope: - interCEde's contract ends at the CE/scheduler interface; scheduling fidelity beyond it is the - backend's problem. -- The version support window (§8) multiplies images and CI jobs per backend. Contained by running - only the leading edge (plus one anchor) on PRs and the full cross-product on the schedule, and by - keeping the window to a small anchor set rather than every historical release. -- The canary lane will produce noise when upstream nightlies are broken through no fault of ours. - Mitigated by non-blocking status and issue deduplication. -- GitHub-hosted runners only (linux/amd64). No arm64, no macOS backend testing. Revisit only if a - consumer materializes there. - -## Rejected Ideas - -**Kubernetes (kind/k3s) as the orchestration layer.** Buys multi-node realism and Helm-chart -reuse (diracx-charts) at the price of cluster boot time, YAML volume, and debuggability inside a -CI job. Nothing in the CE contract requires more than "daemon reachable on a port"; compose -delivers that in one file per stack. Revisit if interCEde ever tests in-cluster deployment -concerns, which today it explicitly doesn't have. - -**testcontainers-python / per-test containers.** Excellent for a Postgres; wrong for daemons with -30–60 s startup (ARC, slurmctld+munge). Session-scoped stacks amortize startup once per job. -Per-test isolation is recovered logically (unique job tags/working dirs per test), not by -container churn. - -**One mega-compose with every backend.** Single job, shared fate: one flaky daemon reds the -world, resource ceilings are shared, and log spelunking spans six daemons. Parallel per-stack -jobs are faster and attribute failures for free. - -**GitHub Actions `services:`.** No compose semantics, no build, no startup ordering beyond -health of individual containers, awkward log retrieval. Fine for a Redis sidecar; not for a CE -and its LRMS. - -**Installing backends directly on the runner (apt/yum in the job).** Fast to prototype, -impossible to pin properly (Ubuntu's Slurm is whatever the distro ships), nothing is reusable -locally, and ARC on Ubuntu is not a supported combination. Containers or nothing. - -**Reusing DIRAC's certification/integration environment.** It exists, but it drags in the full -DIRAC server stack and its configuration system — the exact coupling interCEde was extracted to -escape. interCEde must be testable by someone who has never installed DIRAC. - -**Tracking upstream only via the canary (no pinning).** Green-today-red-tomorrow CI with no -diff to point at. Pinning plus Renovate keeps every version change attached to a reviewable PR; -the canary is an early-warning supplement, not the mechanism of record. - -**Nightly-only integration tests (nothing on PR).** Decouples breakage from the change that -caused it and lets contract violations merge. The whole value proposition is the red X *on the -PR that introduces the problem* — including Renovate's version-bump PRs. - -## Open Issues - -- Exact Renovate datasource for EL9 RPM pins (`repology` vs. a custom endpoint for the NorduGrid - repo) — to be settled when the ARC Dockerfile lands. -- Whether `htcondor` stack `basic` uses `htcondor/mini` behind the CE or a cm/execute/submit - trio; `mini` is preferred until a test needs role separation. -- Whether Cloud backend testing (mock OpenStack/EC2) becomes a seventh stack or stays unit-level; - deferred until the Cloud backend is scheduled. diff --git a/docs/adr/IC-ADR-003_credentials.md b/docs/adr/IC-ADR-003_credentials.md deleted file mode 100644 index 624d377..0000000 --- a/docs/adr/IC-ADR-003_credentials.md +++ /dev/null @@ -1,301 +0,0 @@ -# IC-ADR-003: Backend credentials — typed, declared, provider-supplied - -## Metadata - -- **Created By:** Alexandre Boyer -- **Date:** 2026-07-03 -- **Status:** Draft -- **Decision Maker(s):** Federico Stagni, Christophe Haen, Chris Burr -- **Stakeholders:** interCEde backend implementers; DIRAC/DiracX SiteDirector and PushJobAgent maintainers; DiracX token-issuance maintainers; site/VO operators -- **Depends on:** IC-ADR-001 (contract shape, lifecycle, Tier A/B/C surface) - -> **Scope and altitude.** Direction-setting, like IC-ADR-001. This ADR decides how *backend* -> authentication credentials — bearer tokens and X.509 proxies — are **typed, declared, supplied, -> refreshed, scoped and materialised**. It does not decide token *issuance* (DiracX's), *payload* -> credential renewal (pilot-side, DIRAC's `_monitorProxy` lineage), or the exact dataclass fields -> (implementation). Long-lived transport secrets (SSH keys) are configuration, not credentials, and -> are only delimited here. - -## Abstract - -Backend auth is on the critical path for the ARC and HTCondor-CE backends, and DIRAC's incumbent -model does not survive the move to a stateless, cached, async library: credentials are **mutated -onto** long-lived CE objects (`setProxy`/`setToken`), freshness policy is **duplicated across -callers**, token opt-in is a **stringly-typed CS tag**, and audience is a bare attribute. This ADR -replaces that with three pieces: **typed, immutable credential values** (`BearerToken`, -`X509Proxy`, grouped in a `CredentialSet`); **backend-declared `CredentialRequirements`** (which -kinds it accepts, for which audience/scopes — data computed from the backend's resolved config, -replacing `Tag: Token[:vo]` and `audienceName`); and **provider-based supply** — backends pull a -fresh `CredentialSet` from a caller-supplied `CredentialProvider` when theirs is missing or near -expiry, so freshness *mechanics* live in one place while issuance/renewal *policy* stays entirely -in the consumer (DiracX token machinery, DIRAC's proxy manager, or a static file). Materialisation -(token/proxy files, env injection for CLI backends) is internal (Tier C). ARC's proxy delegation -remains backend-internal mechanics consuming a provider-supplied proxy. - -## Motivation - -What the DIRAC code actually does today (all verified against the current codebase): - -1. **Mutating setters on cached objects.** `ComputingElement.setProxy`/`setToken` mutate CE - instances that `QueueCECache` caches and reuses across agent cycles. Credential state and - connection state are entangled on the same long-lived object — exactly the hazard for a library - whose backends are cached and context-managed (IC-ADR-001 §2, lifecycle). -2. **Caller-side renewal policy, duplicated.** `SiteDirector._setCredentials` inspects - `ce.proxy.getRemainingSecs()` and re-supplies; `WMSUtilities.setPilotCredentials` reimplements - the same gate for the pilot-kill/log service path. Two copies of the same freshness loop is the - documented drift pattern IC-ADR-001 exists to end. -3. **Stringly-typed opt-in.** A CE accepts tokens iff `"Token"` or `f"Token:{vo}"` appears in its - CS `Tag` list. A *capability declaration* is encoded in a free-form tag shared with scheduling - metadata. -4. **Audience as a bare attribute.** Callers mint tokens with `audience=ce.audienceName` — an - untyped per-CE string (AREX: `https://:`; HTCondorCE: `:9619`) with no - accompanying scopes/kind information. -5. **Per-backend materialisation, ad hoc.** HTCondorCE writes the token to a temp file and injects - `_CONDOR_*` env vars around each CLI call; the base writes proxies to files and exports - `X509_USER_PROXY`; Cloud reads a `cloud.auth` ini with a magic `PROXY` secret that dumps the - pilot proxy. -6. **ARC needs more than a header.** AREX supports bearer tokens *and* X.509 proxy delegations - (create a delegation via CSR, sign with the proxy chain, upload, renew) — and can need **both at - once** (`AlwaysIncludeProxy`). -7. **HTCondorCE needs both at once — today, in production.** Even when SCITOKENS authenticates the - channel, DIRAC's submit description *unconditionally* contains `use_x509userproxy = true`, and - the code says why: *"For now, we still need to include a proxy in the submit file — HTCondor - extracts VOMS attribute from it for the sites"* — the proxy rides along for **site-side - consumption** (per-VO attribution in site accounting, i.e. the APEL pipeline), not for channel - auth. `_executeCondorCommand` even refuses to run with neither token nor proxy, and token mode - sets `_CONDOR_DELEGATE_JOB_GSI_CREDENTIALS=false` to work around a condor 24.4 delegation bug - (HTCONDOR-2904). Together with ARC's pairing, any model that assumes "one credential per - backend" is wrong on day one — twice. - -### Drivers - -- **Stateless and cache-safe.** Credentials must not be hidden mutable state on cached backends. -- **Policy with the consumer.** Where credentials *come from* (DiracX token service, proxy - download, a file) and *when they are renewed* is consumer policy; interCEde owns only the - mechanics of asking at the right time and using them correctly. -- **Typed declaration.** A consumer must be able to ask a backend "what do you need?" and get data - — not parse tags. -- **One supply path for one and many credentials.** Token-only, proxy-only, and token+proxy - backends use the same machinery. -- **Testable in containers.** The model must work with IC-ADR-002's ephemeral per-run credentials - (arcctl test-CA, IDTOKENS, ssh-keygen). - -## Specification - -### 1. Credential values (Tier A) - -Immutable, typed values with an expiry the machinery can read: - -```python -@dataclass(frozen=True) -class BearerToken: - value: str # opaque to interCEde; never logged - expires_at: datetime | None - -@dataclass(frozen=True) -class X509Proxy: - pem: bytes # full chain, PEM; opaque to interCEde beyond expiry - expires_at: datetime | None - -Credential = BearerToken | X509Proxy - -@dataclass(frozen=True) -class CredentialSet: # what a provider returns; may satisfy >1 kind - def get(self, kind: type[Credential]) -> Credential | None: ... -``` - -`CredentialSet` exists because of ARC's token+proxy case: requirements may name several kinds, and -one provider call returns everything needed, atomically. interCEde never inspects credential -*contents* (no VOMS parsing, no JWT decoding) beyond expiry bookkeeping. - -### 2. Requirements — the backend declares, as data (Tier A) - -```python -@dataclass(frozen=True) -class CredentialRequirements: - kinds: frozenset[type[Credential]] # the kinds needed TOGETHER (all-of, not a menu) - audience: str | None = None # token audience, if kinds include BearerToken - scopes: frozenset[str] = frozenset() # e.g. compute.create/compute.read (WLCG profile) -``` - -A backend computes its `CredentialRequirements` from its resolved configuration (it knows its -endpoint, hence its audience) and exposes them as a read-only property. This single piece of data -replaces both the `Tag: Token[:vo]` opt-in *and* `audienceName`: the consumer reads the -requirements and mints/fetches accordingly. Configuration may still *narrow* a backend that -accepts several kinds ("this site: proxy only"); it can never widen beyond what the backend -declares. - -Two semantic rules, both forced by the grid CEs: - -- **`kinds` means "all of these, together".** Requirements are a concrete ask for the backend's - *active configuration*, never a menu of alternatives — both first-party grid CEs need a *set* - (ARC `AlwaysIncludeProxy`; HTCondorCE token + proxy, Motivation 7). A backend that supports - alternative modes (token-only *or* proxy-only) exposes the mode as configuration, and its - requirements reflect the configured mode. -- **A required kind need not authenticate the channel.** HTCondorCE's proxy is materialised into - the submit description (`use_x509userproxy`) for *site-side* consumption — VOMS attributes - feeding the site's accounting (APEL) — while the token authenticates the channel. The provider - sees no difference (it supplies the set); what each member is *for* is backend mechanics - (Tier C). - -### 3. Supply — a provider the backend pulls from (Tier A) - -```python -@runtime_checkable -class CredentialProvider(Protocol): - async def get(self, requirements: CredentialRequirements) -> CredentialSet: ... -``` - -- The provider is part of the backend's construction config (registry request). A trivial - `StaticCredentials(CredentialSet)` provider covers files/fixed tokens — and the integration - stacks. -- **The backend pulls; the consumer implements.** Before an operation, if the backend's current - set is missing or within a freshness margin of expiry, it awaits `provider.get(...)` — once, in - one place, inside the library. What the provider *does* (call DiracX's token issuer, download a - proxy, read a refreshed file) is consumer code. This keeps the mechanism/policy split of - IC-ADR-001 §9 and deletes the SiteDirector/WMSUtilities duplication by construction. -- Provider calls are per-backend, not per-job: one credential authenticates the *channel/CE*, not - each submission. -- Failures raise the typed auth branch of `InterCEdeError` as whole-operation failures (IC-ADR-001 - §2 partial-failure rule: auth failure is never a per-id outcome). - -### 4. Materialisation (Tier C) - -Backends that drive CLIs or need files get internal helpers, never a public contract: secure temp -files (0600, private dir), env injection (`X509_USER_PROXY`, HTCondor `_CONDOR_*`/SciTokens env), -scoped to the operation and cleaned up deterministically (the backend's context-manager lifecycle -from IC-ADR-001 §2 is the natural cleanup boundary). Credential values never appear in logs or -exception messages. - -### 5. Delegation is backend mechanics (Tier C) - -ARC proxy delegation (CSR → sign with the provider-supplied `X509Proxy` → upload; renew before -expiry; reuse across submissions where valid) is `ARCBackend`-internal. The provider supplies the -proxy; everything downstream — delegation ids, renewal, `AlwaysIncludeProxy`-style pairing with a -token — is invisible to the consumer. - -### 6. Boundaries - -- **SSH keys are transport configuration**, not rotating credentials: long-lived, file/agent-based, - no audience. They stay in `SSHTransport` config; revisit only if key rotation becomes a real - requirement. -- **Payload credentials are pilot-side.** Renewing the proxy *of the running payload* (DIRAC's - `_monitorProxy`, `GENERIC_PILOT` branch) belongs to the severed pilot/runner domain, not here. -- **Issuance is the consumer's.** interCEde never holds refresh tokens, client secrets, or CA - material; it asks a provider and uses what it gets. -- **Cloud provider auth** (Libcloud driver keys, the `cloud.auth` ini) is `CloudBackend` internal - config; whether it adopts the provider model is deferred with the Cloud backend itself. - -### 7. Usage sketches (informative) - -**(a) The consumer implements the provider once; every backend pulls from it.** A DiracX -submission task wires its token/proxy machinery into one object and never polices freshness -again (this is the code that today exists twice, in `SiteDirector._setCredentials` and -`WMSUtilities.setPilotCredentials`): - -```python -class DiracXPilotCredentials: # consumer-side; satisfies CredentialProvider - async def get(self, req: CredentialRequirements) -> CredentialSet: - creds = [] - if BearerToken in req.kinds: # mint scoped to what the backend declared - tok = await token_service.mint(audience=req.audience, scopes=req.scopes) - creds.append(BearerToken(tok.value, tok.expires_at)) - if X509Proxy in req.kinds: # gProxyManager lineage, behind the provider - pem = await proxy_store.download(pilot_dn, pilot_group, lifetime=86400) - creds.append(X509Proxy(pem, expires_at=proxy_expiry(pem))) - return CredentialSet(creds) - -backend = registry.backend( - {"type": "htcondor-ce", "endpoint": "ce01.example.org:9619", ...}, - credentials=DiracXPilotCredentials(), -) -async with backend: # lifecycle from IC-ADR-001 §2 - sub = await backend.submit(spec, count=50) - # before the operation the backend compared its cached CredentialSet against - # backend.credential_requirements and awaited provider.get(...) only if stale -``` - -**(b) What the HTCondorCE backend declares, and what it does with the set (Tier C).** The -requirements make the Motivation-7 pairing explicit and typed; the materialisation reproduces -DIRAC's mechanics without the caller knowing any of it: - -```python -backend.credential_requirements == CredentialRequirements( - kinds=frozenset({BearerToken, X509Proxy}), # all-of: token for the channel, - audience="ce01.example.org:9619", # proxy for site-side VOMS/APEL accounting -) -# internally, per operation: -# BearerToken -> 0600 temp file; _CONDOR_SEC_CLIENT_AUTHENTICATION_METHODS=SCITOKENS, -# _CONDOR_SCITOKENS_FILE= -# X509Proxy -> 0600 temp file; X509_USER_PROXY=, referenced by the submit -# description's `use_x509userproxy = true` -# both cleaned up at the operation/lifecycle boundary; values never logged -``` - -**(c) Integration stacks and standalone use — static files, no issuance machinery.** The -IC-ADR-002 stacks mint ephemeral credentials into the shared volume; the test harness (or any -non-DiracX user) wraps them statically: - -```python -provider = StaticCredentials(CredentialSet([ - BearerToken(Path("credentials/htcondor/idtoken").read_text().strip(), expires_at=None), -])) -backend = registry.backend({"type": "htcondor-ce", ...}, credentials=provider) -``` - -## Rationale - -- **Provider-pull over mutating setters.** Setters put hidden state on cached objects and force - every caller to police freshness (two DIRAC copies prove the cost). A pull model puts the - *check* in one library-side place while leaving the *source and policy* in consumer code — the - same mechanism/policy line IC-ADR-001 draws for throttling. -- **Declared requirements over tag sniffing.** `Tag: Token` conflates scheduling metadata with an - auth capability and is invisible to type checkers and tooling. A typed declaration is - enumerable, testable, and lets the conformance suite assert that a backend's declared kinds are - the ones it actually uses. -- **`CredentialSet` over single credential.** ARC's token+proxy pairing is a first-party - requirement, not an edge case; modelling it from day one avoids a v2 of the provider protocol. -- **Immutability.** Frozen values make "refresh" a *replacement*, never an in-place mutation — - cache-safe and race-free under concurrent bulk operations. - -## Rejected Ideas - -- **DIRAC-style `set_credential()` mutators.** Hidden state on cached backends; freshness policy - smeared across callers; the incumbent model this ADR exists to replace. -- **Backend-driven issuance/refresh** (backend holds a refresh token or talks to the IdP). - Couples the library to DiracX/IdP specifics, embeds secrets in the library, and moves policy - inside — the opposite of the IC-ADR-001 §9 mechanism/policy split. -- **Credentials in the `SubmissionSpec`** (name provisional — IC-ADR-001 Open Issues). Auth is - per-backend/channel, not per-job; putting it on - the spec would force every task (status, fetch, kill) to re-thread it and would leak payload vs - backend credential confusion back in. -- **A single `Credential` per backend (no set).** Breaks on ARC's token+proxy pairing. -- **Keeping the `Tag: Token[:vo]` opt-in.** Stringly-typed, CS-coupled, invisible to types and - tooling. -- **An interCEde-owned credential store/daemon.** interCEde is stateless (IC-ADR-001 §1); caching - beyond the in-memory current set is the consumer's business. - -## Open Issues - -- **Freshness margin.** Fixed library default vs per-backend/per-provider configuration; ARC - delegation renewal wants a larger margin than a bearer-token header. -- **Requirements granularity.** Whether `CredentialRequirements` stays per-backend or needs - per-operation variance (e.g. a backend whose *fetch* endpoint needs a different scope than - *submit*). Start per-backend; split only on evidence. -- **Proxy representation.** Raw PEM bytes (current spec) vs a `cryptography` object; and whether - delegation needs key material the consumer must supply alongside the chain (DIRAC signs the - delegation CSR with the proxy's own key — implies the provider hands over key+chain, which - `X509Proxy.pem` as "full chain" must be explicit about). -- **Multi-VO backends.** One backend instance serving several VOs would need per-VO credential - sets; today DIRAC instantiates per-queue/VO CEs, and interCEde's per-backend provider assumes - the same. Confirm with DiracX's multi-VO design. -- **DiracX alignment.** The provider implementation on the DiracX side (token service, scopes, - pilot-credential flows) — tracked with the DiracX transition ADRs, not here. -- **Proxy-for-accounting lifetime.** HTCondorCE's proxy requirement is a WLCG-transition artifact - (sites attribute usage via the proxy's VOMS attributes, the APEL pipeline). When sites account - on token claims instead, the backend's declared requirements shrink to token-only — and because - requirements are *data*, that is a backend-version/configuration change, not an API break. - Track the WLCG token-transition timeline before hard-coding the pairing as permanent. -- **Conformance coverage.** IC-ADR-002 already plans token-vs-proxy configuration axes for the - ARC stack; extend the conformance suite with a "requirements honesty" check (backend declares X, - suite verifies it authenticates with exactly X). diff --git a/docs/adr/IC-ADR-XXX_template.md b/docs/adr/IC-ADR-XXX_template.md deleted file mode 100644 index fa37760..0000000 --- a/docs/adr/IC-ADR-XXX_template.md +++ /dev/null @@ -1,45 +0,0 @@ -# IC-ADR-\[NUMBER\]: [Title of Decision] - -## Metadata - -- **Created By:** [Name] -- **Date:** [YYYY-MM-DD] -- **Status:** [Draft | Accepted | Rejected | Deprecated by IC-ADR-YYY | Supersedes IC-ADR-XXX] -- **Decision Maker(s):** [Name(s)] -- **Stakeholders:** [Name(s) / Role(s), only used for decisions which affect a subset of communities] -- **Depends on:** [IC-ADR-YYY — optional; upstream ADRs this decision builds on] - -> **Scope and altitude.** [Optional but recommended, especially for long ADRs. Two to four -> sentences: what kind of decision this is (direction-setting vs detailed), what is deliberately -> deferred and to where, and which sections a reviewer must read versus may skim.] - -## Abstract - -A short (~200 word) summary of the decision being made and why it matters. Prefer one -plain-language sentence per decision over a dense paragraph — this is the only part many -stakeholders will read. - -## Motivation - -Why is this decision needed now? What problem or limitation in the current system does it address? What are the functional and non-functional drivers? - -## Specification - -Describe the chosen solution in concrete detail — APIs, interfaces, configuration, behaviour. This is the "what we're building" section. - -## Rationale - -Explain *why* the chosen design looks the way it does. Why these trade-offs? Why this level of abstraction? Connect specific design choices back to the drivers in Motivation. - -## Evolution - -Optional. How the decision accommodates future change: what can be added without breaking -(additive extensions), and what would require superseding this ADR. - -## Rejected Ideas - -Why were the non-chosen options ultimately set aside? This is distinct from the pros/cons listing above — it's the narrative of what tipped the scales. Include any ideas that came up in discussion but weren't even promoted to full options, and why. - -## Open Issues - -Any points still being decided or discussed. Remove this section once the status moves to Accepted. diff --git a/docs/adr/index.md b/docs/adr/index.md deleted file mode 100644 index 2a53f1b..0000000 --- a/docs/adr/index.md +++ /dev/null @@ -1,13 +0,0 @@ -# Architecture Decision Records - -Architecture Decision Records (ADRs) document significant design decisions made in the interCEde project. - -| ADR | Title | Status | -| --- | ----- | ------ | -| [IC-ADR-001](IC-ADR-001_computing_elements.md) | Computing Element interfaces and the DIRAC to interCEde migration | Draft | -| [IC-ADR-002](IC-ADR-002_integration_tests.md) | Integration testing against containerized backends | Draft | -| [IC-ADR-003](IC-ADR-003_credentials.md) | Backend credentials — typed, declared, provider-supplied | Draft | - -## Template - -New ADRs should follow the [ADR template](IC-ADR-XXX_template.md). diff --git a/docs/dev/explanations/integration-testing.md b/docs/dev/explanations/integration-testing.md index f8210de..8073c7e 100644 --- a/docs/dev/explanations/integration-testing.md +++ b/docs/dev/explanations/integration-testing.md @@ -1,7 +1,6 @@ # Why integration stacks — testing against real backend daemons -The full reasoning is [IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md); this page is the -short version plus the phasing plan. +Why the integration suite runs against real backend daemons, and how it is phased. ## Why real daemons diff --git a/docs/dev/how-to/run-integration-tests-locally.md b/docs/dev/how-to/run-integration-tests-locally.md index 9225a9b..ab099b3 100644 --- a/docs/dev/how-to/run-integration-tests-locally.md +++ b/docs/dev/how-to/run-integration-tests-locally.md @@ -1,8 +1,6 @@ # Run the integration tests locally -Everything CI does is reproducible on a laptop — a design rule of -[IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md): CI does nothing a developer cannot do -locally. +Everything CI does is reproducible on a laptop: CI does nothing a developer cannot do locally. ## Run one stack diff --git a/docs/dev/index.md b/docs/dev/index.md index ae92204..6d1705e 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -7,5 +7,5 @@ Organised following [Diátaxis](https://diataxis.fr/), as in the other DiracX-fa - [Reference](reference/index.md) — information-oriented technical description - [Explanations](explanations/index.md) — understanding-oriented discussion -Architecture decisions are recorded separately in the [ADRs](../adr/index.md). User-facing -documentation (`docs/user/`) is deferred until the library has a public API to document. +User-facing documentation (`docs/user/`) is deferred until the library has a public API to +document. diff --git a/docs/dev/reference/integration-images.md b/docs/dev/reference/integration-images.md index 8082fe4..366d235 100644 --- a/docs/dev/reference/integration-images.md +++ b/docs/dev/reference/integration-images.md @@ -1,11 +1,11 @@ # Integration images — Dockerfile conventions Dockerfiles under `tests/integration/stacks/_images/` are shared across stacks (`arc/`, `slurm/`, -`htcondor-ce/`, …) and follow [IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md) §3 and §6: +`htcondor-ce/`, …) and follow these conventions: 1. **EL9 base** (`almalinux:9`), pinned by tag; backend daemons installed from **pinned RPMs**. 2. **Every version is a Renovate-annotated `ARG`** so upstream releases arrive as red-or-green - Renovate PRs (Lane 1): + Renovate PRs: ```dockerfile FROM almalinux:9 @@ -22,7 +22,7 @@ Dockerfiles under `tests/integration/stacks/_images/` are shared across stacks ( [`renovate.json`](../../../renovate.json) (per-backend grouping and the leading-edge/anchor convention are documented in [the Renovate reference](renovate.md)) — **adding a stack needs no `renovate.json` edit**, only the annotated `ARG`. The annotation carries its own `datasource`, - so the exact datasource for the NorduGrid/EL9 repos (an open point of IC-ADR-002) is chosen when + so the exact datasource for the NorduGrid/EL9 repos (still open) is chosen when the ARC Dockerfile lands, without touching the manager config. 3. **Foreground entrypoint** — daemons run under a plain entrypoint script or minimal supervisor, never systemd, and **log to stdout/stderr** so `docker compose logs` captures everything (a @@ -31,10 +31,10 @@ Dockerfiles under `tests/integration/stacks/_images/` are shared across stacks ( 4. **Ephemeral credentials at start**: the entrypoint mints whatever the stack needs (ARC: `arcctl test-ca` CA + host cert + client credential; HTCondor(-CE): IDTOKENS) and writes client-side material world-readable into the host-readable `./credentials` bind mount - (`credentials//…`, IC-ADR-002 §4). A TLS host certificate carries a SAN for the name the + (`credentials//…`). A TLS host certificate carries a SAN for the name the client dials (the compose hostname *and* `localhost` for host-run stacks); ARC on EL9 needs the crypto-policy set to `LEGACY` for the proxy chain. -5. **Version windows**: where a backend declares an anchor + leading edge (IC-ADR-002 §8), the +5. **Version windows**: where a backend declares an anchor + leading edge, the image accepts the version as `ARG`/build-arg so the same Dockerfile serves every entry of the `versions:` axis; only the leading-edge `ARG` default is Renovate-tracked. 6. **Roles and intra-stack secrets** (multi-service stacks). A CE + LRMS or a Slurm diff --git a/docs/dev/reference/integration-stacks.md b/docs/dev/reference/integration-stacks.md index d1cebca..b164212 100644 --- a/docs/dev/reference/integration-stacks.md +++ b/docs/dev/reference/integration-stacks.md @@ -1,7 +1,6 @@ # Integration stacks — layout, manifest schema, stack contract -Reference for the integration-test infrastructure decided in -[IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md). +Reference for the integration-test infrastructure. ## Layout @@ -28,7 +27,7 @@ tests/integration/ | `id` | yes | Stack name; also the directory under `stacks/` unless `stack:` is set | | `stack` | no | Alias: reuse another stack's compose environment (e.g. `local-slurm` → `ssh-slurm`) | | `configs` | yes | Configuration names under `stacks//config/`; start with `[basic]` | -| `versions` | no | Backend support-window entries to run (IC-ADR-002 §8); absent = single leading-edge pin | +| `versions` | no | Backend support-window entries to run; absent = single leading-edge pin | | `markers` | yes | `pytest -m` expression selecting the applicable contract-suite subset | | `exec_in_container` | no | Run pytest inside the backend container instead of against it (default `false`) | @@ -59,21 +58,21 @@ A stack that lands in `stacks//` commits to all of the following (enforced p 2. **Healthchecks on every service.** `up --wait` gates on them; hand-rolled sleep-and-retry loops are not accepted. 3. **Foreground daemons, never systemd.** A plain entrypoint or minimal supervisor — privileged - systemd containers are fragile on GitHub runners and hide daemon logs from `docker logs` - (IC-ADR-002 §3). Daemons **log to stdout/stderr** so `docker compose logs` is the whole failure - story; a file-logging daemon redirects its log to `/dev/stdout` rather than expecting a bespoke - log-directory upload (the CI failure artifact is compose logs only). + systemd containers are fragile on GitHub runners and hide daemon logs from `docker logs`. Daemons + **log to stdout/stderr** so `docker compose logs` is the whole failure story; a file-logging + daemon redirects its log to `/dev/stdout` rather than expecting a bespoke log-directory upload + (the CI failure artifact is compose logs only). 4. **Images from `stacks/_images/`,** EL9 base, pinned Renovate-annotated `ARG`s (see [image conventions](integration-images.md)). 5. **Ephemeral credentials, minted at container start** into the shared, host-readable `./credentials` bind mount (gitignored) — never committed, and never a Docker *named* volume (host pytest cannot read a root-owned named volume for `exec_in_container: false` stacks). The - layout `credentials//…` is the interface between stack and harness (IC-ADR-002 §4). Mint + layout `credentials//…` is the interface between stack and harness. Mint throwaway Test-CA material world-readable; the harness copies anything needing private modes (0400/0600) into a private dir itself. 6. **Configurations are atomic directories.** `config//` holds the backend config *and* `intercede-client.toml` together; a new auth/filesystem/queue variant is a new directory plus - a `configs:` entry in `stacks.yml`, never a mutation of `basic` (IC-ADR-002 §7). + a `configs:` entry in `stacks.yml`, never a mutation of `basic`. 7. **One `stacks.yml` entry.** That is how CI finds the stack — adding a stack requires no workflow edits. 8. **Host reachability + TLS naming** (host-run stacks). A stack the harness drives from the host @@ -102,6 +101,6 @@ the Phase-2 contract suite that calls it, alongside the interCEde protocols — not part of this scaffold, whose signature would otherwise be guessed ahead of its callers. A *selected* stack whose `intercede-client.toml` is missing **fails** under CI (`$CI` set) rather -than skipping: an all-skipped run would report a green integration cell that verified nothing -(IC-ADR-002 §5). Locally the same case skips, which stays convenient while a stack is being +than skipping: an all-skipped run would report a green integration cell that verified nothing. +Locally the same case skips, which stays convenient while a stack is being brought up. diff --git a/docs/dev/reference/renovate.md b/docs/dev/reference/renovate.md index 0b13613..d5e3192 100644 --- a/docs/dev/reference/renovate.md +++ b/docs/dev/reference/renovate.md @@ -24,9 +24,9 @@ The configuration is defined in [`renovate.json`](../../../renovate.json). ## Integration stack backends -The integration stacks ([IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md) §6) pin the backend -daemon versions they test against, and Renovate tracks those pins so a new upstream release arrives -as a reviewable red-or-green PR rather than tribal knowledge. Three managers are enabled for this: +The integration stacks pin the backend daemon versions they test against, and Renovate tracks those +pins so a new upstream release arrives as a reviewable red-or-green PR rather than tribal knowledge. +Three managers are enabled for this: - **`docker-compose`** — upstream image tags referenced directly in a stack's `compose.yml` (e.g. `htcondor/mini:`). @@ -44,7 +44,7 @@ as a reviewable red-or-green PR rather than tribal knowledge. Three managers are manager and per-backend grouping already configured here pick it up. Bumps are grouped per backend (`arc`, `htcondor`, `slurm`) and never mixed. -**Version windows (§8) are half-automatic.** Renovate tracks the *leading-edge* `ARG` normally. +**Version windows are half-automatic.** Renovate tracks the *leading-edge* `ARG` normally. The *older anchor* versions (e.g. an HTCondor LTS line) are human-managed: pin them and add a `packageRule` with `allowedVersions`/`matchCurrentVersion` (or `enabled: false`) — landed alongside the stack that introduces the anchor — so Renovate offers only in-major patch bumps and never diff --git a/docs/notes/reuse-survey.md b/docs/notes/reuse-survey.md deleted file mode 100644 index a7a9504..0000000 --- a/docs/notes/reuse-survey.md +++ /dev/null @@ -1,68 +0,0 @@ -# Prior-art survey: existing job-submission abstractions - -> Companion note to [IC-ADR-001](../adr/IC-ADR-001_computing_elements.md) (Rejected Ideas, -> *"Reuse an existing job-submission abstraction instead of defining our own"*). This records the -> survey evidence behind that rejection; the ADR keeps only the conclusion. Surveyed 2026-06/07 — -> per-tool facts here go stale independently of the decision. - -## What interCEde needs - -A modern async, typed Python library covering, together: SSH + local batch (Slurm, HTCondor, -LSF, …), ARC-CE over its REST interface, HTCondor-CE, and cloud (Libcloud-style providers). No -surveyed tool covers this set. - -## HTCondor GAHP / blahp - -Not a library but a *protocol* — no C/Python binding; you spawn a helper binary and speak ASCII -over stdin/stdout — and a *family* of binaries: `blahp` covers only **local** batch -(Slurm/PBS/LSF/SGE/Condor), while ARC and cloud are separate `arc_gahp`/`ec2_gahp`/`gce_gahp`/ -`azure_gahp`. blahp has **no SSH** — remote-cluster access is a `condor_gridmanager` + -`condor_remote_cluster`/BOSCO feature — so "SSH + Slurm" would drag in the whole HTCondor grid -stack. The protocol is internal, self-described as "SECOND DRAFT", with no back-compat guarantee. - -Apache-2.0. Its per-LRMS submit/status/cancel scripts and its async request-id/`RESULTS`-poll -model are worth *learning from* — they are cited in IC-ADR-001's Rationale as prior art for the -`Scheduler` protocol and the async, bulk, poll-based contract. - -The reverse direction — HTCondor reusing interCEde — is a non-goal: its gridmanager is C++ and -speaks a process protocol, and HTCondor-CE is a backend interCEde *targets*, so the layering runs -the other way. - -## ATLAS PanDA / Harvester - -Python, Apache-2.0, and has the exact submitter/monitor/sweeper plugin shape interCEde wants -(HTCondor, ARC via **aCT**, Slurm/LSF/PBS, cloud, k8s, HPC) — but its communicator talks only to -the PanDA server and its data model is PanDA specs, so it is not an importable standalone -library. Notably, ATLAS built Harvester *because* aCT was too tied to ARC-CE for US HPCs — the -same "generic beats CE-specific" lesson IC-ADR-001 applies. - -## ALICE JAliEn - -Its `alien.site.batchqueue` layer (ARC, HTCondor, direct Slurm/PBS, NERSC SuperFacility) is the -closest HEP analogue and does *both* grid-CE and direct-batch — but it is Java, never released as -an artifact (CVMFS-only), coupled to JAliEn's LDAP/central/token machinery, and of unconfirmed -licence. - -## Generic libraries - -RADICAL-SAGA, PSI/J, DRMAA, Parsl, AiiDA, Dask-Jobqueue are batch/HPC-scheduler oriented: the -ARC-CE REST **+** HTCondor-CE **+** cloud triple never co-occurs in any one of them. In fact -**none of the six speaks ARC at all**, in either the modern ARC-CE REST or the classic -ARC0/EMI-ES interface (RADICAL-SAGA never shipped a NorduGrid adaptor in any release — the Python -`radical.saga` is distinct from the old C++/Java SAGA; AiiDA has no `aiida-arc` plugin in core or -its registry). Where they touch "HTCondor" it is vanilla HTCondor/Condor-G, not the HTCondor-CE -grid gateway. - -ARC-CE REST is itself well-supported — HTCondor's `arc_gahp` speaks it in C++ (libcurl against -`/arex/rest/1.0`), and interCEde uses the Python `pyarcrest` — but only through ARC-specific -clients, never through one of these generic scheduler abstractions. - -Concurrency is *not* the differentiator: most of the six are async (futures/callbacks or a -non-blocking submit + poll/wait); DRMAA is the lone synchronous-style one, and it is unmaintained. - -## Conclusion - -Every HEP experiment builds its own CE abstraction and none is a reusable standalone library. The -genuinely reusable layer is the WLCG grid-CE ecosystem itself (HTCondor-CE, ARC-CE) plus -first-party per-backend clients (`pyarcrest`, the `htcondor` bindings, Apache Libcloud). -interCEde composes those behind a typed, async contract rather than adopting any monolith. diff --git a/pyproject.toml b/pyproject.toml index ca0b2fa..705b810 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ asyncio_mode = "auto" markers = [ "backend_only: restrict a test to the named backend fixture(s)", "xfail_backend: strict xfail a test on the named backend fixture(s)", - # Integration-suite markers (IC-ADR-002 §2): stacks.yml selects per-stack subsets with -m. + # Integration-suite markers: stacks.yml selects per-stack subsets with -m. "remote: drives a remote CE backend (grid gateway: ARC, HTCondor-CE)", "scheduler: drives a Transport x Scheduler composition (BatchBackend)", "arc: requires an ARC CE", diff --git a/renovate.json b/renovate.json index efbad4f..988ddc1 100644 --- a/renovate.json +++ b/renovate.json @@ -25,7 +25,7 @@ "customManagers": [ { "customType": "regex", - "description": "Track pinned backend versions in stack image Dockerfiles via `# renovate:`-annotated ARGs (IC-ADR-002 §6). The datasource/depName/versioning ride in the annotation, so a new stack image needs no renovate.json edit.", + "description": "Track pinned backend versions in stack image Dockerfiles via `# renovate:`-annotated ARGs. The datasource/depName/versioning ride in the annotation, so a new stack image needs no renovate.json edit.", "fileMatch": [ "(^|/)tests/integration/stacks/_images/.+/Dockerfile$" ], @@ -82,21 +82,21 @@ "labels": ["dependencies", "testing"] }, { - "description": "Group ARC backend bumps, never mixed with other backends so a red PR names its culprit (IC-ADR-002 §6). Renovate tracks the leading-edge ARG; anchors are human-managed (§8).", + "description": "Group ARC backend bumps, never mixed with other backends so a red PR names its culprit. Renovate tracks the leading-edge ARG; anchors are human-managed.", "matchManagers": ["dockerfile", "docker-compose", "custom.regex"], "matchPackageNames": ["/nordugrid-arc/"], "groupName": "ARC backend (integration stack)", "labels": ["dependencies", "integration-stack", "arc"] }, { - "description": "Group HTCondor / HTCondor-CE backend bumps (IC-ADR-002 §6, §8)", + "description": "Group HTCondor / HTCondor-CE backend bumps", "matchManagers": ["dockerfile", "docker-compose", "custom.regex"], "matchPackageNames": ["/htcondor/"], "groupName": "HTCondor backend (integration stack)", "labels": ["dependencies", "integration-stack", "htcondor"] }, { - "description": "Group Slurm backend bumps (IC-ADR-002 §6, §8)", + "description": "Group Slurm backend bumps", "matchManagers": ["dockerfile", "docker-compose", "custom.regex"], "matchPackageNames": ["/slurm/"], "groupName": "Slurm backend (integration stack)", diff --git a/tests/integration/README.md b/tests/integration/README.md index 8fc27da..8c627ae 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,7 +1,6 @@ # Integration tests -Backend-agnostic tests that run against real, containerised backend daemons ("stacks"), per -[IC-ADR-002](../../docs/adr/IC-ADR-002_integration_tests.md). +Backend-agnostic tests that run against real, containerised backend daemons ("stacks"). Quickstart: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index deeccf9..fd77d6b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,4 +1,4 @@ -"""Stack-aware fixtures for the integration suite (IC-ADR-002). +"""Stack-aware fixtures for the integration suite. Tests are written once, against the interCEde contract, and parameterized by a *stack* — a named, self-contained backend environment described in ``stacks.yml`` and started with @@ -55,14 +55,14 @@ def client_config( The client config is resolved from the entry's *own* id, never the ``stack:`` alias: an aliased entry (e.g. ``local-slurm`` reusing ``ssh-slurm``'s compose) drives the shared backend over a different transport/endpoint and so keeps its own client half. Backend config and client config - change together under one configuration name (IC-ADR-002 §1); this fixture is the client half. + change together under one configuration name; this fixture is the client half. """ config_name = request.config.getoption("--config") path = STACKS_DIR / stack["id"] / "config" / config_name / "intercede-client.toml" if not path.is_file(): msg = f"stack {stack['id']!r} has no {config_name!r} configuration ({path} is missing)" # In CI a *selected* stack with no config is a real failure, not a skip: an all-skipped run - # would report a green integration cell that verified nothing (IC-ADR-002 §5). Locally a + # would report a green integration cell that verified nothing. Locally a # skip stays convenient while a stack is still being brought up. if os.environ.get("CI"): pytest.fail(msg) diff --git a/tests/integration/generate_matrix.py b/tests/integration/generate_matrix.py index 96b423d..6f9dc84 100644 --- a/tests/integration/generate_matrix.py +++ b/tests/integration/generate_matrix.py @@ -1,4 +1,4 @@ -"""Generate the integration CI matrix from stacks.yml (IC-ADR-002 §5). +"""Generate the integration CI matrix from stacks.yml. Run it locally to see exactly which {stack x config x version} jobs CI would run: diff --git a/tests/integration/stacks.yml b/tests/integration/stacks.yml index 488e0d4..3052b0d 100644 --- a/tests/integration/stacks.yml +++ b/tests/integration/stacks.yml @@ -1,4 +1,4 @@ -# THE manifest: single source of truth for the integration CI matrix (IC-ADR-002 §1, §5). +# THE manifest: single source of truth for the integration CI matrix. # # The field reference and the stack contract live in docs/dev/reference/integration-stacks.md — # this file is data, not documentation. Adding a stack, a configuration, or a supported version is @@ -9,12 +9,12 @@ stacks: - id: arc-slurm configs: [basic] - versions: [latest] # ARC 7.x leading edge; floor is 7.x (IC-ADR-002 §8) + versions: [latest] # ARC 7.x leading edge; floor is 7.x markers: "remote and arc" - id: htcondor configs: [basic] - versions: [lts, latest] # HTCondor LTS anchor + leading edge (IC-ADR-002 §8) + versions: [lts, latest] # HTCondor LTS anchor + leading edge markers: "remote and htcondor" - id: ssh-slurm diff --git a/tests/integration/stacks/_template/compose.yml b/tests/integration/stacks/_template/compose.yml index cde717a..0715589 100644 --- a/tests/integration/stacks/_template/compose.yml +++ b/tests/integration/stacks/_template/compose.yml @@ -6,15 +6,15 @@ # - `docker compose -f compose.yml up --wait` must yield a ready backend: healthchecks are # mandatory on every service (`up --wait` gates on them — no sleep-and-retry loops). # - Daemons run in the foreground under a plain entrypoint or minimal supervisor — never -# systemd (IC-ADR-002 §3) — and log to stdout/stderr so `docker compose logs` captures them +# systemd — and log to stdout/stderr so `docker compose logs` captures them # (a file-logging daemon redirects its log to /dev/stdout; the CI failure artifact is compose # logs only). # - Credentials are minted ephemerally at container start into the shared, host-readable # `./credentials` bind mount (gitignored) — a Docker *named* volume is root-owned under # /var/lib/docker and unreadable by the host pytest that drives `exec_in_container: false` -# stacks. The layout `credentials//…` is the interface between stack and harness -# (IC-ADR-002 §4). Mint throwaway Test-CA material world-readable; the harness copies anything -# needing private modes (0400/0600) into a private dir itself. +# stacks. The layout `credentials//…` is the interface between stack and harness. Mint +# throwaway Test-CA material world-readable; the harness copies anything needing private +# modes (0400/0600) into a private dir itself. # - Images come from stacks/_images/ Dockerfiles with pinned, Renovate-annotated versions; # CI pulls prebuilt images from GHCR once issue #14 lands (build: is the local fallback). # @@ -44,7 +44,7 @@ services: volumes: # host-readable bind mount, NOT a named volume (see the credentials note above); gitignored. - ./credentials:/shared/credentials - # - ./config/basic/backend.conf:/etc/…:ro # backend half of the configuration (§1) + # - ./config/basic/backend.conf:/etc/…:ro # backend half of the configuration healthcheck: # Real readiness probe, e.g.: # Slurm: ["CMD-SHELL", "sinfo | grep -q idle"] diff --git a/tests/integration/stacks/_template/config/basic/intercede-client.toml b/tests/integration/stacks/_template/config/basic/intercede-client.toml index 174fad8..ddcaf45 100644 --- a/tests/integration/stacks/_template/config/basic/intercede-client.toml +++ b/tests/integration/stacks/_template/config/basic/intercede-client.toml @@ -2,7 +2,7 @@ # (the `client_config` fixture in tests/integration/conftest.py). # # Backend configuration (arc.conf, condor config, slurm.conf, mounted by compose.yml) and this -# file change together, atomically, under one configuration name (IC-ADR-002 §1). A new +# file change together, atomically, under one configuration name. A new # configuration (e.g. token vs proxy auth) is a new config// directory plus a stacks.yml # `configs:` entry — never a mutation of `basic`. @@ -17,4 +17,4 @@ queue = "CHANGEME" # target queue/partition name inside the container [credentials] type = "CHANGEME" # token | proxy | ssh-key -path = "credentials/CHANGEME/" # under the stack's host-readable ./credentials bind mount (IC-ADR-002 §4) +path = "credentials/CHANGEME/" # under the stack's host-readable ./credentials bind mount diff --git a/tests/integration/test_manifest.py b/tests/integration/test_manifest.py index ec74df9..664c23b 100644 --- a/tests/integration/test_manifest.py +++ b/tests/integration/test_manifest.py @@ -1,4 +1,4 @@ -"""Validate stacks.yml — the single source of truth for the CI matrix (IC-ADR-002 §1). +"""Validate stacks.yml — the single source of truth for the CI matrix. These tests need no running stack: they keep the manifest schema honest and enforce the stack contract layout (docs/dev/reference/integration-stacks.md) for every stack that has landed. They From ef647cbdb44e35692f6cb8f066384f646eeb96de Mon Sep 17 00:00:00 2001 From: aldbr Date: Fri, 23 Oct 2026 11:01:19 +0200 Subject: [PATCH 4/7] test(integration): share compose environments via a `compose:` directory ssh-slurm and local-slurm test one Slurm cluster through two transports, but `stack: ssh-slurm` made local-slurm borrow ssh-slurm's environment. Replace the stack alias with a `compose:` key naming a directory under stacks/, so both stacks point at a neutral stacks/slurm/ (compose.yml + backend config) and keep only their own intercede-client.toml. A stack is now harmonized only once its own config/ has landed too, so a shared compose.yml no longer turns sibling stacks into CI jobs on its own. Assisted-by: Claude:claude-opus-5-5 --- .github/workflows/integration.yml | 2 +- docs/dev/how-to/add-an-integration-stack.md | 10 +++-- .../how-to/run-integration-tests-locally.md | 5 ++- docs/dev/reference/integration-stacks.md | 37 +++++++++++-------- tests/integration/conftest.py | 13 ++++--- tests/integration/generate_matrix.py | 31 +++++++++++----- tests/integration/stacks.yml | 13 ++++--- .../config/basic/intercede-client.toml | 4 +- tests/integration/test_manifest.py | 34 ++++++++++------- 9 files changed, 90 insertions(+), 59 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4e37124..f6fccf4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -48,7 +48,7 @@ jobs: include: ${{ fromJSON(needs.matrix.outputs.matrix) }} name: ${{ matrix.stack }} / ${{ matrix.config }} / ${{ matrix.version }} env: - COMPOSE_FILE: tests/integration/stacks/${{ matrix.compose_stack }}/compose.yml + COMPOSE_FILE: tests/integration/stacks/${{ matrix.compose_env }}/compose.yml BACKEND_VERSION: ${{ matrix.version }} steps: - uses: actions/checkout@v6 diff --git a/docs/dev/how-to/add-an-integration-stack.md b/docs/dev/how-to/add-an-integration-stack.md index f1128a9..005498d 100644 --- a/docs/dev/how-to/add-an-integration-stack.md +++ b/docs/dev/how-to/add-an-integration-stack.md @@ -12,14 +12,16 @@ reference it. 1. **Copy the template**: `tests/integration/stacks/_template/` → `tests/integration/stacks//`. Fill in `compose.yml` (real healthcheck, host-readable `./credentials` bind mount) and `config/basic/` (backend config + `intercede-client.toml`, - which change together as one atomic configuration). An **aliased** stack (`stack:` reusing - another's compose) ships only `config/` under its own id — no `compose.yml` of its own — and its - `markers` exclude the borrowed transport (e.g. `and not ssh`). + which change together as one atomic configuration). Stacks that test **one backend through + different transports** share a neutral compose directory (e.g. `stacks/slurm/`, holding + `compose.yml` and the backend config) via `compose:`; each such stack ships only its own + `config//intercede-client.toml`, and a `local-*` stack's `markers` exclude the SSH tests + (`and not ssh`). 2. **Add the image** under `tests/integration/stacks/_images//` following the [image conventions](../reference/integration-images.md) (EL9, pinned Renovate-annotated `ARG`s, foreground daemons, ephemeral credentials at start). 3. **Add the manifest entry** in `tests/integration/stacks.yml` (`id`, `configs`, `markers`, - optionally `versions` / `stack:` alias / `exec_in_container`). + optionally `versions` / `compose:` / `exec_in_container`). 4. **Verify locally**: ```sh diff --git a/docs/dev/how-to/run-integration-tests-locally.md b/docs/dev/how-to/run-integration-tests-locally.md index ab099b3..79d16bd 100644 --- a/docs/dev/how-to/run-integration-tests-locally.md +++ b/docs/dev/how-to/run-integration-tests-locally.md @@ -22,14 +22,15 @@ For `local-*` stacks (`exec_in_container: true`) the tests run *inside* the batc instead: ```sh -docker compose -f tests/integration/stacks//compose.yml exec --user backend \ +docker compose -f tests/integration/stacks//compose.yml exec --user backend \ pytest tests/integration -m "" --stack= --config=basic ``` Two conditions this recipe depends on (both open TODOs, #13/#14): the image must ship the suite and pytest (mounted or baked in), and it must run as an **unprivileged** user — HTCondor and Slurm refuse to run jobs as root, hence `--user ` (or a compose `user:`). Note `--stack` -still names the aliased entry (e.g. `local-slurm`) so its own `intercede-client.toml` is used. +names the stack, not the compose directory (e.g. `local-slurm`, not `slurm`), so its own +`intercede-client.toml` is used. Stacks double as development environments: leave the stack up and point your client config at `localhost` to develop against a real backend. diff --git a/docs/dev/reference/integration-stacks.md b/docs/dev/reference/integration-stacks.md index b164212..9375e83 100644 --- a/docs/dev/reference/integration-stacks.md +++ b/docs/dev/reference/integration-stacks.md @@ -14,18 +14,23 @@ tests/integration/ └── stacks/ ├── _template/ # copy me: compose.yml + config/basic/intercede-client.toml ├── _images/ # shared Dockerfiles (see integration-images.md) - └── / - ├── compose.yml - └── config/ - └── basic/ # backend config + intercede-client.toml, changed together + ├── / + │ ├── compose.yml + │ └── config/ + │ └── basic/ # backend config + intercede-client.toml, changed together + ├── slurm/ # shared compose environment (`compose: slurm`) + │ ├── compose.yml + │ └── config/basic/ # backend config only (slurm.conf, cgroup.conf) + ├── ssh-slurm/config/basic/ # intercede-client.toml only: SSH transport + └── local-slurm/config/basic/ # intercede-client.toml only: Local transport ``` ## Manifest schema (`stacks.yml`) | Field | Required | Meaning | | --- | --- | --- | -| `id` | yes | Stack name; also the directory under `stacks/` unless `stack:` is set | -| `stack` | no | Alias: reuse another stack's compose environment (e.g. `local-slurm` → `ssh-slurm`) | +| `id` | yes | Stack name; holds the stack's client config, and its compose environment unless `compose:` is set | +| `compose` | no | Directory under `stacks/` holding a compose environment shared by several stacks (e.g. `slurm` for `ssh-slurm` and `local-slurm`) | | `configs` | yes | Configuration names under `stacks//config/`; start with `[basic]` | | `versions` | no | Backend support-window entries to run; absent = single leading-edge pin | | `markers` | yes | `pytest -m` expression selecting the applicable contract-suite subset | @@ -33,15 +38,17 @@ tests/integration/ The CI matrix is the expansion `{stack × config × version}` produced by `tests/integration/generate_matrix.py`; each matrix entry becomes its own CI job. Only -*harmonized* stacks — those with a `stacks//compose.yml` — become jobs; a +*harmonized* stacks — those with a `stacks//compose.yml` and their own +`stacks//config/` — become jobs; a seeded-only entry is announced with a `::notice` by `generate_matrix.py` and skipped. -**Aliases resolve two ways.** `stack:` reuses another stack's *compose environment* (so -`local-slurm` starts `ssh-slurm`'s containers), but the *client configuration* is always resolved -from the entry's **own** id: `local-slurm` ships its own `stacks/local-slurm/config/basic/ -intercede-client.toml` (a different `kind`/endpoint — Local vs SSH transport) and its `markers` -exclude the borrowed transport's tests (`scheduler and slurm and not ssh` — the pattern every -future `local-*` alias needs so an `ssh`-marked test does not run in both the ssh and local jobs). +**Shared compose environments.** `compose:` names a directory, not another stack, so stacks that +test one backend through different transports are peers: `ssh-slurm` and `local-slurm` both start +`stacks/slurm/compose.yml`, which owns the backend config. The *client configuration* is always +resolved from the entry's **own** id: each ships its own `stacks//config/basic/ +intercede-client.toml` (a different `kind`/endpoint — SSH vs Local transport), and `local-slurm`'s +`markers` exclude the SSH-transport tests (`scheduler and slurm and not ssh` — the pattern every +future `local-*` stack needs so an `ssh`-marked test does not run in both the ssh and local jobs). **`exec_in_container: true` runs pytest inside the batch container** (Local transport, no port 22). The image must ship the suite + pytest — mounted or baked in (open TODO #13/#14) — and an @@ -91,8 +98,8 @@ Options and fixtures provided by `tests/integration/conftest.py`: | `--stack` | option | Stack id from `stacks.yml`; stack-bound tests self-skip when absent (registered in `tests/conftest.py`, so every `pytest` invocation form accepts it) | | `--config` | option | Configuration name (default `basic`) | | `stack` | fixture | The selected manifest entry | -| `stack_dir` | fixture | Directory with the stack's `compose.yml` (follows the `stack:` alias) | -| `client_config` | fixture | Parsed `intercede-client.toml`, resolved under the entry's *own* id (not the alias) | +| `stack_dir` | fixture | Directory with the stack's `compose.yml` (`stacks//` when set) | +| `client_config` | fixture | Parsed `intercede-client.toml`, resolved under the entry's *own* id (never `compose:`) | Markers, registered in `pyproject.toml`: `remote`, `scheduler`, `arc`, `htcondor`, `slurm`, `ssh`, `destructive_fetch`. Each stack's `markers:` expression selects its subset. The diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fd77d6b..eb397fe 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -19,7 +19,7 @@ import pytest -from tests.integration.generate_matrix import STACKS_DIR, compose_stack, load_manifest +from tests.integration.generate_matrix import STACKS_DIR, compose_env, load_manifest @pytest.fixture(scope="session") @@ -42,8 +42,8 @@ def stack(request: pytest.FixtureRequest) -> dict[str, Any]: @pytest.fixture(scope="session") def stack_dir(stack: dict[str, Any]) -> Path: - """Directory holding the stack's compose.yml, following the `stack:` alias if set.""" - return STACKS_DIR / compose_stack(stack) + """Directory holding the stack's compose.yml: stacks// if set, else stacks//.""" + return STACKS_DIR / compose_env(stack) @pytest.fixture(scope="session") @@ -52,9 +52,10 @@ def client_config( ) -> dict[str, Any]: """Load intercede-client.toml for --stack/--config: endpoint, credential type, queue names. - The client config is resolved from the entry's *own* id, never the ``stack:`` alias: an aliased - entry (e.g. ``local-slurm`` reusing ``ssh-slurm``'s compose) drives the shared backend over a - different transport/endpoint and so keeps its own client half. Backend config and client config + The client config is resolved from the entry's *own* id, never its ``compose:`` directory: + stacks sharing a compose environment (``ssh-slurm`` and ``local-slurm`` on ``stacks/slurm/``) + drive the shared backend over different transports/endpoints and so each keep their own client + half. Backend config and client config change together under one configuration name; this fixture is the client half. """ config_name = request.config.getoption("--config") diff --git a/tests/integration/generate_matrix.py b/tests/integration/generate_matrix.py index 6f9dc84..d519b55 100644 --- a/tests/integration/generate_matrix.py +++ b/tests/integration/generate_matrix.py @@ -5,11 +5,11 @@ pixi run -e py313 python tests/integration/generate_matrix.py The integration workflow (.github/workflows/integration.yml) calls it with --github-output to -append the matrix to $GITHUB_OUTPUT. Only *harmonized* stacks — those whose stacks//compose.yml -has landed — become jobs; stacks that are merely seeded in the manifest are announced and skipped +append the matrix to $GITHUB_OUTPUT. Only *harmonized* stacks — those whose compose.yml and own +client config have landed — become jobs; stacks that are merely seeded in the manifest are announced and skipped here (rather than by a per-job step), so an unharmonized stack never turns a matrix job red. -This module is also the single home of the manifest helpers (``load_manifest``, ``compose_stack``): +This module is also the single home of the manifest helpers (``load_manifest``, ``compose_env``): conftest.py and test_manifest.py import them from here. It must therefore stay import-light — the workflow runs it with the runner's *system* python3, so it may not import pytest. See docs/dev/reference/integration-stacks.md for the manifest schema. @@ -41,14 +41,24 @@ def load_manifest() -> list[dict[str, Any]]: return data["stacks"] -def compose_stack(entry: dict[str, Any]) -> str: - """Return the compose environment a stack runs in: its own id, or the `stack:` alias.""" - return entry.get("stack", entry["id"]) +def compose_env(entry: dict[str, Any]) -> str: + """Return the directory under stacks/ holding the stack's compose.yml: `compose:`, else its id. + + `compose:` names a directory, not another stack, so stacks that share a backend (ssh-slurm and + local-slurm on stacks/slurm/) are peers rather than one borrowing the other's environment. + """ + return entry.get("compose", entry["id"]) def is_harmonized(entry: dict[str, Any]) -> bool: - """Report whether the stack's compose.yml has landed (vs. merely seeded in the manifest).""" - return (STACKS_DIR / compose_stack(entry) / "compose.yml").is_file() + """Report whether the stack has landed: its compose.yml *and* its own client config. + + Requiring the stack's own config/ means a stack sharing a compose environment does not become + a CI job just because a sibling stack landed that environment. + """ + return (STACKS_DIR / compose_env(entry) / "compose.yml").is_file() and ( + STACKS_DIR / entry["id"] / "config" + ).is_dir() def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -67,7 +77,7 @@ def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]: include.append( { "stack": stack_id, - "compose_stack": compose_stack(entry), + "compose_env": compose_env(entry), "config": config, "version": version, "markers": entry["markers"], @@ -97,7 +107,8 @@ def main() -> None: # stays visible without a skipped-job placeholder in the check UI. for entry in seeded: note = ( - f"stack {entry.get('id')!r} is seeded in stacks.yml but has no compose.yml yet" + f"stack {entry.get('id')!r} is seeded in stacks.yml but has no compose.yml" + " or client config yet" " - skipping" ) if args.github_output: diff --git a/tests/integration/stacks.yml b/tests/integration/stacks.yml index 3052b0d..d06fd87 100644 --- a/tests/integration/stacks.yml +++ b/tests/integration/stacks.yml @@ -5,7 +5,7 @@ # one entry here plus files under stacks//; no workflow edits. # # The entries below are seeded for the three in-flight stack PRs (issues #5, #12, #13); a stack -# becomes a CI job only once its stacks//compose.yml lands ("harmonized"). +# becomes a CI job only once its compose.yml and its own client config land ("harmonized"). stacks: - id: arc-slurm configs: [basic] @@ -17,15 +17,18 @@ stacks: versions: [lts, latest] # HTCondor LTS anchor + leading edge markers: "remote and htcondor" + # ssh-slurm and local-slurm share one Slurm cluster (stacks/slurm/compose.yml) and differ only + # in how interCEde reaches it: each keeps its own client config under stacks//config/. - id: ssh-slurm + compose: slurm configs: [basic] markers: "scheduler and slurm" - id: local-slurm - stack: ssh-slurm # reuse ssh-slurm's compose environment... + compose: slurm configs: [basic] - # ...but drive Slurm over the Local transport, so exclude the SSH-transport tests. Every - # future local-* alias needs the same `and not ssh` guard: the `ssh` marker is otherwise - # byte-identical here and in ssh-slurm, so an ssh test would run in both jobs. + # Local transport, so exclude the SSH-transport tests. Every future local-* stack sharing a + # compose environment with an ssh-* one needs the same `and not ssh` guard: the `ssh` marker + # is otherwise byte-identical here and in ssh-slurm, so an ssh test would run in both jobs. markers: "scheduler and slurm and not ssh" exec_in_container: true diff --git a/tests/integration/stacks/_template/config/basic/intercede-client.toml b/tests/integration/stacks/_template/config/basic/intercede-client.toml index ddcaf45..6d56b61 100644 --- a/tests/integration/stacks/_template/config/basic/intercede-client.toml +++ b/tests/integration/stacks/_template/config/basic/intercede-client.toml @@ -7,8 +7,8 @@ # `configs:` entry — never a mutation of `basic`. [stack] -id = "CHANGEME" # must match THIS entry's own id in stacks.yml — an aliased entry (reusing another - # stack's compose) still keeps its own client config under its own id +id = "CHANGEME" # must match THIS entry's own id in stacks.yml — a stack sharing a `compose:` + # environment still keeps its own client config under its own id [backend] kind = "CHANGEME" # arc | htcondor-ce | htcondor | ssh | local — which client path the tests drive diff --git a/tests/integration/test_manifest.py b/tests/integration/test_manifest.py index 664c23b..ef8e97f 100644 --- a/tests/integration/test_manifest.py +++ b/tests/integration/test_manifest.py @@ -13,7 +13,7 @@ from tests.integration.generate_matrix import ( STACKS_DIR, build_matrix, - compose_stack, + compose_env, load_manifest, ) @@ -34,46 +34,52 @@ def test_stack_ids_are_unique(): assert len(ids) == len(set(ids)), f"duplicate stack ids in stacks.yml: {ids}" -def test_stack_aliases_resolve(): - ids = {entry["id"] for entry in load_manifest()} +def test_compose_names_a_stacks_directory(): + """`compose:` is a plain directory name under stacks/, not a path or a reserved _ directory.""" for entry in load_manifest(): - alias = entry.get("stack") - if alias is not None: - assert alias in ids, ( - f"stack {entry['id']}: alias {alias!r} is not a known stack id" + compose = entry.get("compose") + if compose is not None: + assert ( + isinstance(compose, str) + and compose + and "/" not in compose + and not compose.startswith((".", "_")) + ), ( + f"stack {entry['id']}: `compose: {compose!r}` must be a directory name under stacks/" ) def test_matrix_expansion(): - """generate_matrix expands the versions axis and resolves stack aliases.""" + """generate_matrix expands the versions axis and resolves `compose:` directories.""" by_stack: dict[str, list[dict[str, Any]]] = {} for job in build_matrix(load_manifest()): by_stack.setdefault(job["stack"], []).append(job) assert { "stack", - "compose_stack", + "compose_env", "config", "version", "markers", "exec_in_container", } <= set(job) assert {job["version"] for job in by_stack["htcondor"]} == {"lts", "latest"} - assert all(job["compose_stack"] == "ssh-slurm" for job in by_stack["local-slurm"]) + for stack_id in ("ssh-slurm", "local-slurm"): + assert all(job["compose_env"] == "slurm" for job in by_stack[stack_id]) def test_harmonized_stacks_follow_the_layout(): """A stack whose directory has landed must follow the stack contract. - ``compose.yml`` lives with the *compose* stack (the ``stack:`` alias target when set); the - client config lives with the entry's *own* id, so an aliased entry (``local-slurm``) still ships - its own ``intercede-client.toml`` even though it borrows another stack's compose environment. + ``compose.yml`` lives in the compose environment (``stacks//`` when ``compose:`` is + set); the client config lives under the entry's *own* id, so ``ssh-slurm`` and ``local-slurm`` + each ship their own ``intercede-client.toml`` next to the shared ``stacks/slurm/``. The contract is documented in docs/dev/reference/integration-stacks.md. """ for entry in load_manifest(): own_dir = STACKS_DIR / entry["id"] if not own_dir.is_dir(): continue # seeded in the manifest, not harmonized yet — files land with the stack PR - compose_dir = STACKS_DIR / compose_stack(entry) + compose_dir = STACKS_DIR / compose_env(entry) assert (compose_dir / "compose.yml").is_file(), ( f"{compose_dir} is missing compose.yml" ) From ff94e707598cad72048bcab2f0813a4b87e32086 Mon Sep 17 00:00:00 2001 From: aldbr Date: Fri, 23 Oct 2026 11:13:26 +0200 Subject: [PATCH 5/7] test(integration): trim comments and keep only the contract tests Comments now point at docs/dev instead of repeating it. Drop the matrix expansion and `compose:` name tests: they pinned generator internals and manifest values, while the layout test already catches a wrong `compose:`. Assisted-by: Claude:claude-opus-5-5 --- .github/actions/pixi-setup/action.yml | 7 +- .github/workflows/integration.yml | 32 +------- pyproject.toml | 2 +- tests/conftest.py | 10 +-- tests/integration/conftest.py | 34 ++------- tests/integration/generate_matrix.py | 41 ++--------- tests/integration/stacks.yml | 20 +---- .../integration/stacks/_template/compose.yml | 40 ++-------- .../config/basic/intercede-client.toml | 21 ++---- tests/integration/test_manifest.py | 73 ++----------------- 10 files changed, 47 insertions(+), 233 deletions(-) diff --git a/.github/actions/pixi-setup/action.yml b/.github/actions/pixi-setup/action.yml index d4accdb..3215ea3 100644 --- a/.github/actions/pixi-setup/action.yml +++ b/.github/actions/pixi-setup/action.yml @@ -1,8 +1,6 @@ name: Set up pixi description: > - Install pixi and the requested environment(s), applying the editable-install workaround for - prefix-dev/pixi#3762 before the environment is solved. Shared by ci.yml and integration.yml so - the workaround lives in one place. + Install pixi and the requested environment(s), with the prefix-dev/pixi#3762 workaround. inputs: environments: @@ -19,10 +17,9 @@ runs: - name: Workaround for prefix-dev/pixi#3762 shell: bash run: | - # Editable installs break under pixi in CI; pin to non-editable before the env is solved. sed -i.bak 's@editable = true@editable = false@g' pyproject.toml rm pyproject.toml.bak - git diff --stat # show what the workaround changed + git diff --stat - uses: prefix-dev/setup-pixi@v0.9.3 with: cache: false diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f6fccf4..14edb59 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,15 +1,4 @@ -# Skeleton of the manifest-driven integration workflow (scaffold issue #9). -# -# The matrix is generated from tests/integration/stacks.yml — adding a stack, configuration or -# version is a manifest entry plus files, never a workflow edit. generate_matrix.py emits jobs only -# for *harmonized* stacks (those with a stacks//compose.yml); stacks that are seeded in the -# manifest but not yet harmonized are announced with a `::notice` in the matrix job and produce no -# job here (so this workflow has zero jobs until the first stack lands). -# -# Triggers on any PR that touches the integration suite (so a stack PR self-tests the moment it -# lands files — no workflow edit), plus manual/reusable dispatch. It is NOT yet gating: until issue -# #14 wires it into ci.yml, has CI pull prebuilt images from GHCR (instead of building on the -# runner), and makes it a required check, a red run here does not block merges. +# Matrix comes from tests/integration/stacks.yml. Not a required check until #14. name: Integration tests on: @@ -33,9 +22,6 @@ jobs: - uses: actions/checkout@v6 - name: Generate {stack x config x version} matrix from stacks.yml id: gen - # Same script developers run locally (without --github-output) to preview the matrix. It - # also filters to harmonized stacks and prints a notice for the seeded-only ones. - # PyYAML ships with the runner's system python; `pip install pyyaml` here if that changes. run: python3 tests/integration/generate_matrix.py --github-output integration: @@ -53,8 +39,6 @@ jobs: steps: - uses: actions/checkout@v6 - # Every job in the matrix is a harmonized stack (the matrix job filtered out the rest), so - # compose.yml is guaranteed present — no per-step "has this stack landed?" guard is needed. - name: Start the stack (healthchecks gate readiness) run: docker compose -f "$COMPOSE_FILE" up --wait @@ -66,22 +50,16 @@ jobs: - name: Run the contract suite run: | if [ "${{ matrix.exec_in_container }}" = "true" ]; then - # local-* stacks: run pytest inside the batch container. The batch - # scheduler typically refuses to run as root (HTCondor, Slurm), so the image must ship - # an unprivileged test user and the suite + pytest must be mounted or baked in. - # TODO(#13/#14): mount/install the suite and pin the exec user (compose `user:` or - # `exec --user `); see docs/dev/reference/integration-stacks.md. + # TODO(#13/#14): ship the suite in the image and exec as an unprivileged user. runner=(docker compose -f "$COMPOSE_FILE" exec -T backend pytest) else runner=(pixi run -e py313 pytest) fi - set +e # keep going so exit 5 (no tests selected) can be handled below + set +e "${runner[@]}" tests/integration -m "${{ matrix.markers }}" \ --stack="${{ matrix.stack }}" --config="${{ matrix.config }}" rc=$? - # pytest exits 5 when the stack's markers select no tests. Phase 0 ships only the - # manifest tests, so the first harmonized stack would otherwise go red for "no tests - # ran"; tolerate exit 5 with a notice until the Phase-2 contract suite lands. + # Exit 5 = no tests selected; tolerated until the contract suite lands. if [ "$rc" -eq 5 ]; then echo "::notice::stack '${{ matrix.stack }}': no tests matched markers '${{ matrix.markers }}' (pytest exit 5) - passing until the contract suite lands" rc=0 @@ -90,8 +68,6 @@ jobs: - name: Collect daemon logs if: failure() - # Daemons run in the foreground and log to stdout/stderr, so compose logs - # are the whole failure story — a file-logging daemon redirects its log to /dev/stdout. run: docker compose -f "$COMPOSE_FILE" logs --no-color > compose-logs.txt || true - name: Upload logs as artifacts diff --git a/pyproject.toml b/pyproject.toml index 705b810..8b418c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ asyncio_mode = "auto" markers = [ "backend_only: restrict a test to the named backend fixture(s)", "xfail_backend: strict xfail a test on the named backend fixture(s)", - # Integration-suite markers: stacks.yml selects per-stack subsets with -m. + # integration suite (selected per stack in tests/integration/stacks.yml) "remote: drives a remote CE backend (grid gateway: ARC, HTCondor-CE)", "scheduler: drives a Transport x Scheduler composition (BatchBackend)", "arc: requires an ARC CE", diff --git a/tests/conftest.py b/tests/conftest.py index 6f86f89..15f275b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,7 @@ -"""Session-wide pytest options for interCEde (must live at the tests/ root). +"""Pytest options for the integration suite. -``--stack``/``--config`` are consumed by the integration suite (tests/integration/conftest.py), but -pytest only honours ``pytest_addoption`` from *initial* conftest files — those in the rootdir and -the invocation's target directories. Registering the options here (the root of ``testpaths``) means -every invocation form accepts them: bare ``pytest``, ``pytest tests`` and -``pytest tests/integration`` all work, instead of only the last. The fixtures that read the options -stay in tests/integration/conftest.py. +Registered at the tests/ root so that every invocation (``pytest``, ``pytest tests/integration``) +accepts them; pytest only reads ``pytest_addoption`` from the initial conftest files. """ from __future__ import annotations diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index eb397fe..6833ae8 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,14 +1,4 @@ -"""Stack-aware fixtures for the integration suite. - -Tests are written once, against the interCEde contract, and parameterized by a *stack* — a named, -self-contained backend environment described in ``stacks.yml`` and started with -``docker compose -f stacks//compose.yml up --wait``. Select the stack with ``--stack`` and the -configuration with ``--config`` (registered in tests/conftest.py; see -docs/dev/how-to/run-integration-tests-locally.md). - -Stack-bound tests skip themselves when no ``--stack`` is given, so the unit-test run -(``pixi run pytest``) stays green without any compose environment. -""" +"""Stack fixtures for the integration suite, selected with --stack and --config.""" from __future__ import annotations @@ -24,15 +14,12 @@ @pytest.fixture(scope="session") def stack(request: pytest.FixtureRequest) -> dict[str, Any]: - """Return the stacks.yml entry selected by --stack; skip the test when no stack is given.""" + """Return the stacks.yml entry selected by --stack; skip the test when none is given.""" stack_id = request.config.getoption("--stack") if stack_id is None: pytest.skip("no --stack given (this test runs against a compose stack)") entries = {entry["id"]: entry for entry in load_manifest()} if stack_id not in entries: - # A bad --stack is a usage error for the whole session, not a per-test failure: raising - # UsageError from a fixture yields per-test tracebacks (repeated per xdist worker) and no - # exit-4; pytest.exit stops the run once, cleanly. pytest.exit( f"unknown stack {stack_id!r}; known stacks: {', '.join(sorted(entries))}", returncode=pytest.ExitCode.USAGE_ERROR, @@ -42,7 +29,7 @@ def stack(request: pytest.FixtureRequest) -> dict[str, Any]: @pytest.fixture(scope="session") def stack_dir(stack: dict[str, Any]) -> Path: - """Directory holding the stack's compose.yml: stacks// if set, else stacks//.""" + """Return the directory holding the stack's compose.yml.""" return STACKS_DIR / compose_env(stack) @@ -50,23 +37,14 @@ def stack_dir(stack: dict[str, Any]) -> Path: def client_config( request: pytest.FixtureRequest, stack: dict[str, Any] ) -> dict[str, Any]: - """Load intercede-client.toml for --stack/--config: endpoint, credential type, queue names. - - The client config is resolved from the entry's *own* id, never its ``compose:`` directory: - stacks sharing a compose environment (``ssh-slurm`` and ``local-slurm`` on ``stacks/slurm/``) - drive the shared backend over different transports/endpoints and so each keep their own client - half. Backend config and client config - change together under one configuration name; this fixture is the client half. - """ + """Load stacks//config//intercede-client.toml.""" config_name = request.config.getoption("--config") path = STACKS_DIR / stack["id"] / "config" / config_name / "intercede-client.toml" if not path.is_file(): msg = f"stack {stack['id']!r} has no {config_name!r} configuration ({path} is missing)" - # In CI a *selected* stack with no config is a real failure, not a skip: an all-skipped run - # would report a green integration cell that verified nothing. Locally a - # skip stays convenient while a stack is still being brought up. + # In CI a missing config must fail: an all-skipped job would be green yet test nothing. if os.environ.get("CI"): pytest.fail(msg) - pytest.skip(f"{msg} - skipping (set CI=1 to make this a failure)") + pytest.skip(msg) with path.open("rb") as f: return tomllib.load(f) diff --git a/tests/integration/generate_matrix.py b/tests/integration/generate_matrix.py index d519b55..afad1fa 100644 --- a/tests/integration/generate_matrix.py +++ b/tests/integration/generate_matrix.py @@ -1,18 +1,7 @@ """Generate the integration CI matrix from stacks.yml. -Run it locally to see exactly which {stack x config x version} jobs CI would run: - - pixi run -e py313 python tests/integration/generate_matrix.py - -The integration workflow (.github/workflows/integration.yml) calls it with --github-output to -append the matrix to $GITHUB_OUTPUT. Only *harmonized* stacks — those whose compose.yml and own -client config have landed — become jobs; stacks that are merely seeded in the manifest are announced and skipped -here (rather than by a per-job step), so an unharmonized stack never turns a matrix job red. - -This module is also the single home of the manifest helpers (``load_manifest``, ``compose_env``): -conftest.py and test_manifest.py import them from here. It must therefore stay import-light — the -workflow runs it with the runner's *system* python3, so it may not import pytest. See -docs/dev/reference/integration-stacks.md for the manifest schema. +Preview locally with ``python tests/integration/generate_matrix.py``. Runs on the CI runner's +system python3, so it must not import pytest. """ from __future__ import annotations @@ -33,7 +22,7 @@ def load_manifest() -> list[dict[str, Any]]: - """Return the list of stack entries from stacks.yml (the CI-matrix source of truth).""" + """Return the stack entries from stacks.yml.""" with STACKS_FILE.open() as f: data = yaml.safe_load(f) if not isinstance(data, dict) or "stacks" not in data: @@ -42,33 +31,24 @@ def load_manifest() -> list[dict[str, Any]]: def compose_env(entry: dict[str, Any]) -> str: - """Return the directory under stacks/ holding the stack's compose.yml: `compose:`, else its id. - - `compose:` names a directory, not another stack, so stacks that share a backend (ssh-slurm and - local-slurm on stacks/slurm/) are peers rather than one borrowing the other's environment. - """ + """Return the directory under stacks/ holding the stack's compose.yml.""" return entry.get("compose", entry["id"]) def is_harmonized(entry: dict[str, Any]) -> bool: - """Report whether the stack has landed: its compose.yml *and* its own client config. - - Requiring the stack's own config/ means a stack sharing a compose environment does not become - a CI job just because a sibling stack landed that environment. - """ + """Return whether both the stack's compose.yml and its own config/ have landed.""" return (STACKS_DIR / compose_env(entry) / "compose.yml").is_file() and ( STACKS_DIR / entry["id"] / "config" ).is_dir() def build_matrix(stacks: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Expand manifest entries into the flat {stack x config x version} job list.""" + """Expand manifest entries into {stack x config x version} jobs.""" include = [] for entry in stacks: stack_id = entry.get("id") for key in REQUIRED_KEYS: if key not in entry: - # Name the offending stack instead of dying with a bare KeyError deep in a loop. raise SystemExit( f"stack {stack_id or entry!r}: missing required key {key!r}" ) @@ -102,15 +82,8 @@ def main() -> None: (harmonized if is_harmonized(entry) else seeded).append(entry) matrix = build_matrix(harmonized) - # Seeded-but-not-yet-harmonized stacks are not jobs. Announce them — as a CI annotation under - # --github-output, else on stderr so the JSON on stdout stays clean — so the manifest's intent - # stays visible without a skipped-job placeholder in the check UI. for entry in seeded: - note = ( - f"stack {entry.get('id')!r} is seeded in stacks.yml but has no compose.yml" - " or client config yet" - " - skipping" - ) + note = f"stack {entry.get('id')!r} has no compose.yml or config/ yet - skipping" if args.github_output: print(f"::notice::{note}") else: diff --git a/tests/integration/stacks.yml b/tests/integration/stacks.yml index d06fd87..1d97b44 100644 --- a/tests/integration/stacks.yml +++ b/tests/integration/stacks.yml @@ -1,24 +1,15 @@ -# THE manifest: single source of truth for the integration CI matrix. -# -# The field reference and the stack contract live in docs/dev/reference/integration-stacks.md — -# this file is data, not documentation. Adding a stack, a configuration, or a supported version is -# one entry here plus files under stacks//; no workflow edits. -# -# The entries below are seeded for the three in-flight stack PRs (issues #5, #12, #13); a stack -# becomes a CI job only once its compose.yml and its own client config land ("harmonized"). +# Integration CI matrix. Schema: docs/dev/reference/integration-stacks.md stacks: - id: arc-slurm configs: [basic] - versions: [latest] # ARC 7.x leading edge; floor is 7.x + versions: [latest] markers: "remote and arc" - id: htcondor configs: [basic] - versions: [lts, latest] # HTCondor LTS anchor + leading edge + versions: [lts, latest] markers: "remote and htcondor" - # ssh-slurm and local-slurm share one Slurm cluster (stacks/slurm/compose.yml) and differ only - # in how interCEde reaches it: each keeps its own client config under stacks//config/. - id: ssh-slurm compose: slurm configs: [basic] @@ -27,8 +18,5 @@ stacks: - id: local-slurm compose: slurm configs: [basic] - # Local transport, so exclude the SSH-transport tests. Every future local-* stack sharing a - # compose environment with an ssh-* one needs the same `and not ssh` guard: the `ssh` marker - # is otherwise byte-identical here and in ssh-slurm, so an ssh test would run in both jobs. - markers: "scheduler and slurm and not ssh" + markers: "scheduler and slurm and not ssh" # ssh tests already run in ssh-slurm exec_in_container: true diff --git a/tests/integration/stacks/_template/compose.yml b/tests/integration/stacks/_template/compose.yml index 0715589..96ab629 100644 --- a/tests/integration/stacks/_template/compose.yml +++ b/tests/integration/stacks/_template/compose.yml @@ -1,29 +1,5 @@ -# Template stack — copy to stacks//compose.yml and adapt. -# Stack contract: docs/dev/reference/integration-stacks.md -# How-to: docs/dev/how-to/add-an-integration-stack.md -# -# Contract highlights: -# - `docker compose -f compose.yml up --wait` must yield a ready backend: healthchecks are -# mandatory on every service (`up --wait` gates on them — no sleep-and-retry loops). -# - Daemons run in the foreground under a plain entrypoint or minimal supervisor — never -# systemd — and log to stdout/stderr so `docker compose logs` captures them -# (a file-logging daemon redirects its log to /dev/stdout; the CI failure artifact is compose -# logs only). -# - Credentials are minted ephemerally at container start into the shared, host-readable -# `./credentials` bind mount (gitignored) — a Docker *named* volume is root-owned under -# /var/lib/docker and unreadable by the host pytest that drives `exec_in_container: false` -# stacks. The layout `credentials//…` is the interface between stack and harness. Mint -# throwaway Test-CA material world-readable; the harness copies anything needing private -# modes (0400/0600) into a private dir itself. -# - Images come from stacks/_images/ Dockerfiles with pinned, Renovate-annotated versions; -# CI pulls prebuilt images from GHCR once issue #14 lands (build: is the local fallback). -# -# `backend` is the service the harness drives. A multi-service stack (a CE + its LRMS, or a Slurm -# controller + worker) adds more services here and names one of them `backend`; parameterize a -# per-role image rather than duplicating it (integration-images.md). Intra-stack secrets (e.g. a -# munge key shared between Slurm services) follow the same rule as credentials — minted at start -# into a stack-internal volume, never baked into an image (a published image with a baked key is a -# leaked credential). +# Copy to stacks//compose.yml. Contract: docs/dev/reference/integration-stacks.md +# The harness drives the service named `backend`. services: backend: # image: ghcr.io/diracgrid/intercede-testenv/:${BACKEND_VERSION:-latest} @@ -34,25 +10,21 @@ services: hostname: backend networks: - stack - # For a host-run harness (exec_in_container: false) publish the backend's port on localhost and - # record the same host:port in config//intercede-client.toml. A TLS backend's Test-CA host - # certificate must carry a SAN for the name the client actually dials — mint it for both the - # compose hostname and localhost (ARC on EL9 additionally needs crypto-policies set to LEGACY - # for the proxy chain). + # Host-run harness: publish the port and put the same host:port in intercede-client.toml. # ports: # - "127.0.0.1:2222:22" # e.g. sshd volumes: - # host-readable bind mount, NOT a named volume (see the credentials note above); gitignored. + # bind mount, not a named volume: the host-run harness must read it - ./credentials:/shared/credentials # - ./config/basic/backend.conf:/etc/…:ro # backend half of the configuration healthcheck: - # Real readiness probe, e.g.: + # e.g.: # Slurm: ["CMD-SHELL", "sinfo | grep -q idle"] # HTCondor: ["CMD", "condor_status", "-limit", "1"] # HTCondor-CE: ["CMD", "condor_ce_status"] # ARC REST: ["CMD-SHELL", "curl -ksf https://localhost:443/arex/rest/1.0 -o /dev/null"] # sshd: ["CMD-SHELL", "true < /dev/tcp/localhost/22"] - test: ["CMD", "false"] # CHANGEME — a stack without a real healthcheck must not pass review + test: ["CMD", "false"] # CHANGEME interval: 5s timeout: 10s retries: 24 diff --git a/tests/integration/stacks/_template/config/basic/intercede-client.toml b/tests/integration/stacks/_template/config/basic/intercede-client.toml index 6d56b61..785b352 100644 --- a/tests/integration/stacks/_template/config/basic/intercede-client.toml +++ b/tests/integration/stacks/_template/config/basic/intercede-client.toml @@ -1,20 +1,13 @@ -# Client-side half of the `basic` configuration for this stack, consumed by the test harness -# (the `client_config` fixture in tests/integration/conftest.py). -# -# Backend configuration (arc.conf, condor config, slurm.conf, mounted by compose.yml) and this -# file change together, atomically, under one configuration name. A new -# configuration (e.g. token vs proxy auth) is a new config// directory plus a stacks.yml -# `configs:` entry — never a mutation of `basic`. +# Client half of the `basic` configuration. See docs/dev/reference/integration-stacks.md [stack] -id = "CHANGEME" # must match THIS entry's own id in stacks.yml — a stack sharing a `compose:` - # environment still keeps its own client config under its own id +id = "CHANGEME" # this stack's id in stacks.yml [backend] -kind = "CHANGEME" # arc | htcondor-ce | htcondor | ssh | local — which client path the tests drive -endpoint = "CHANGEME" # host:port the harness dials, e.g. "https://localhost:443/arex" or "localhost:2222" -queue = "CHANGEME" # target queue/partition name inside the container +kind = "CHANGEME" # arc | htcondor-ce | htcondor | ssh | local +endpoint = "CHANGEME" # e.g. "https://localhost:443/arex" or "localhost:2222" +queue = "CHANGEME" [credentials] -type = "CHANGEME" # token | proxy | ssh-key -path = "credentials/CHANGEME/" # under the stack's host-readable ./credentials bind mount +type = "CHANGEME" # token | proxy | ssh-key +path = "credentials/CHANGEME/" # under the ./credentials bind mount diff --git a/tests/integration/test_manifest.py b/tests/integration/test_manifest.py index ef8e97f..8304ae5 100644 --- a/tests/integration/test_manifest.py +++ b/tests/integration/test_manifest.py @@ -1,32 +1,15 @@ -"""Validate stacks.yml — the single source of truth for the CI matrix. - -These tests need no running stack: they keep the manifest schema honest and enforce the stack -contract layout (docs/dev/reference/integration-stacks.md) for every stack that has landed. They -run in the plain unit-test job. The manifest helpers come from generate_matrix (kept pytest-free so -the workflow can run it with the system python3). -""" +"""Check stacks.yml and the layout of every stack that has landed.""" from __future__ import annotations -from typing import Any - -from tests.integration.generate_matrix import ( - STACKS_DIR, - build_matrix, - compose_env, - load_manifest, -) +from tests.integration.generate_matrix import STACKS_DIR, compose_env, load_manifest def test_entries_have_required_keys(): for entry in load_manifest(): assert "id" in entry, f"manifest entry without an id: {entry}" - assert entry.get("configs"), ( - f"stack {entry.get('id')}: `configs` must be a non-empty list" - ) - assert entry.get("markers"), ( - f"stack {entry.get('id')}: `markers` expression is required" - ) + assert entry.get("configs"), f"stack {entry['id']}: `configs` is required" + assert entry.get("markers"), f"stack {entry['id']}: `markers` is required" def test_stack_ids_are_unique(): @@ -34,57 +17,15 @@ def test_stack_ids_are_unique(): assert len(ids) == len(set(ids)), f"duplicate stack ids in stacks.yml: {ids}" -def test_compose_names_a_stacks_directory(): - """`compose:` is a plain directory name under stacks/, not a path or a reserved _ directory.""" - for entry in load_manifest(): - compose = entry.get("compose") - if compose is not None: - assert ( - isinstance(compose, str) - and compose - and "/" not in compose - and not compose.startswith((".", "_")) - ), ( - f"stack {entry['id']}: `compose: {compose!r}` must be a directory name under stacks/" - ) - - -def test_matrix_expansion(): - """generate_matrix expands the versions axis and resolves `compose:` directories.""" - by_stack: dict[str, list[dict[str, Any]]] = {} - for job in build_matrix(load_manifest()): - by_stack.setdefault(job["stack"], []).append(job) - assert { - "stack", - "compose_env", - "config", - "version", - "markers", - "exec_in_container", - } <= set(job) - assert {job["version"] for job in by_stack["htcondor"]} == {"lts", "latest"} - for stack_id in ("ssh-slurm", "local-slurm"): - assert all(job["compose_env"] == "slurm" for job in by_stack[stack_id]) - - -def test_harmonized_stacks_follow_the_layout(): - """A stack whose directory has landed must follow the stack contract. - - ``compose.yml`` lives in the compose environment (``stacks//`` when ``compose:`` is - set); the client config lives under the entry's *own* id, so ``ssh-slurm`` and ``local-slurm`` - each ship their own ``intercede-client.toml`` next to the shared ``stacks/slurm/``. - The contract is documented in docs/dev/reference/integration-stacks.md. - """ +def test_landed_stacks_follow_the_layout(): for entry in load_manifest(): own_dir = STACKS_DIR / entry["id"] if not own_dir.is_dir(): - continue # seeded in the manifest, not harmonized yet — files land with the stack PR + continue compose_dir = STACKS_DIR / compose_env(entry) assert (compose_dir / "compose.yml").is_file(), ( f"{compose_dir} is missing compose.yml" ) for config in entry["configs"]: client_toml = own_dir / "config" / config / "intercede-client.toml" - assert client_toml.is_file(), ( - f"{own_dir} is missing config/{config}/intercede-client.toml" - ) + assert client_toml.is_file(), f"{client_toml} is missing" From 8cf5411646d2e1e24c56d16b951691ded0efc3f8 Mon Sep 17 00:00:00 2001 From: aldbr Date: Fri, 23 Oct 2026 11:39:11 +0200 Subject: [PATCH 6/7] ci(integration): drop the pixi-setup composite action Only integration.yml still used it after main inlined the pixi steps in ci.yml. Use main's steps and pins instead (checkout v7, setup-pixi v0.11.2 with pixi v0.82.0, upload-artifact v7, persist-credentials: false), and run the workflow when tests/conftest.py or pyproject.toml change. Assisted-by: Claude:claude-opus-5-5 --- .github/actions/pixi-setup/action.yml | 26 -------------------------- .github/workflows/integration.yml | 25 +++++++++++++++++++------ 2 files changed, 19 insertions(+), 32 deletions(-) delete mode 100644 .github/actions/pixi-setup/action.yml diff --git a/.github/actions/pixi-setup/action.yml b/.github/actions/pixi-setup/action.yml deleted file mode 100644 index 3215ea3..0000000 --- a/.github/actions/pixi-setup/action.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Set up pixi -description: > - Install pixi and the requested environment(s), with the prefix-dev/pixi#3762 workaround. - -inputs: - environments: - description: Space-separated pixi environment name(s) to install (e.g. "py313"). - required: true - -runs: - using: composite - steps: - - uses: prefix-dev/setup-pixi@v0.9.3 - with: - run-install: false - post-cleanup: false - - name: Workaround for prefix-dev/pixi#3762 - shell: bash - run: | - sed -i.bak 's@editable = true@editable = false@g' pyproject.toml - rm pyproject.toml.bak - git diff --stat - - uses: prefix-dev/setup-pixi@v0.9.3 - with: - cache: false - environments: ${{ inputs.environments }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 14edb59..d3c860b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,7 +8,8 @@ on: paths: - "tests/integration/**" - ".github/workflows/integration.yml" - - ".github/actions/pixi-setup/**" + - "tests/conftest.py" + - "pyproject.toml" permissions: contents: read @@ -19,7 +20,9 @@ jobs: outputs: matrix: ${{ steps.gen.outputs.matrix }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Generate {stack x config x version} matrix from stacks.yml id: gen run: python3 tests/integration/generate_matrix.py --github-output @@ -37,14 +40,24 @@ jobs: COMPOSE_FILE: tests/integration/stacks/${{ matrix.compose_env }}/compose.yml BACKEND_VERSION: ${{ matrix.version }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - name: Start the stack (healthchecks gate readiness) + - name: Start the stack (waits for the healthchecks) run: docker compose -f "$COMPOSE_FILE" up --wait - - uses: ./.github/actions/pixi-setup + - name: Apply workarounds + if: matrix.exec_in_container == false + run: | + # Workaround for https://github.com/prefix-dev/pixi/issues/3762 + sed -i.bak 's@editable = true@editable = false@g' pyproject.toml + rm pyproject.toml.bak + - uses: prefix-dev/setup-pixi@ad1da6449d9e506fd127fd67ab2d054a769f6a94 # v0.11.2 if: matrix.exec_in_container == false with: + pixi-version: v0.82.0 + cache: false environments: py313 - name: Run the contract suite @@ -72,7 +85,7 @@ jobs: - name: Upload logs as artifacts if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: logs-${{ matrix.stack }}-${{ matrix.config }}-${{ matrix.version }} path: compose-logs.txt From 57fe770ef928f80acf3f2e41d2122f596a5a3264 Mon Sep 17 00:00:00 2001 From: aldbr Date: Fri, 23 Oct 2026 11:39:11 +0200 Subject: [PATCH 7/7] docs: rewrite the integration docs in plain English Short sentences, terms defined once, each rule stated in one place and linked from the other pages. is_harmonized becomes is_ready to match. Assisted-by: Claude:claude-opus-5-5 --- docs/dev/explanations/index.md | 2 +- docs/dev/explanations/integration-testing.md | 53 +++-- docs/dev/how-to/add-an-integration-stack.md | 72 +++--- .../how-to/run-integration-tests-locally.md | 42 ++-- docs/dev/index.md | 14 +- docs/dev/reference/index.md | 6 +- docs/dev/reference/integration-images.md | 58 +++-- docs/dev/reference/integration-stacks.md | 205 +++++++++--------- docs/dev/reference/renovate.md | 39 ++-- docs/dev/tutorials/index.md | 6 +- tests/integration/README.md | 18 +- tests/integration/generate_matrix.py | 12 +- tests/integration/stacks/_images/README.md | 9 +- 13 files changed, 254 insertions(+), 282 deletions(-) diff --git a/docs/dev/explanations/index.md b/docs/dev/explanations/index.md index 0f02a04..0bbd5d1 100644 --- a/docs/dev/explanations/index.md +++ b/docs/dev/explanations/index.md @@ -1,3 +1,3 @@ # Explanations -- [Why integration stacks — testing against real backend daemons](integration-testing.md) +- [Why we test against real backends](integration-testing.md) diff --git a/docs/dev/explanations/integration-testing.md b/docs/dev/explanations/integration-testing.md index 8073c7e..575093d 100644 --- a/docs/dev/explanations/integration-testing.md +++ b/docs/dev/explanations/integration-testing.md @@ -1,37 +1,34 @@ -# Why integration stacks — testing against real backend daemons +# Why we test against real backends -Why the integration suite runs against real backend daemons, and how it is phased. +## The problem with fakes -## Why real daemons +The unit tests check interCEde against fake backends. A fake can agree with our code while the +real service behaves differently. Many bugs that interCEde must handle come from the real services: +behaviour that changes between versions, output that can be downloaded only once, unusual job +states. Unit tests cannot find these bugs. -interCEde's architecture is deliberately mock-friendly — everything is a `Protocol`, and the unit -suite exercises contracts against fakes. That is also its greatest testing risk: a structural -protocol and a fake will happily agree with each other forever while the real ARC REST endpoint -or the real `condor_submit -spool` handshake drift away underneath. The bugs interCEde exists to -absorb (version-specific daemon behaviour, destructive output retrieval, status-mapping quirks) -are exactly the bugs unit tests structurally cannot see — HTCondor ≥ 25.8 silently turning -spooled-output retrieval into a one-shot operation is the canonical example. +Example: since HTCondor 25.8, downloading the output of a spooled job deletes it on the server. You +can download it only once. A fake HTCondor does not do this. -So the integration suite runs **real backend daemons** in containerised **stacks**, on every pull -request, and every backend version is pinned and tracked by Renovate so an upstream release that -breaks a contract surfaces as a red, bisectable PR. +## What the integration tests do -## Capability symmetry +The integration tests start real backend services (ARC, HTCondor, Slurm) in containers and run +on pull requests. Each backend version is pinned. When a new version is released, Renovate opens a +pull request that updates it. If the new version breaks something, that pull request fails, and we +know which version caused the problem. -The integration suite is the executable form of the contract: it narrows to optional capabilities -exactly the way the library's consumers do (`isinstance` against a `runtime_checkable` protocol) -and skips where a backend genuinely lacks a capability. If a capability check works in the test -harness, it works for DiracX; if a backend's structural typing lies, a real daemon is where the lie -is caught. The skip helper that does this lands with the Phase-2 contract suite that calls it — -its signature would otherwise be guessed ahead of its callers. +## Optional features -## Phase 0 vs Phase 2 +Some backends do not support every feature. The tests check for a feature in the same way that +users of the library do (`isinstance` against a `runtime_checkable` protocol), and skip the test +when the backend does not have it. The helper for this will be added in phase 2, together with the +first tests that use it. -**Phase 0 (now):** the `intercede` package's protocols don't exist yet, so stack PRs validate -their stacks with **native client tools** (`arcsub`/`arcstat`/`arcget`, `condor_submit`, -`sbatch`) in marker-selected test modules. +## Phases -**Phase 2:** a single backend-agnostic **contract suite** (`test_submit.py`, `test_status.py`, -`test_output.py`, `test_kill.py`) written against the interCEde protocols replaces the -native-tool tests. Backend-specific semantics (e.g. HTCondor's destructive fetch) get dedicated -marker-selected tests (`destructive_fetch`) rather than conditionals inside generic ones. +- **Phase 0 (now).** The interCEde protocols do not exist yet. Each stack is tested with the + backend's own command-line tools (`arcsub`, `arcstat`, `arcget`, `condor_submit`, `sbatch`). +- **Phase 2.** One set of tests, written against the interCEde protocols, runs on every stack + (`test_submit.py`, `test_status.py`, `test_output.py`, `test_kill.py`). It replaces the phase 0 + tests. Behaviour that only one backend has (for example HTCondor's one-time output download) + gets its own test and marker (`destructive_fetch`), not an `if` inside a shared test. diff --git a/docs/dev/how-to/add-an-integration-stack.md b/docs/dev/how-to/add-an-integration-stack.md index 005498d..ff9b8e4 100644 --- a/docs/dev/how-to/add-an-integration-stack.md +++ b/docs/dev/how-to/add-an-integration-stack.md @@ -1,54 +1,42 @@ # Add an integration stack -A *stack* is a named, self-contained backend environment (one `docker compose` file plus -versioned configuration) that the integration suite runs against. Adding one requires **no -workflow edits** — the CI matrix is generated from the manifest. +Words like *stack*, *compose environment* and *configuration* are explained in +[Integration stacks](../reference/integration-stacks.md#words-used-here). -Before starting, read the [stack contract](../reference/integration-stacks.md) — the rules below -reference it. +You do not need to change any CI workflow. CI finds the stack in `stacks.yml`. ## Steps -1. **Copy the template**: `tests/integration/stacks/_template/` → - `tests/integration/stacks//`. Fill in `compose.yml` (real healthcheck, host-readable - `./credentials` bind mount) and `config/basic/` (backend config + `intercede-client.toml`, - which change together as one atomic configuration). Stacks that test **one backend through - different transports** share a neutral compose directory (e.g. `stacks/slurm/`, holding - `compose.yml` and the backend config) via `compose:`; each such stack ships only its own - `config//intercede-client.toml`, and a `local-*` stack's `markers` exclude the SSH tests - (`and not ssh`). -2. **Add the image** under `tests/integration/stacks/_images//` following the - [image conventions](../reference/integration-images.md) (EL9, pinned Renovate-annotated - `ARG`s, foreground daemons, ephemeral credentials at start). -3. **Add the manifest entry** in `tests/integration/stacks.yml` (`id`, `configs`, `markers`, - optionally `versions` / `compose:` / `exec_in_container`). -4. **Verify locally**: +1. **Copy the template.** Copy `tests/integration/stacks/_template/` to + `tests/integration/stacks//`. Then fill in: + - `compose.yml`: a real healthcheck, and the `./credentials` bind mount; + - `config/basic/`: the backend config files and `intercede-client.toml`. + + If your stack uses a backend that another stack already starts (for example a second way to + connect to Slurm), do not copy `compose.yml`. Set `compose:` to the existing folder, and add + only `config/basic/intercede-client.toml`. See + [Sharing a compose environment](../reference/integration-stacks.md#sharing-a-compose-environment). +2. **Add the image** in `tests/integration/stacks/_images//`. Follow the + [image rules](../reference/integration-images.md). +3. **Add the stack to `tests/integration/stacks.yml`**: `id`, `configs`, `markers`, and if needed + `compose`, `versions`, `exec_in_container`. +4. **Test it locally:** ```sh docker compose -f tests/integration/stacks//compose.yml up --wait pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic - pixi run -e py313 pytest tests/integration/test_manifest.py # layout checks + pixi run -e py313 pytest tests/integration/test_manifest.py ``` -## Harmonization checklist - -For a stack PR (including the in-flight prototype stacks converging onto this layout): - -- [ ] Files under `stacks//` following `_template/` (compose.yml + `config/basic/`) -- [ ] `docker compose … up --wait` succeeds locally; healthchecks probe real readiness -- [ ] Daemons foregrounded (no systemd), logging to stdout/stderr; Docker (not Podman) on - EL9-based images -- [ ] Dockerfile under `stacks/_images//` with pinned, `# renovate:`-annotated `ARG`s -- [ ] Credentials generated at start, world-readable, into the host-readable `./credentials` bind - mount under `credentials//…` (gitignored) -- [ ] `intercede-client.toml` filled in (own `id`, endpoint, credential type/path, queue); a - host-run stack publishes its port on localhost with a matching Test-CA SAN -- [ ] `stacks.yml` entry present; `pytest tests/integration/test_manifest.py` green -- [ ] At least one marker-selected Phase-0 smoke test ships with the stack (so its job isn't just - "no tests ran" — the workflow tolerates that, but a smoke test makes the job meaningful) -- [ ] Bespoke per-PR workflow removed: the manifest-driven `integration.yml` now runs - automatically on any PR touching `tests/integration/**`, so the stack self-tests. It stays - **non-blocking** until issue #14 (GHCR image pulls + `ci.yml` invocation + required check), - so keep a bespoke workflow only if you need a *gating* check in the interim -- [ ] `docker compose logs` captures the daemons' output on failure (daemons log to stdout; a - file-logging daemon redirects to `/dev/stdout`) +## Checklist for the pull request + +- [ ] The stack follows the [rules for a stack](../reference/integration-stacks.md#rules-for-a-stack). +- [ ] `docker compose ... up --wait` works on your machine. +- [ ] The images use Docker, not Podman. +- [ ] `intercede-client.toml` is filled in: `id`, endpoint, queue, credential type and path. +- [ ] `pytest tests/integration/test_manifest.py` passes. +- [ ] At least one test runs for this stack (phase 0: a test that uses the backend's own + command-line tools). Without one, the CI job passes but tests nothing. +- [ ] If your pull request had its own workflow for this stack, remove it. `integration.yml` runs + on every pull request that changes `tests/integration/`. It does not block merging yet + (#14), so keep your own workflow only if you need a check that blocks merging. diff --git a/docs/dev/how-to/run-integration-tests-locally.md b/docs/dev/how-to/run-integration-tests-locally.md index 79d16bd..6b720b4 100644 --- a/docs/dev/how-to/run-integration-tests-locally.md +++ b/docs/dev/how-to/run-integration-tests-locally.md @@ -1,54 +1,50 @@ # Run the integration tests locally -Everything CI does is reproducible on a laptop: CI does nothing a developer cannot do locally. +CI runs the same commands as below, so you can reproduce any CI job on your machine. -## Run one stack +## Run the tests for one stack -1. Start the stack; healthchecks gate readiness, so when `up --wait` returns the backend is - usable: +1. Start the stack. The command returns when all healthchecks pass, so the backend is ready: ```sh - docker compose -f tests/integration/stacks//compose.yml up --wait + docker compose -f tests/integration/stacks//compose.yml up --wait ``` -2. Run the applicable test subset — the stack's `markers:` expression is in - `tests/integration/stacks.yml`: + `` is the stack's `compose` value in `tests/integration/stacks.yml`, or its `id` if it + has none. + +2. Run the tests. Copy `` from the stack's entry in `stacks.yml`: ```sh pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic ``` -For `local-*` stacks (`exec_in_container: true`) the tests run *inside* the batch container -instead: +For stacks with `exec_in_container: true` (the `local-*` stacks), run the tests inside the +container instead: ```sh docker compose -f tests/integration/stacks//compose.yml exec --user backend \ pytest tests/integration -m "" --stack= --config=basic ``` -Two conditions this recipe depends on (both open TODOs, #13/#14): the image must ship the suite -and pytest (mounted or baked in), and it must run as an **unprivileged** user — HTCondor and Slurm -refuse to run jobs as root, hence `--user ` (or a compose `user:`). Note `--stack` -names the stack, not the compose directory (e.g. `local-slurm`, not `slurm`), so its own -`intercede-client.toml` is used. - -Stacks double as development environments: leave the stack up and point your client config at -`localhost` to develop against a real backend. +This does not work yet (#13, #14): the image must first contain pytest and the tests, and have a +normal (not root) user. `--stack` is the stack `id` (for example `local-slurm`), not the compose +folder (`slurm`). -## Preview the CI matrix +You can also leave a stack running and use it to develop against a real backend: point your client +config at `localhost`. -The CI matrix is generated from `stacks.yml` by a plain script — run it locally to see exactly -which `{stack × config × version}` jobs CI would create: +## See which CI jobs will run ```sh pixi run -e py313 python tests/integration/generate_matrix.py ``` -(The workflow calls the same script with `--github-output`.) +This prints the jobs that CI will create from `stacks.yml`. -## Manifest-only checks (no Docker needed) +## Check `stacks.yml` and the stack folders -The manifest schema and stack-layout checks run as ordinary unit tests: +No Docker needed: ```sh pixi run -e py313 pytest tests/integration/test_manifest.py diff --git a/docs/dev/index.md b/docs/dev/index.md index 6d1705e..67d045a 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -1,11 +1,11 @@ # Developer documentation -Organised following [Diátaxis](https://diataxis.fr/), as in the other DiracX-family projects: +The documentation is organised following [Diátaxis](https://diataxis.fr/), like the other +DiracX projects: -- [Tutorials](tutorials/index.md) — learning-oriented, step-by-step lessons -- [How-to guides](how-to/index.md) — task-oriented recipes -- [Reference](reference/index.md) — information-oriented technical description -- [Explanations](explanations/index.md) — understanding-oriented discussion +- [Tutorials](tutorials/index.md): lessons to learn step by step +- [How-to guides](how-to/index.md): steps for one task +- [Reference](reference/index.md): technical details +- [Explanations](explanations/index.md): why things are done this way -User-facing documentation (`docs/user/`) is deferred until the library has a public API to -document. +User documentation (`docs/user/`) will come once the library has a public API. diff --git a/docs/dev/reference/index.md b/docs/dev/reference/index.md index c1fc846..10fc106 100644 --- a/docs/dev/reference/index.md +++ b/docs/dev/reference/index.md @@ -1,5 +1,5 @@ # Reference -- [Integration stacks — layout, manifest schema, stack contract](integration-stacks.md) -- [Integration images — Dockerfile conventions](integration-images.md) -- [Renovate — dependency and version tracking](renovate.md) +- [Integration stacks: files, `stacks.yml` fields, rules](integration-stacks.md) +- [Integration images: Dockerfile rules](integration-images.md) +- [Renovate: automatic version updates](renovate.md) diff --git a/docs/dev/reference/integration-images.md b/docs/dev/reference/integration-images.md index 366d235..60f461a 100644 --- a/docs/dev/reference/integration-images.md +++ b/docs/dev/reference/integration-images.md @@ -1,11 +1,11 @@ -# Integration images — Dockerfile conventions +# Integration images -Dockerfiles under `tests/integration/stacks/_images/` are shared across stacks (`arc/`, `slurm/`, -`htcondor-ce/`, …) and follow these conventions: +Dockerfiles for the stacks are in `tests/integration/stacks/_images//`, for example +`arc/`, `slurm/` or `htcondor-ce/`. They follow these rules. -1. **EL9 base** (`almalinux:9`), pinned by tag; backend daemons installed from **pinned RPMs**. -2. **Every version is a Renovate-annotated `ARG`** so upstream releases arrive as red-or-green - Renovate PRs: +1. **EL9 base.** Use `almalinux:9` and install the backend from pinned RPM versions. +2. **Each version is an `ARG` with a `# renovate:` comment.** Renovate reads the comment and + opens a pull request when a new version is released: ```dockerfile FROM almalinux:9 @@ -18,31 +18,23 @@ Dockerfiles under `tests/integration/stacks/_images/` are shared across stacks ( && dnf clean all ``` - The `dockerfile` + `custom.regex` managers that read these annotations are already enabled in - [`renovate.json`](../../../renovate.json) (per-backend grouping and the leading-edge/anchor - convention are documented in [the Renovate reference](renovate.md)) — **adding a stack needs no - `renovate.json` edit**, only the annotated `ARG`. The annotation carries its own `datasource`, - so the exact datasource for the NorduGrid/EL9 repos (still open) is chosen when - the ARC Dockerfile lands, without touching the manager config. -3. **Foreground entrypoint** — daemons run under a plain entrypoint script or minimal supervisor, - never systemd, and **log to stdout/stderr** so `docker compose logs` captures everything (a - file-logging daemon redirects its log to `/dev/stdout`). The CI failure artifact is compose - logs only; there is no mounted log-dir upload. -4. **Ephemeral credentials at start**: the entrypoint mints whatever the stack needs (ARC: - `arcctl test-ca` CA + host cert + client credential; HTCondor(-CE): IDTOKENS) and writes - client-side material world-readable into the host-readable `./credentials` bind mount - (`credentials//…`). A TLS host certificate carries a SAN for the name the - client dials (the compose hostname *and* `localhost` for host-run stacks); ARC on EL9 needs the - crypto-policy set to `LEGACY` for the proxy chain. -5. **Version windows**: where a backend declares an anchor + leading edge, the - image accepts the version as `ARG`/build-arg so the same Dockerfile serves every entry of the - `versions:` axis; only the leading-edge `ARG` default is Renovate-tracked. -6. **Roles and intra-stack secrets** (multi-service stacks). A CE + LRMS or a Slurm - controller + worker is one parameterized image, not several: a `case "$ROLE"` entrypoint - selects the daemon per service rather than duplicating Dockerfiles. Secrets shared *between* - services (a munge key, an internal CA) obey the same rule as client credentials — minted at - container start into a stack-internal volume, **never baked into the image** (a GHCR-published - image with a baked munge key is a leaked credential). + You do not need to change `renovate.json` for a new image. The comment gives Renovate + everything it needs. See [Renovate](renovate.md). +3. **No systemd, logs to stdout/stderr.** See rules 3 and 4 in + [Rules for a stack](integration-stacks.md#rules-for-a-stack). +4. **Credentials are created when the container starts**, by the entrypoint. For example ARC + creates a test CA, a host certificate and a client certificate with `arcctl test-ca`, and + HTCondor creates IDTOKENS. See rules 6 and 9 in + [Rules for a stack](integration-stacks.md#rules-for-a-stack). +5. **The version is a build argument**, so one Dockerfile can build every version listed in + `versions`. Renovate updates only the default value of the `ARG`. +6. **One image per backend, even with several services.** For example a Slurm controller and a + Slurm worker use the same image. The entrypoint chooses the daemon from a `ROLE` variable + (`case "$ROLE"`). +7. **Secrets shared between services are created at start too.** For example the Slurm munge key + or an internal CA. Write them to a volume inside the stack. Never put them in the image: the + images will be published, so a secret in an image is public. -Images are built and pushed to `ghcr.io/diracgrid/intercede-testenv/:` by the -`build-images` workflow; until that lands, stacks may `build:` locally from these directories. +Later, CI will build the images and publish them as +`ghcr.io/diracgrid/intercede-testenv/:` (#14). Until then, stacks build them +locally with `build:`. diff --git a/docs/dev/reference/integration-stacks.md b/docs/dev/reference/integration-stacks.md index 9375e83..fa8fb84 100644 --- a/docs/dev/reference/integration-stacks.md +++ b/docs/dev/reference/integration-stacks.md @@ -1,113 +1,120 @@ -# Integration stacks — layout, manifest schema, stack contract +# Integration stacks -Reference for the integration-test infrastructure. +## Words used here -## Layout +- **Stack**: one backend that the tests run against, for example `htcondor` or `ssh-slurm`. Each + stack is one entry in `tests/integration/stacks.yml`. +- **Compose environment**: the `compose.yml` file that starts the backend containers. It is + usually in `stacks//`. Several stacks can share one (see `compose` below). +- **Configuration**: one way to set up the backend, for example `basic`. It is a folder + `config//` with the backend's config files and `intercede-client.toml`, which tells the + tests how to connect. +- **CI job**: one stack, with one configuration, at one backend version. + +## Files ``` tests/integration/ -├── stacks.yml # THE manifest: single source of truth for the CI matrix -├── conftest.py # --stack/--config options, stack fixtures, capability skip -├── generate_matrix.py # manifest → CI matrix expansion (runnable locally) -├── test_manifest.py # manifest schema + layout checks (run in plain unit CI) -├── test_*.py # the backend-agnostic contract suite (Phase 2) +├── stacks.yml # list of stacks; CI creates its jobs from this file +├── conftest.py # pytest fixtures: stack, stack_dir, client_config +├── generate_matrix.py # turns stacks.yml into the list of CI jobs +├── test_manifest.py # checks stacks.yml and the stack folders (no Docker needed) └── stacks/ - ├── _template/ # copy me: compose.yml + config/basic/intercede-client.toml - ├── _images/ # shared Dockerfiles (see integration-images.md) - ├── / + ├── _template/ # copy this to start a new stack + ├── _images/ # Dockerfiles (see integration-images.md) + ├── htcondor/ # a stack with its own compose environment │ ├── compose.yml - │ └── config/ - │ └── basic/ # backend config + intercede-client.toml, changed together - ├── slurm/ # shared compose environment (`compose: slurm`) + │ └── config/basic/ # backend config files + intercede-client.toml + ├── slurm/ # a compose environment shared by two stacks │ ├── compose.yml - │ └── config/basic/ # backend config only (slurm.conf, cgroup.conf) - ├── ssh-slurm/config/basic/ # intercede-client.toml only: SSH transport - └── local-slurm/config/basic/ # intercede-client.toml only: Local transport + │ └── config/basic/ # backend config files only (slurm.conf, cgroup.conf) + ├── ssh-slurm/ + │ └── config/basic/ # intercede-client.toml only (connects over SSH) + └── local-slurm/ + └── config/basic/ # intercede-client.toml only (runs inside the container) ``` -## Manifest schema (`stacks.yml`) +## `stacks.yml` fields | Field | Required | Meaning | | --- | --- | --- | -| `id` | yes | Stack name; holds the stack's client config, and its compose environment unless `compose:` is set | -| `compose` | no | Directory under `stacks/` holding a compose environment shared by several stacks (e.g. `slurm` for `ssh-slurm` and `local-slurm`) | -| `configs` | yes | Configuration names under `stacks//config/`; start with `[basic]` | -| `versions` | no | Backend support-window entries to run; absent = single leading-edge pin | -| `markers` | yes | `pytest -m` expression selecting the applicable contract-suite subset | -| `exec_in_container` | no | Run pytest inside the backend container instead of against it (default `false`) | - -The CI matrix is the expansion `{stack × config × version}` produced by -`tests/integration/generate_matrix.py`; each matrix entry becomes its own CI job. Only -*harmonized* stacks — those with a `stacks//compose.yml` and their own -`stacks//config/` — become jobs; a -seeded-only entry is announced with a `::notice` by `generate_matrix.py` and skipped. - -**Shared compose environments.** `compose:` names a directory, not another stack, so stacks that -test one backend through different transports are peers: `ssh-slurm` and `local-slurm` both start -`stacks/slurm/compose.yml`, which owns the backend config. The *client configuration* is always -resolved from the entry's **own** id: each ships its own `stacks//config/basic/ -intercede-client.toml` (a different `kind`/endpoint — SSH vs Local transport), and `local-slurm`'s -`markers` exclude the SSH-transport tests (`scheduler and slurm and not ssh` — the pattern every -future `local-*` stack needs so an `ssh`-marked test does not run in both the ssh and local jobs). - -**`exec_in_container: true` runs pytest inside the batch container** (Local transport, no port 22). -The image must ship the suite + pytest — mounted or baked in (open TODO #13/#14) — and an -unprivileged test user, since HTCondor and Slurm refuse to run jobs as root; the workflow (and the -local `compose exec` recipe) pin that user with compose `user:` / `exec --user`. - -## The stack contract - -A stack that lands in `stacks//` commits to all of the following (enforced partly by -`test_manifest.py`, partly by review): - -1. **One `compose.yml`, runnable locally.** CI does nothing a developer cannot do on a laptop - (see [the how-to](../how-to/run-integration-tests-locally.md)). -2. **Healthchecks on every service.** `up --wait` gates on them; hand-rolled sleep-and-retry - loops are not accepted. -3. **Foreground daemons, never systemd.** A plain entrypoint or minimal supervisor — privileged - systemd containers are fragile on GitHub runners and hide daemon logs from `docker logs`. Daemons - **log to stdout/stderr** so `docker compose logs` is the whole failure story; a file-logging - daemon redirects its log to `/dev/stdout` rather than expecting a bespoke log-directory upload - (the CI failure artifact is compose logs only). -4. **Images from `stacks/_images/`,** EL9 base, pinned Renovate-annotated `ARG`s (see - [image conventions](integration-images.md)). -5. **Ephemeral credentials, minted at container start** into the shared, host-readable - `./credentials` bind mount (gitignored) — never committed, and never a Docker *named* volume - (host pytest cannot read a root-owned named volume for `exec_in_container: false` stacks). The - layout `credentials//…` is the interface between stack and harness. Mint - throwaway Test-CA material world-readable; the harness copies anything needing private modes - (0400/0600) into a private dir itself. -6. **Configurations are atomic directories.** `config//` holds the backend config *and* - `intercede-client.toml` together; a new auth/filesystem/queue variant is a new directory plus - a `configs:` entry in `stacks.yml`, never a mutation of `basic`. -7. **One `stacks.yml` entry.** That is how CI finds the stack — adding a stack requires no - workflow edits. -8. **Host reachability + TLS naming** (host-run stacks). A stack the harness drives from the host - (`exec_in_container: false`) publishes its backend port on localhost (compose `ports:`) and - records the same `host:port` in `intercede-client.toml`. A TLS backend's Test-CA host - certificate must carry a SAN for the name the client actually dials — mint it for both the - compose hostname and `localhost`. ARC on EL9 also needs the system crypto-policy set to `LEGACY` - for the proxy chain to validate. - -## Test harness - -Options and fixtures provided by `tests/integration/conftest.py`: - -| Name | Kind | Meaning | +| `id` | yes | Name of the stack. Its client config is in `stacks//config/`. | +| `compose` | no | Folder in `stacks/` with the `compose.yml` to use. Default: the `id`. | +| `configs` | yes | Configuration names, folders in `config/`. Start with `[basic]`. | +| `versions` | no | Backend versions to test, for example `[lts, latest]`. Default: `[latest]`. | +| `markers` | yes | `pytest -m` expression that selects the tests for this stack. | +| `exec_in_container` | no | `true`: run pytest inside the backend container. Default: `false`. | + +## When a stack gets CI jobs + +A stack can be listed in `stacks.yml` before its files exist, for example while the pull request +that adds it is still open. `generate_matrix.py` creates CI jobs for a stack only when both of +these exist: + +- `stacks//compose.yml` +- `stacks//config/` + +Otherwise it prints a notice and skips the stack. + +## Sharing a compose environment + +`ssh-slurm` and `local-slurm` test the same Slurm cluster. The only difference is how interCEde +connects to it. Both set `compose: slurm`, so both start `stacks/slurm/compose.yml`. Each stack has +its own `intercede-client.toml`. + +`local-slurm` uses the markers `scheduler and slurm and not ssh`. Without `and not ssh`, the SSH +tests would run in both jobs. Add the same `and not ssh` to every new `local-*` stack. + +## `exec_in_container: true` + +pytest runs inside the backend container, and interCEde talks to the scheduler directly, without +SSH. For this to work (still to do, see #13 and #14): + +- the image must contain pytest and the tests; +- pytest must run as a normal user, not root, because HTCondor and Slurm refuse to run jobs as + root. + +## Rules for a stack + +Review checks these rules. `test_manifest.py` checks the file layout. + +1. **It runs locally.** One `docker compose up --wait` starts it on a laptop, exactly as in CI. +2. **Every service has a healthcheck.** `up --wait` waits for the healthchecks. Do not use + `sleep` and retry loops. +3. **No systemd.** Start daemons with a simple entrypoint script, in the foreground. systemd + containers break easily on GitHub runners. +4. **Logs go to stdout/stderr.** On failure, CI saves only `docker compose logs`. If a daemon can + only write to a file, use `/dev/stdout` as that file. +5. **Images come from `stacks/_images/`.** See [image rules](integration-images.md). +6. **Credentials are created when the container starts.** Never commit them. Write them, readable + by everyone, to the `./credentials` bind mount, in `credentials//`. Do not use a Docker + named volume: it belongs to root, so pytest on the host cannot read it. These are throwaway test + credentials. The tests copy any file that needs strict permissions (0400/0600) to a private + folder. +7. **A new configuration is a new folder.** Backend config and `intercede-client.toml` change + together, in `config//`. To test something else (another auth method, queue, ...), add + a new folder and a new name in `configs`. Do not change `basic`. +8. **The stack is listed in `stacks.yml`.** CI needs nothing else. You do not change the + workflow. +9. **The host can reach it** (only when `exec_in_container` is `false`). Publish the backend port + on `localhost` (compose `ports:`) and put the same `host:port` in `intercede-client.toml`. A + TLS certificate must be valid for the name the client uses: create it for both the compose + hostname and `localhost`. ARC on EL9 also needs the crypto policy set to `LEGACY`. + +## pytest options and fixtures + +Defined in `tests/conftest.py` (options) and `tests/integration/conftest.py` (fixtures): + +| Name | Type | Meaning | | --- | --- | --- | -| `--stack` | option | Stack id from `stacks.yml`; stack-bound tests self-skip when absent (registered in `tests/conftest.py`, so every `pytest` invocation form accepts it) | -| `--config` | option | Configuration name (default `basic`) | -| `stack` | fixture | The selected manifest entry | -| `stack_dir` | fixture | Directory with the stack's `compose.yml` (`stacks//` when set) | -| `client_config` | fixture | Parsed `intercede-client.toml`, resolved under the entry's *own* id (never `compose:`) | - -Markers, registered in `pyproject.toml`: `remote`, `scheduler`, `arc`, `htcondor`, `slurm`, -`ssh`, `destructive_fetch`. Each stack's `markers:` expression selects its subset. The -capability-narrowing skip helper (`isinstance` against a `runtime_checkable` protocol) lands with -the Phase-2 contract suite that calls it, alongside the interCEde protocols — it is intentionally -not part of this scaffold, whose signature would otherwise be guessed ahead of its callers. - -A *selected* stack whose `intercede-client.toml` is missing **fails** under CI (`$CI` set) rather -than skipping: an all-skipped run would report a green integration cell that verified nothing. -Locally the same case skips, which stays convenient while a stack is being -brought up. +| `--stack` | option | Stack `id` to test. Without it, tests that need a stack are skipped. | +| `--config` | option | Configuration name. Default: `basic`. | +| `stack` | fixture | The stack's entry from `stacks.yml`. | +| `stack_dir` | fixture | Folder with the stack's `compose.yml`. | +| `client_config` | fixture | The stack's `intercede-client.toml`, already parsed. | + +If `intercede-client.toml` is missing, the test fails in CI (when `$CI` is set) and is skipped +locally. In CI, a job where every test is skipped would look green but test nothing. + +Markers, defined in `pyproject.toml`: `remote`, `scheduler`, `arc`, `htcondor`, `slurm`, `ssh`, +`destructive_fetch`. diff --git a/docs/dev/reference/renovate.md b/docs/dev/reference/renovate.md index d5e3192..6b36b55 100644 --- a/docs/dev/reference/renovate.md +++ b/docs/dev/reference/renovate.md @@ -9,8 +9,8 @@ The configuration is defined in [`renovate.json`](../../../renovate.json). 2. **Testing dependencies** — Pytest and related testing tools 3. **Build dependencies** — Hatchling and build tools 4. **GitHub Actions** — GitHub Actions workflow dependencies -5. **Integration-stack backends** — ARC, HTCondor(-CE), and Slurm daemon versions, grouped per - backend so a red bump PR names its culprit (see below) +5. **Integration-stack backends** — ARC, HTCondor(-CE) and Slurm versions, one group per + backend, so a failing update shows which backend caused it (see below) ## Update strategy @@ -20,35 +20,32 @@ The configuration is defined in [`renovate.json`](../../../renovate.json). - **Grouping**: related dependencies are grouped together - **Major versions**: handled separately for better control - **Automerge**: disabled (requires manual review) -- **Security**: vulnerability alerts are enabled; security updates are labeled and prioritized +- **Security**: vulnerability alerts are enabled; security updates are labeled and pri## Integration stack backends -## Integration stack backends +The integration stacks pin the version of each backend they test. Renovate opens a pull request +when a new version is released. If the tests fail, we know the new version is the cause. -The integration stacks pin the backend daemon versions they test against, and Renovate tracks those -pins so a new upstream release arrives as a reviewable red-or-green PR rather than tribal knowledge. -Three managers are enabled for this: +Renovate finds the versions in three places: -- **`docker-compose`** — upstream image tags referenced directly in a stack's `compose.yml` - (e.g. `htcondor/mini:`). -- **`dockerfile`** — base images in `tests/integration/stacks/_images/*/Dockerfile`. -- **`custom.regex`** — pinned RPM/package versions carried as `# renovate:`-annotated `ARG`s in - those Dockerfiles. The annotation itself declares the datasource, depName and (optionally) - versioning: +- **`docker-compose`**: image tags in a stack's `compose.yml` (for example `htcondor/mini:`). +- **`dockerfile`**: base images in `tests/integration/stacks/_images/*/Dockerfile`. +- **`custom.regex`**: package versions in those Dockerfiles, written as an `ARG` with a + `# renovate:` comment. The comment tells Renovate where to look for new versions: ```dockerfile # renovate: datasource=repology depName=fedora_epel_9/nordugrid-arc ARG ARC_VERSION=7.1.1 ``` -**Adding a stack needs no `renovate.json` edit.** A stack author writes the annotated `ARG`; the -manager and per-backend grouping already configured here pick it up. Bumps are grouped per backend -(`arc`, `htcondor`, `slurm`) and never mixed. +**A new stack needs no change to `renovate.json`.** Write the `ARG` with its comment and Renovate +finds it. Updates are grouped per backend (`arc`, `htcondor`, `slurm`), never mixed. -**Version windows are half-automatic.** Renovate tracks the *leading-edge* `ARG` normally. -The *older anchor* versions (e.g. an HTCondor LTS line) are human-managed: pin them and add a -`packageRule` with `allowedVersions`/`matchCurrentVersion` (or `enabled: false`) — landed alongside -the stack that introduces the anchor — so Renovate offers only in-major patch bumps and never -silently moves the support window. +**Only the newest version is updated automatically.** Older versions that we still support (for +example an HTCondor LTS version in `versions: [lts, latest]`) are updated by hand. When you add +such a version, also add a `packageRule` (`allowedVersions` or `matchCurrentVersion`, or +`enabled: false`) so that Renovate proposes only patch updates for it. + + window. ## How it works diff --git a/docs/dev/tutorials/index.md b/docs/dev/tutorials/index.md index 9b9d039..a156720 100644 --- a/docs/dev/tutorials/index.md +++ b/docs/dev/tutorials/index.md @@ -1,6 +1,6 @@ # Tutorials -None yet — the first candidate, once a stack has landed, is *"Run your first integration stack -locally"* (boot a stack, submit a job through the native client tools, watch it complete). +None yet. Once a stack exists, a first tutorial could be "Run your first integration stack": +start a stack, submit a job with the backend's own tools, and watch it finish. -For task-oriented material in the meantime, see the [how-to guides](../how-to/index.md). +Until then, see the [how-to guides](../how-to/index.md). diff --git a/tests/integration/README.md b/tests/integration/README.md index 8c627ae..2bbbadf 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,19 +1,17 @@ # Integration tests -Backend-agnostic tests that run against real, containerised backend daemons ("stacks"). - -Quickstart: +Tests that run against real backends (ARC, HTCondor, Slurm) in containers. Each backend setup is +called a *stack*. ```sh docker compose -f tests/integration/stacks//compose.yml up --wait pixi run -e py313 pytest tests/integration -m "" --stack= --config=basic ``` -Documentation lives in [`docs/dev/`](../../docs/dev/index.md): +Documentation: -- How-to: [run the tests locally](../../docs/dev/how-to/run-integration-tests-locally.md) · - [add a stack](../../docs/dev/how-to/add-an-integration-stack.md) (incl. the harmonization - checklist) -- Reference: [layout, manifest schema, stack contract](../../docs/dev/reference/integration-stacks.md) · - [image conventions](../../docs/dev/reference/integration-images.md) -- Explanation: [why integration stacks](../../docs/dev/explanations/integration-testing.md) +- [Run the tests locally](../../docs/dev/how-to/run-integration-tests-locally.md) +- [Add a stack](../../docs/dev/how-to/add-an-integration-stack.md) +- [Integration stacks: files, `stacks.yml` fields, rules](../../docs/dev/reference/integration-stacks.md) +- [Image rules](../../docs/dev/reference/integration-images.md) +- [Why we test against real backends](../../docs/dev/explanations/integration-testing.md) diff --git a/tests/integration/generate_matrix.py b/tests/integration/generate_matrix.py index afad1fa..b812193 100644 --- a/tests/integration/generate_matrix.py +++ b/tests/integration/generate_matrix.py @@ -35,7 +35,7 @@ def compose_env(entry: dict[str, Any]) -> str: return entry.get("compose", entry["id"]) -def is_harmonized(entry: dict[str, Any]) -> bool: +def is_ready(entry: dict[str, Any]) -> bool: """Return whether both the stack's compose.yml and its own config/ have landed.""" return (STACKS_DIR / compose_env(entry) / "compose.yml").is_file() and ( STACKS_DIR / entry["id"] / "config" @@ -76,13 +76,13 @@ def main() -> None: ) args = parser.parse_args() - harmonized: list[dict[str, Any]] = [] - seeded: list[dict[str, Any]] = [] + ready: list[dict[str, Any]] = [] + not_ready: list[dict[str, Any]] = [] for entry in load_manifest(): - (harmonized if is_harmonized(entry) else seeded).append(entry) - matrix = build_matrix(harmonized) + (ready if is_ready(entry) else not_ready).append(entry) + matrix = build_matrix(ready) - for entry in seeded: + for entry in not_ready: note = f"stack {entry.get('id')!r} has no compose.yml or config/ yet - skipping" if args.github_output: print(f"::notice::{note}") diff --git a/tests/integration/stacks/_images/README.md b/tests/integration/stacks/_images/README.md index ba8694c..05b3bf3 100644 --- a/tests/integration/stacks/_images/README.md +++ b/tests/integration/stacks/_images/README.md @@ -1,7 +1,4 @@ -# Shared stack images +# Stack images -Dockerfiles shared across stacks (`arc/`, `slurm/`, `htcondor-ce/`, …). - -Conventions (EL9 base, pinned Renovate-annotated `ARG`s, foreground daemons, ephemeral -credentials, GHCR publishing) are documented in -[docs/dev/reference/integration-images.md](../../../../docs/dev/reference/integration-images.md). +Dockerfiles for the integration stacks (`arc/`, `slurm/`, `htcondor-ce/`, ...). See the +[image rules](../../../../docs/dev/reference/integration-images.md).