feat: integration test scaffolding

.github/workflows/integration.yml

@@ -0,0 +1,91 @@
+# Matrix comes from tests/integration/stacks.yml. Not a required check until #14.
+name: Integration tests
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  pull_request:
+    paths:
+      - "tests/integration/**"
+      - ".github/workflows/integration.yml"
+      - "tests/conftest.py"
+      - "pyproject.toml"
+
+permissions:
+  contents: read
+
+jobs:
+  matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.gen.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Generate {stack x config x version} matrix from stacks.yml
+        id: gen
+        run: python3 tests/integration/generate_matrix.py --github-output
+
+  integration:
+    needs: matrix
+    if: needs.matrix.outputs.matrix != '[]'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.matrix.outputs.matrix) }}
+    name: ${{ matrix.stack }} / ${{ matrix.config }} / ${{ matrix.version }}
+    env:
+      COMPOSE_FILE: tests/integration/stacks/${{ matrix.compose_env }}/compose.yml
+      BACKEND_VERSION: ${{ matrix.version }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Start the stack (waits for the healthchecks)
+        run: docker compose -f "$COMPOSE_FILE" up --wait
+
+      - name: Apply workarounds
+        if: matrix.exec_in_container == false
+        run: |
+          # Workaround for https://github.com/prefix-dev/pixi/issues/3762
+          sed -i.bak 's@editable = true@editable = false@g' pyproject.toml
+          rm pyproject.toml.bak
+      - uses: prefix-dev/setup-pixi@ad1da6449d9e506fd127fd67ab2d054a769f6a94 # v0.11.2
+        if: matrix.exec_in_container == false
+        with:
+          pixi-version: v0.82.0
+          cache: false
+          environments: py313
+
+      - name: Run the contract suite
+        run: |
+          if [ "${{ matrix.exec_in_container }}" = "true" ]; then
+            # TODO(#13/#14): ship the suite in the image and exec as an unprivileged user.
+            runner=(docker compose -f "$COMPOSE_FILE" exec -T backend pytest)
+          else
+            runner=(pixi run -e py313 pytest)
+          fi
+          set +e
+          "${runner[@]}" tests/integration -m "${{ matrix.markers }}" \
+            --stack="${{ matrix.stack }}" --config="${{ matrix.config }}"
+          rc=$?
+          # Exit 5 = no tests selected; tolerated until the contract suite lands.
+          if [ "$rc" -eq 5 ]; then
+            echo "::notice::stack '${{ matrix.stack }}': no tests matched markers '${{ matrix.markers }}' (pytest exit 5) - passing until the contract suite lands"
+            rc=0
+          fi
+          exit "$rc"
+
+      - name: Collect daemon logs
+        if: failure()
+        run: docker compose -f "$COMPOSE_FILE" logs --no-color > compose-logs.txt || true
+
+      - name: Upload logs as artifacts
+        if: failure()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: logs-${{ matrix.stack }}-${{ matrix.config }}-${{ matrix.version }}
+          path: compose-logs.txt

.gitignore

@@ -76,6 +76,9 @@ htmlcov/
 #remove in case we want a specific LHCb one
 .pylintrc
 
+# Integration-stack credentials: minted per run into a bind mount, never committed
+tests/integration/stacks/*/credentials/
+
 # docs (auto-generated build output only; docs/assets is tracked)
 docs/_build/
 site/

docs/dev/explanations/index.md

@@ -0,0 +1,3 @@
+# Explanations
+
+- [Why we test against real backends](integration-testing.md)

docs/dev/explanations/integration-testing.md

@@ -0,0 +1,34 @@
+# Why we test against real backends
+
+## The problem with fakes
+
+The unit tests check interCEde against fake backends. A fake can agree with our code while the
+real service behaves differently. Many bugs that interCEde must handle come from the real services:
+behaviour that changes between versions, output that can be downloaded only once, unusual job
+states. Unit tests cannot find these bugs.
+
+Example: since HTCondor 25.8, downloading the output of a spooled job deletes it on the server. You
+can download it only once. A fake HTCondor does not do this.
+
+## What the integration tests do
+
+The integration tests start real backend services (ARC, HTCondor, Slurm) in containers and run
+on pull requests. Each backend version is pinned. When a new version is released, Renovate opens a
+pull request that updates it. If the new version breaks something, that pull request fails, and we
+know which version caused the problem.
+
+## Optional features
+
+Some backends do not support every feature. The tests check for a feature in the same way that
+users of the library do (`isinstance` against a `runtime_checkable` protocol), and skip the test
+when the backend does not have it. The helper for this will be added in phase 2, together with the
+first tests that use it.
+
+## Phases
+
+- **Phase 0 (now).** The interCEde protocols do not exist yet. Each stack is tested with the
+  backend's own command-line tools (`arcsub`, `arcstat`, `arcget`, `condor_submit`, `sbatch`).
+- **Phase 2.** One set of tests, written against the interCEde protocols, runs on every stack
+  (`test_submit.py`, `test_status.py`, `test_output.py`, `test_kill.py`). It replaces the phase 0
+  tests. Behaviour that only one backend has (for example HTCondor's one-time output download)
+  gets its own test and marker (`destructive_fetch`), not an `if` inside a shared test.

docs/dev/how-to/add-an-integration-stack.md

@@ -0,0 +1,42 @@
+# Add an integration stack
+
+Words like *stack*, *compose environment* and *configuration* are explained in
+[Integration stacks](../reference/integration-stacks.md#words-used-here).
+
+You do not need to change any CI workflow. CI finds the stack in `stacks.yml`.
+
+## Steps
+
+1. **Copy the template.** Copy `tests/integration/stacks/_template/` to
+   `tests/integration/stacks/<id>/`. Then fill in:
+   - `compose.yml`: a real healthcheck, and the `./credentials` bind mount;
+   - `config/basic/`: the backend config files and `intercede-client.toml`.
+
+   If your stack uses a backend that another stack already starts (for example a second way to
+   connect to Slurm), do not copy `compose.yml`. Set `compose:` to the existing folder, and add
+   only `config/basic/intercede-client.toml`. See
+   [Sharing a compose environment](../reference/integration-stacks.md#sharing-a-compose-environment).
+2. **Add the image** in `tests/integration/stacks/_images/<name>/`. Follow the
+   [image rules](../reference/integration-images.md).
+3. **Add the stack to `tests/integration/stacks.yml`**: `id`, `configs`, `markers`, and if needed
+   `compose`, `versions`, `exec_in_container`.
+4. **Test it locally:**
+
+   ```sh
+   docker compose -f tests/integration/stacks/<id>/compose.yml up --wait
+   pixi run -e py313 pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+   pixi run -e py313 pytest tests/integration/test_manifest.py
+   ```
+
+## Checklist for the pull request
+
+- [ ] The stack follows the [rules for a stack](../reference/integration-stacks.md#rules-for-a-stack).
+- [ ] `docker compose ... up --wait` works on your machine.
+- [ ] The images use Docker, not Podman.
+- [ ] `intercede-client.toml` is filled in: `id`, endpoint, queue, credential type and path.
+- [ ] `pytest tests/integration/test_manifest.py` passes.
+- [ ] At least one test runs for this stack (phase 0: a test that uses the backend's own
+      command-line tools). Without one, the CI job passes but tests nothing.
+- [ ] If your pull request had its own workflow for this stack, remove it. `integration.yml` runs
+      on every pull request that changes `tests/integration/`. It does not block merging yet
+      (#14), so keep your own workflow only if you need a check that blocks merging.

docs/dev/how-to/index.md

@@ -0,0 +1,4 @@
+# How-to guides
+
+- [Run the integration tests locally](run-integration-tests-locally.md)
+- [Add an integration stack](add-an-integration-stack.md)

docs/dev/how-to/run-integration-tests-locally.md

@@ -0,0 +1,51 @@
+# Run the integration tests locally
+
+CI runs the same commands as below, so you can reproduce any CI job on your machine.
+
+## Run the tests for one stack
+
+1. Start the stack. The command returns when all healthchecks pass, so the backend is ready:
+
+   ```sh
+   docker compose -f tests/integration/stacks/<compose>/compose.yml up --wait
+   ```
+
+   `<compose>` is the stack's `compose` value in `tests/integration/stacks.yml`, or its `id` if it
+   has none.
+
+2. Run the tests. Copy `<markers>` from the stack's entry in `stacks.yml`:
+
+   ```sh
+   pixi run -e py313 pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+   ```
+
+For stacks with `exec_in_container: true` (the `local-*` stacks), run the tests inside the
+container instead:
+
+```sh
+docker compose -f tests/integration/stacks/<compose>/compose.yml exec --user <test-user> backend \
+  pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+```
+
+This does not work yet (#13, #14): the image must first contain pytest and the tests, and have a
+normal (not root) user. `--stack` is the stack `id` (for example `local-slurm`), not the compose
+folder (`slurm`).
+
+You can also leave a stack running and use it to develop against a real backend: point your client
+config at `localhost`.
+
+## See which CI jobs will run
+
+```sh
+pixi run -e py313 python tests/integration/generate_matrix.py
+```
+
+This prints the jobs that CI will create from `stacks.yml`.
+
+## Check `stacks.yml` and the stack folders
+
+No Docker needed:
+
+```sh
+pixi run -e py313 pytest tests/integration/test_manifest.py
+```

docs/dev/index.md

@@ -0,0 +1,11 @@
+# Developer documentation
+
+The documentation is organised following [Diátaxis](https://diataxis.fr/), like the other
+DiracX projects:
+
+- [Tutorials](tutorials/index.md): lessons to learn step by step
+- [How-to guides](how-to/index.md): steps for one task
+- [Reference](reference/index.md): technical details
+- [Explanations](explanations/index.md): why things are done this way
+
+User documentation (`docs/user/`) will come once the library has a public API.

docs/dev/reference/index.md

@@ -0,0 +1,5 @@
+# Reference
+
+- [Integration stacks: files, `stacks.yml` fields, rules](integration-stacks.md)
+- [Integration images: Dockerfile rules](integration-images.md)
+- [Renovate: automatic version updates](renovate.md)

docs/dev/reference/integration-images.md

@@ -0,0 +1,40 @@
+# Integration images
+
+Dockerfiles for the stacks are in `tests/integration/stacks/_images/<name>/`, for example
+`arc/`, `slurm/` or `htcondor-ce/`. They follow these rules.
+
+1. **EL9 base.** Use `almalinux:9` and install the backend from pinned RPM versions.
+2. **Each version is an `ARG` with a `# renovate:` comment.** Renovate reads the comment and
+   opens a pull request when a new version is released:
+
+   ```dockerfile
+   FROM almalinux:9
+
+   # renovate: datasource=repology depName=fedora_epel_9/nordugrid-arc
+   ARG ARC_VERSION=7.1.1
+
+   RUN dnf install -y epel-release \
+       && dnf install -y "nordugrid-arc-arex-${ARC_VERSION}" \
+       && dnf clean all
+   ```
+
+   You do not need to change `renovate.json` for a new image. The comment gives Renovate
+   everything it needs. See [Renovate](renovate.md).
+3. **No systemd, logs to stdout/stderr.** See rules 3 and 4 in
+   [Rules for a stack](integration-stacks.md#rules-for-a-stack).
+4. **Credentials are created when the container starts**, by the entrypoint. For example ARC
+   creates a test CA, a host certificate and a client certificate with `arcctl test-ca`, and
+   HTCondor creates IDTOKENS. See rules 6 and 9 in
+   [Rules for a stack](integration-stacks.md#rules-for-a-stack).
+5. **The version is a build argument**, so one Dockerfile can build every version listed in
+   `versions`. Renovate updates only the default value of the `ARG`.
+6. **One image per backend, even with several services.** For example a Slurm controller and a
+   Slurm worker use the same image. The entrypoint chooses the daemon from a `ROLE` variable
+   (`case "$ROLE"`).
+7. **Secrets shared between services are created at start too.** For example the Slurm munge key
+   or an internal CA. Write them to a volume inside the stack. Never put them in the image: the
+   images will be published, so a secret in an image is public.
+
+Later, CI will build the images and publish them as
+`ghcr.io/diracgrid/intercede-testenv/<name>:<version>` (#14). Until then, stacks build them
+locally with `build:`.