From 7d4d8f4278309e393ff4588a4c396f77053de355 Mon Sep 17 00:00:00 2001 From: Dev <9416650+aoinoikaz@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:00:39 -0500 Subject: [PATCH] CI: the server is deployed as its own user, sandboxed, on loopback The unit ran the server as ubuntu, the host's login user with sudo, with no sandbox, and the server listened on every interface, behind a firewall rule alone. systemd-analyze scored the unit 9.2 of 10 for exposure. deploy/nova-cache.service runs it as nova-cache, a user that deploy/nova-cache.sysusers declares, with one writable directory, the signing key as a credential from a file only root reads, and loopback as its only network. It scores 0.9. deploy/smoke-test.sh starts the server under the unit on the runner, the host's Ubuntu release, and checks that it serves, reads its key, takes an authenticated write, refuses another, takes over a store another user owned, and does not answer off loopback. The job runs on every pull request and deploys on a push to main, so a deploy installs only what just ran, and it fails if the server does not answer afterwards. Caddy's packaged unit names no restart policy. The deploy installs one. --- .github/workflows/ci.yml | 66 +++++++++++------------- deploy/caddy-restart.conf | 5 ++ deploy/nova-cache.service | 61 ++++++++++++++++++++++ deploy/nova-cache.sysusers | 2 + deploy/smoke-test.sh | 102 +++++++++++++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 36 deletions(-) create mode 100644 deploy/caddy-restart.conf create mode 100644 deploy/nova-cache.service create mode 100644 deploy/nova-cache.sysusers create mode 100755 deploy/smoke-test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20ca26c..dca6098 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -345,16 +345,19 @@ jobs: run: cabal test nova-cache-bzip2-test --enable-tests # --------------------------------------------------------------------------- - # Deploy (main branch only): build on the hosted arm64 image, ship the - # binary. anv26 is an Ampere box on the same Ubuntu 24.04, so the - # runner's glibc matches - and the production host deliberately carries - # no compiler toolchain. + # Server (arm64): build the server on the hosted arm64 image, start it + # there under deploy/nova-cache.service, and on a push to main ship both + # to the public cache's host. anv26 is an Ampere box on the same Ubuntu + # 24.04, so the runner's glibc matches and its systemd is the release the + # host runs, and the production host deliberately carries no compiler + # toolchain. The smoke test runs on every pull request, so a unit the + # server cannot run under fails before it merges, and a deploy installs + # only what just ran. # --------------------------------------------------------------------------- - deploy-server: - name: Deploy Server + server: + name: Server (arm64) needs: [build] - if: github.ref == 'refs/heads/main' && github.event_name == 'push' runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v7 @@ -377,7 +380,11 @@ jobs: cabal build --flag server --ghc-options="-Wall -Werror" cp "$(cabal list-bin --flag server nova-cache-server)" /tmp/nova-cache-server + - name: Run the server under its unit + run: deploy/smoke-test.sh /tmp/nova-cache-server + - name: Deploy to OCI + if: github.ref == 'refs/heads/main' && github.event_name == 'push' env: OCI_SSH_KEY: ${{ secrets.OCI_SSH_KEY }} CACHE_API_KEY: ${{ secrets.NOVA_CACHE_API_KEY }} @@ -407,35 +414,12 @@ jobs: StrictHostKeyChecking accept-new SSHCFG - # Render the service unit. The API key is not embedded in the - # unit: systemd loads it from /etc/nix-cache/env (written below, - # root-only), so the unit file carries no credentials. - cat > /tmp/nova-cache.service <<'EOF' - [Unit] - Description=Nova Cache Server - After=network.target - - [Service] - Type=simple - User=ubuntu - Group=ubuntu - ExecStart=/usr/local/bin/nova-cache-server --port 5000 --store /var/lib/nix-cache - Restart=always - RestartSec=5 - Environment=PORT=5000 - Environment=NIX_CACHE_DIR=/var/lib/nix-cache - Environment=SIGNING_KEY_FILE=/etc/nix-cache/secret-key - EnvironmentFile=/etc/nix-cache/env - - [Install] - WantedBy=multi-user.target - EOF - - scp /tmp/nova-cache.service anv26:/tmp/nova-cache.service + # The unit, the user it runs as and Caddy's restart policy are the + # files of deploy/, the first two as the smoke test above ran them. # The binary rides the same Access tunnel as everything else; the # service keeps serving during the transfer and stops only for # the install below. - scp /tmp/nova-cache-server anv26:/tmp/nova-cache-server + scp deploy/nova-cache.service deploy/nova-cache.sysusers deploy/caddy-restart.conf /tmp/nova-cache-server anv26:/tmp/ # Provision the API key as a root-only environment file. The value # travels over stdin, so it appears in no command line, no log, and @@ -443,11 +427,21 @@ jobs: printf 'CACHE_API_KEY=%s\n' "$CACHE_API_KEY" | ssh anv26 \ "sudo sh -c 'umask 077; mkdir -p /etc/nix-cache; cat > /etc/nix-cache/env; chown root:root /etc/nix-cache/env; chmod 600 /etc/nix-cache/env'" + # The signing key becomes root's: the unit hands the service a copy + # as a credential, so the service's user does not read the file. + # The store changes hands when the unit starts. The last command + # fails the deploy if the server does not answer. ssh anv26 "\ + sudo install -D -m 0644 /tmp/nova-cache.sysusers /etc/sysusers.d/nova-cache.conf && \ + sudo systemd-sysusers /etc/sysusers.d/nova-cache.conf && \ + sudo chown root:root /etc/nix-cache/secret-key && \ + sudo chmod 600 /etc/nix-cache/secret-key && \ + sudo install -D -m 0644 /tmp/caddy-restart.conf /etc/systemd/system/caddy.service.d/restart.conf && \ sudo systemctl stop nova-cache.service && \ sudo install -m 0755 /tmp/nova-cache-server /usr/local/bin/nova-cache-server && \ - sudo cp /tmp/nova-cache.service /etc/systemd/system/nova-cache.service && \ - rm -f /tmp/nova-cache-server /tmp/nova-cache.service && \ + sudo install -m 0644 /tmp/nova-cache.service /etc/systemd/system/nova-cache.service && \ + rm -f /tmp/nova-cache-server /tmp/nova-cache.service /tmp/nova-cache.sysusers /tmp/caddy-restart.conf && \ sudo systemctl daemon-reload && \ sudo systemctl start nova-cache.service && \ - sudo systemctl status nova-cache.service --no-pager" + sudo systemctl status nova-cache.service --no-pager && \ + curl --fail --silent --show-error --output /dev/null --retry 10 --retry-connrefused --retry-delay 1 http://127.0.0.1:5000/nix-cache-info" diff --git a/deploy/caddy-restart.conf b/deploy/caddy-restart.conf new file mode 100644 index 0000000..d3c1fc2 --- /dev/null +++ b/deploy/caddy-restart.conf @@ -0,0 +1,5 @@ +# Caddy's packaged unit names no restart policy, so a Caddy that exits stays +# down, and the public cache with it. +[Service] +Restart=on-failure +RestartSec=5 diff --git a/deploy/nova-cache.service b/deploy/nova-cache.service new file mode 100644 index 0000000..b1310c1 --- /dev/null +++ b/deploy/nova-cache.service @@ -0,0 +1,61 @@ +# nova-cache-server on the public cache's host: its own user, one writable +# directory, and loopback alone. The deploy job installs this file after +# deploy/smoke-test.sh has started the server under it. + +[Unit] +Description=Nova Cache Server +After=network.target + +[Service] +Type=simple +User=nova-cache +Group=nova-cache +ExecStart=/usr/local/bin/nova-cache-server --host 127.0.0.1 --port 5000 --store %S/nix-cache +Restart=always +RestartSec=5 + +# /var/lib/nix-cache. systemd creates it if it is absent and gives it, with +# everything under it, to the service's user, whoever owned it before. +StateDirectory=nix-cache + +# The signing key reaches the service as a credential: systemd reads the file +# as root and shows a copy to this unit alone, so the file stays root-only. +# The write key comes from an environment file only root reads. +LoadCredential=signing-key:/etc/nix-cache/secret-key +Environment=SIGNING_KEY_FILE=%d/signing-key +EnvironmentFile=/etc/nix-cache/env + +# The reverse proxy on the same host is the service's only client. +IPAddressDeny=any +IPAddressAllow=localhost +RestrictAddressFamilies=AF_INET AF_INET6 + +# The service reads its binary and its two keys, writes its state directory +# and listens on one port. +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +PrivateUsers=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +ProtectProc=invisible +ProcSubset=pid +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RemoveIPC=yes +CapabilityBoundingSet= +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallFilter=~@privileged @resources + +[Install] +WantedBy=multi-user.target diff --git a/deploy/nova-cache.sysusers b/deploy/nova-cache.sysusers new file mode 100644 index 0000000..079be3d --- /dev/null +++ b/deploy/nova-cache.sysusers @@ -0,0 +1,2 @@ +# The user nova-cache.service runs as. +u nova-cache - "nova-cache binary cache server" /var/lib/nix-cache /usr/sbin/nologin diff --git a/deploy/smoke-test.sh b/deploy/smoke-test.sh new file mode 100755 index 0000000..d1011e8 --- /dev/null +++ b/deploy/smoke-test.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Start nova-cache.service on this machine as the deploy installs it, and +# check what the unit's sandbox could break: that the server starts, reads +# its signing key, accepts an authenticated write and refuses another, that +# a store another user owned becomes the service's, and that the server +# answers on loopback alone. +# +# The keys are made here and stand in for the host's. The machine is a +# throwaway: the script installs into /usr/local/bin, /etc and /var/lib and +# removes nothing. +# +# Usage: smoke-test.sh SERVER +# SERVER the nova-cache-server binary to install +set -euo pipefail + +server=${1:?the nova-cache-server binary} +here=$(cd "$(dirname "$0")" && pwd) + +readonly PORT=5000 +readonly STORE=/var/lib/nix-cache +readonly SIGNING_KEY_NAME=smoke-test-1 +readonly WRITE_KEY=smoke-test-write-key +# A NAR's file name is its hash in base 32; the server stores any name of +# that shape. +readonly NAR=0000000000000000000000000000000000000000000000000000.nar +readonly BODY='smoke test' + +base=http://127.0.0.1:$PORT + +pass() { echo "ok $*"; } +fail() { + echo "FAIL $*" >&2 + sudo journalctl -u nova-cache.service --no-pager -n 40 >&2 || true + exit 1 +} + +# A store another user owns, as the host's is before this unit's first +# deploy there. +sudo mkdir -p "$STORE/nar" +echo kept | sudo tee "$STORE/nar/pre-existing" > /dev/null +sudo chown -R "$(id -un)" "$STORE" + +# An Ed25519 key as Nix writes one: the name, then the seed and the public +# key in base 64. Each is the last 32 bytes of its DER encoding. +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +openssl genpkey -algorithm ed25519 -out "$work/key.pem" +{ + printf '%s:' "$SIGNING_KEY_NAME" + { + openssl pkey -in "$work/key.pem" -outform DER | tail -c 32 + openssl pkey -in "$work/key.pem" -pubout -outform DER | tail -c 32 + } | base64 -w 0 + echo +} > "$work/secret-key" +sudo install -D -m 0600 -o root -g root "$work/secret-key" /etc/nix-cache/secret-key +printf 'CACHE_API_KEY=%s\n' "$WRITE_KEY" | + sudo sh -c 'umask 077; cat > /etc/nix-cache/env' + +sudo install -m 0755 "$server" /usr/local/bin/nova-cache-server +sudo install -D -m 0644 "$here/nova-cache.sysusers" /etc/sysusers.d/nova-cache.conf +sudo systemd-sysusers /etc/sysusers.d/nova-cache.conf +sudo install -m 0644 "$here/nova-cache.service" /etc/systemd/system/nova-cache.service +sudo systemctl daemon-reload +sudo systemctl start nova-cache.service + +curl --fail --silent --show-error --output /dev/null \ + --retry 10 --retry-connrefused --retry-delay 1 "$base/nix-cache-info" || + fail "the server does not answer on loopback" +[ "$(systemctl is-active nova-cache.service)" = active ] || fail "the service is not active" +pass "the server runs under the unit and answers on loopback" + +# The landing page prints the public key of the signing key it loaded. +curl --fail --silent --show-error "$base/" | grep -q -F "$SIGNING_KEY_NAME:" || + fail "the landing page does not name the signing key, so the server did not read it" +pass "the server read its signing key" + +code=$(printf '%s' "$BODY" | curl --silent --output /dev/null --write-out '%{http_code}' \ + -X PUT -H "Authorization: Bearer $WRITE_KEY" --data-binary @- "$base/nar/$NAR") +[ "$code" = 200 ] || fail "an authenticated write answered $code" +[ "$(curl --fail --silent --show-error "$base/nar/$NAR")" = "$BODY" ] || + fail "the written NAR does not read back" +pass "an authenticated write is stored and read back" + +code=$(printf '%s' "$BODY" | curl --silent --output /dev/null --write-out '%{http_code}' \ + -X PUT -H "Authorization: Bearer wrong" --data-binary @- "$base/nar/1${NAR#0}") +[ "$code" = 401 ] || fail "a write with the wrong key answered $code" +pass "a write with the wrong key is refused" + +for path in "$STORE" "$STORE/nar/pre-existing" "$STORE/nar/$NAR"; do + owner=$(sudo stat -c %U "$path") + [ "$owner" = nova-cache ] || fail "$path belongs to $owner" +done +pass "the store another user owned is the service's" + +address=$(hostname -I | awk '{ print $1 }') +if curl --silent --max-time 3 --output /dev/null "http://$address:$PORT/nix-cache-info"; then + fail "the server answers on $address" +fi +pass "the server does not answer on $address" + +systemd-analyze security nova-cache.service --no-pager | tail -n 1