diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20ca26c..dca6098 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -345,16 +345,19 @@ jobs: run: cabal test nova-cache-bzip2-test --enable-tests # --------------------------------------------------------------------------- - # Deploy (main branch only): build on the hosted arm64 image, ship the - # binary. anv26 is an Ampere box on the same Ubuntu 24.04, so the - # runner's glibc matches - and the production host deliberately carries - # no compiler toolchain. + # Server (arm64): build the server on the hosted arm64 image, start it + # there under deploy/nova-cache.service, and on a push to main ship both + # to the public cache's host. anv26 is an Ampere box on the same Ubuntu + # 24.04, so the runner's glibc matches and its systemd is the release the + # host runs, and the production host deliberately carries no compiler + # toolchain. The smoke test runs on every pull request, so a unit the + # server cannot run under fails before it merges, and a deploy installs + # only what just ran. # --------------------------------------------------------------------------- - deploy-server: - name: Deploy Server + server: + name: Server (arm64) needs: [build] - if: github.ref == 'refs/heads/main' && github.event_name == 'push' runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v7 @@ -377,7 +380,11 @@ jobs: cabal build --flag server --ghc-options="-Wall -Werror" cp "$(cabal list-bin --flag server nova-cache-server)" /tmp/nova-cache-server + - name: Run the server under its unit + run: deploy/smoke-test.sh /tmp/nova-cache-server + - name: Deploy to OCI + if: github.ref == 'refs/heads/main' && github.event_name == 'push' env: OCI_SSH_KEY: ${{ secrets.OCI_SSH_KEY }} CACHE_API_KEY: ${{ secrets.NOVA_CACHE_API_KEY }} @@ -407,35 +414,12 @@ jobs: StrictHostKeyChecking accept-new SSHCFG - # Render the service unit. The API key is not embedded in the - # unit: systemd loads it from /etc/nix-cache/env (written below, - # root-only), so the unit file carries no credentials. - cat > /tmp/nova-cache.service <<'EOF' - [Unit] - Description=Nova Cache Server - After=network.target - - [Service] - Type=simple - User=ubuntu - Group=ubuntu - ExecStart=/usr/local/bin/nova-cache-server --port 5000 --store /var/lib/nix-cache - Restart=always - RestartSec=5 - Environment=PORT=5000 - Environment=NIX_CACHE_DIR=/var/lib/nix-cache - Environment=SIGNING_KEY_FILE=/etc/nix-cache/secret-key - EnvironmentFile=/etc/nix-cache/env - - [Install] - WantedBy=multi-user.target - EOF - - scp /tmp/nova-cache.service anv26:/tmp/nova-cache.service + # The unit, the user it runs as and Caddy's restart policy are the + # files of deploy/, the first two as the smoke test above ran them. # The binary rides the same Access tunnel as everything else; the # service keeps serving during the transfer and stops only for # the install below. - scp /tmp/nova-cache-server anv26:/tmp/nova-cache-server + scp deploy/nova-cache.service deploy/nova-cache.sysusers deploy/caddy-restart.conf /tmp/nova-cache-server anv26:/tmp/ # Provision the API key as a root-only environment file. The value # travels over stdin, so it appears in no command line, no log, and @@ -443,11 +427,21 @@ jobs: printf 'CACHE_API_KEY=%s\n' "$CACHE_API_KEY" | ssh anv26 \ "sudo sh -c 'umask 077; mkdir -p /etc/nix-cache; cat > /etc/nix-cache/env; chown root:root /etc/nix-cache/env; chmod 600 /etc/nix-cache/env'" + # The signing key becomes root's: the unit hands the service a copy + # as a credential, so the service's user does not read the file. + # The store changes hands when the unit starts. The last command + # fails the deploy if the server does not answer. ssh anv26 "\ + sudo install -D -m 0644 /tmp/nova-cache.sysusers /etc/sysusers.d/nova-cache.conf && \ + sudo systemd-sysusers /etc/sysusers.d/nova-cache.conf && \ + sudo chown root:root /etc/nix-cache/secret-key && \ + sudo chmod 600 /etc/nix-cache/secret-key && \ + sudo install -D -m 0644 /tmp/caddy-restart.conf /etc/systemd/system/caddy.service.d/restart.conf && \ sudo systemctl stop nova-cache.service && \ sudo install -m 0755 /tmp/nova-cache-server /usr/local/bin/nova-cache-server && \ - sudo cp /tmp/nova-cache.service /etc/systemd/system/nova-cache.service && \ - rm -f /tmp/nova-cache-server /tmp/nova-cache.service && \ + sudo install -m 0644 /tmp/nova-cache.service /etc/systemd/system/nova-cache.service && \ + rm -f /tmp/nova-cache-server /tmp/nova-cache.service /tmp/nova-cache.sysusers /tmp/caddy-restart.conf && \ sudo systemctl daemon-reload && \ sudo systemctl start nova-cache.service && \ - sudo systemctl status nova-cache.service --no-pager" + sudo systemctl status nova-cache.service --no-pager && \ + curl --fail --silent --show-error --output /dev/null --retry 10 --retry-connrefused --retry-delay 1 http://127.0.0.1:5000/nix-cache-info" diff --git a/deploy/caddy-restart.conf b/deploy/caddy-restart.conf new file mode 100644 index 0000000..d3c1fc2 --- /dev/null +++ b/deploy/caddy-restart.conf @@ -0,0 +1,5 @@ +# Caddy's packaged unit names no restart policy, so a Caddy that exits stays +# down, and the public cache with it. +[Service] +Restart=on-failure +RestartSec=5 diff --git a/deploy/nova-cache.service b/deploy/nova-cache.service new file mode 100644 index 0000000..b1310c1 --- /dev/null +++ b/deploy/nova-cache.service @@ -0,0 +1,61 @@ +# nova-cache-server on the public cache's host: its own user, one writable +# directory, and loopback alone. The deploy job installs this file after +# deploy/smoke-test.sh has started the server under it. + +[Unit] +Description=Nova Cache Server +After=network.target + +[Service] +Type=simple +User=nova-cache +Group=nova-cache +ExecStart=/usr/local/bin/nova-cache-server --host 127.0.0.1 --port 5000 --store %S/nix-cache +Restart=always +RestartSec=5 + +# /var/lib/nix-cache. systemd creates it if it is absent and gives it, with +# everything under it, to the service's user, whoever owned it before. +StateDirectory=nix-cache + +# The signing key reaches the service as a credential: systemd reads the file +# as root and shows a copy to this unit alone, so the file stays root-only. +# The write key comes from an environment file only root reads. +LoadCredential=signing-key:/etc/nix-cache/secret-key +Environment=SIGNING_KEY_FILE=%d/signing-key +EnvironmentFile=/etc/nix-cache/env + +# The reverse proxy on the same host is the service's only client. +IPAddressDeny=any +IPAddressAllow=localhost +RestrictAddressFamilies=AF_INET AF_INET6 + +# The service reads its binary and its two keys, writes its state directory +# and listens on one port. +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +PrivateUsers=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +ProtectProc=invisible +ProcSubset=pid +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RemoveIPC=yes +CapabilityBoundingSet= +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallFilter=~@privileged @resources + +[Install] +WantedBy=multi-user.target diff --git a/deploy/nova-cache.sysusers b/deploy/nova-cache.sysusers new file mode 100644 index 0000000..079be3d --- /dev/null +++ b/deploy/nova-cache.sysusers @@ -0,0 +1,2 @@ +# The user nova-cache.service runs as. +u nova-cache - "nova-cache binary cache server" /var/lib/nix-cache /usr/sbin/nologin diff --git a/deploy/smoke-test.sh b/deploy/smoke-test.sh new file mode 100755 index 0000000..d1011e8 --- /dev/null +++ b/deploy/smoke-test.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Start nova-cache.service on this machine as the deploy installs it, and +# check what the unit's sandbox could break: that the server starts, reads +# its signing key, accepts an authenticated write and refuses another, that +# a store another user owned becomes the service's, and that the server +# answers on loopback alone. +# +# The keys are made here and stand in for the host's. The machine is a +# throwaway: the script installs into /usr/local/bin, /etc and /var/lib and +# removes nothing. +# +# Usage: smoke-test.sh SERVER +# SERVER the nova-cache-server binary to install +set -euo pipefail + +server=${1:?the nova-cache-server binary} +here=$(cd "$(dirname "$0")" && pwd) + +readonly PORT=5000 +readonly STORE=/var/lib/nix-cache +readonly SIGNING_KEY_NAME=smoke-test-1 +readonly WRITE_KEY=smoke-test-write-key +# A NAR's file name is its hash in base 32; the server stores any name of +# that shape. +readonly NAR=0000000000000000000000000000000000000000000000000000.nar +readonly BODY='smoke test' + +base=http://127.0.0.1:$PORT + +pass() { echo "ok $*"; } +fail() { + echo "FAIL $*" >&2 + sudo journalctl -u nova-cache.service --no-pager -n 40 >&2 || true + exit 1 +} + +# A store another user owns, as the host's is before this unit's first +# deploy there. +sudo mkdir -p "$STORE/nar" +echo kept | sudo tee "$STORE/nar/pre-existing" > /dev/null +sudo chown -R "$(id -un)" "$STORE" + +# An Ed25519 key as Nix writes one: the name, then the seed and the public +# key in base 64. Each is the last 32 bytes of its DER encoding. +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +openssl genpkey -algorithm ed25519 -out "$work/key.pem" +{ + printf '%s:' "$SIGNING_KEY_NAME" + { + openssl pkey -in "$work/key.pem" -outform DER | tail -c 32 + openssl pkey -in "$work/key.pem" -pubout -outform DER | tail -c 32 + } | base64 -w 0 + echo +} > "$work/secret-key" +sudo install -D -m 0600 -o root -g root "$work/secret-key" /etc/nix-cache/secret-key +printf 'CACHE_API_KEY=%s\n' "$WRITE_KEY" | + sudo sh -c 'umask 077; cat > /etc/nix-cache/env' + +sudo install -m 0755 "$server" /usr/local/bin/nova-cache-server +sudo install -D -m 0644 "$here/nova-cache.sysusers" /etc/sysusers.d/nova-cache.conf +sudo systemd-sysusers /etc/sysusers.d/nova-cache.conf +sudo install -m 0644 "$here/nova-cache.service" /etc/systemd/system/nova-cache.service +sudo systemctl daemon-reload +sudo systemctl start nova-cache.service + +curl --fail --silent --show-error --output /dev/null \ + --retry 10 --retry-connrefused --retry-delay 1 "$base/nix-cache-info" || + fail "the server does not answer on loopback" +[ "$(systemctl is-active nova-cache.service)" = active ] || fail "the service is not active" +pass "the server runs under the unit and answers on loopback" + +# The landing page prints the public key of the signing key it loaded. +curl --fail --silent --show-error "$base/" | grep -q -F "$SIGNING_KEY_NAME:" || + fail "the landing page does not name the signing key, so the server did not read it" +pass "the server read its signing key" + +code=$(printf '%s' "$BODY" | curl --silent --output /dev/null --write-out '%{http_code}' \ + -X PUT -H "Authorization: Bearer $WRITE_KEY" --data-binary @- "$base/nar/$NAR") +[ "$code" = 200 ] || fail "an authenticated write answered $code" +[ "$(curl --fail --silent --show-error "$base/nar/$NAR")" = "$BODY" ] || + fail "the written NAR does not read back" +pass "an authenticated write is stored and read back" + +code=$(printf '%s' "$BODY" | curl --silent --output /dev/null --write-out '%{http_code}' \ + -X PUT -H "Authorization: Bearer wrong" --data-binary @- "$base/nar/1${NAR#0}") +[ "$code" = 401 ] || fail "a write with the wrong key answered $code" +pass "a write with the wrong key is refused" + +for path in "$STORE" "$STORE/nar/pre-existing" "$STORE/nar/$NAR"; do + owner=$(sudo stat -c %U "$path") + [ "$owner" = nova-cache ] || fail "$path belongs to $owner" +done +pass "the store another user owned is the service's" + +address=$(hostname -I | awk '{ print $1 }') +if curl --silent --max-time 3 --output /dev/null "http://$address:$PORT/nix-cache-info"; then + fail "the server answers on $address" +fi +pass "the server does not answer on $address" + +systemd-analyze security nova-cache.service --no-pager | tail -n 1