From df0e76ad74e98b92c1590961e52ba1b8c6aaee91 Mon Sep 17 00:00:00 2001 From: Felipe Arce Date: Sat, 31 Oct 2026 18:25:22 -0400 Subject: [PATCH] copper-cloud 0.13.1: device agent relay fixes (re-delivery, per-link cap, non-reading agents, disabled users) - Re-delivery: a call stays pending when its stream ends without a reply; every stream a device (re)attaches gets each request still pending on its link again (same id, deadline_ms = time left), oldest first. Pending calls still fail after the 15 s offline grace. - Per-link cap: at most 8 calls in flight per link (DeviceAgentTuning::max_pending_per_link); one more answers at once, isError "Copper on is busy with other calls; try again shortly." (status busy). - Each relayed call waits in its own task: it ends at its deadline / cancel / revoke / shutdown whether or not the agent reads its SSE answer; hub progress is a bounded try_send (32) and the agent's progress buffer is bounded (32, newest dropped), so a slow reader never holds up a frames POST or other calls. - Disabling or deleting a user (admin API) closes their streams with `revoked`, fails their calls (status user_revoked) and drops their rate-limit buckets (the hub's own per-token GCRA replaces governor there). `copper-cloud admin disable-user` / `delete-user` work on the database; a user sweep (10 s, only while the hub holds anything) reaches the running server. - Tests: re-delivery after a dropped and a superseded stream, busy cap (JSON + SSE), an agent that never reads (16 MiB progress flood, other calls proceed, cancel at the deadline), admin API and CLI disable/delete; hub unit tests. - Docs: api.md (re-delivery, busy, slow readers, revoked), admin-api, architecture, security, operations; CHANGELOG + docs/v0.13.1.md; version 0.13.1. --- CHANGELOG.md | 17 + Cargo.lock | 8 +- Cargo.toml | 2 +- README.md | 4 +- crates/copper-cloud/src/admin.rs | 3 + crates/copper-cloud/src/admin_api.rs | 6 + crates/copper-cloud/tests/device_agent.rs | 441 +++++++++++++++++- crates/core/src/app.rs | 4 +- crates/core/src/config.rs | 8 + crates/core/src/device_agent/hub.rs | 533 ++++++++++++++++++---- crates/core/src/device_agent/mcp.rs | 281 ++++++++---- crates/core/src/device_agent/mod.rs | 70 ++- crates/core/src/state.rs | 1 + docs/admin-api.md | 7 +- docs/api.md | 26 +- docs/architecture.md | 20 +- docs/install.md | 4 +- docs/operations.md | 6 +- docs/security.md | 4 +- docs/v0.13.1.md | 21 + 20 files changed, 1236 insertions(+), 230 deletions(-) create mode 100644 docs/v0.13.1.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a9d1ba..dddcb8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ Newest first. Each release also has fuller notes in `docs/vX.Y.Z.md`. +## 0.13.1 + +Device agent relay fixes ([notes](docs/v0.13.1.md)). No migrations; a binary swap either way. + +- A pending request is sent again (same id, the time it has left) on every stream the phone + opens, so a stream that dies mid-call no longer leaves the agent waiting out the 200 s + deadline; the call fails after 15 s offline as before + ([re-delivery](docs/api.md#get-v1melinksidframes--sse-mailbox)). +- At most 8 calls in flight per link; one more answers at once, `isError: true`, "Copper on + is busy with other calls; try again shortly." (status `busy`). +- Each relayed call waits in its own task: it ends at its deadline whether or not the agent reads + its SSE answer; a slow reader loses progress notifications beyond 32 (never the response) and + never holds up a frames POST or other calls. +- Disabling or deleting a user closes their phones' streams with `revoked`, fails their calls in + flight (status `user_revoked`) and drops their rate limits: at once from the admin API, within + 10 s from `copper-cloud admin disable-user` / `delete-user`. + ## 0.13.0 Agents can use a person's Copper for iOS through the cloud ([notes](docs/v0.13.0.md)). One new diff --git a/Cargo.lock b/Cargo.lock index 0814a32..6cd0482 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -638,7 +638,7 @@ checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] name = "copper-cloud" -version = "0.13.0" +version = "0.13.1" dependencies = [ "anyhow", "axum", @@ -670,7 +670,7 @@ dependencies = [ [[package]] name = "copper-cloud-canvas-link" -version = "0.13.0" +version = "0.13.1" dependencies = [ "anyhow", "axum", @@ -699,7 +699,7 @@ dependencies = [ [[package]] name = "copper-cloud-canvas-migrate" -version = "0.13.0" +version = "0.13.1" dependencies = [ "anyhow", "axum", @@ -723,7 +723,7 @@ dependencies = [ [[package]] name = "copper-cloud-core" -version = "0.13.0" +version = "0.13.1" dependencies = [ "aes-gcm", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 045e156..aa35adc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.13.0" +version = "0.13.1" edition = "2021" license = "MIT" repository = "https://github.com/copper-browser/copper-cloud" diff --git a/README.md b/README.md index 93eff21..8bd9b0e 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ curl -fsSL https://raw.githubusercontent.com/copper-browser/copper-cloud/main/in or from a release tarball: ```sh -tar xzf copper-cloud-0.13.0-linux-x86_64.tar.gz && sudo ./install.sh +tar xzf copper-cloud-0.13.1-linux-x86_64.tar.gz && sudo ./install.sh ``` The installer sets up Postgres (unless you pass `DATABASE_URL`), a `copper-cloud` system user, @@ -150,7 +150,7 @@ All commands take `--config PATH` (default `/etc/copper-cloud/copper-cloud.toml` - [Operations](docs/operations.md) — doctor, logs, metrics, backups, upgrades - [Onboarding a team](docs/onboarding.md) — single sign-on first, link codes, people, keeping links valid - [Single sign-on + agent integrations](docs/sso.md) — Google / OIDC setup, `/start`, agent pairing -- [v0.13.0 release notes](docs/v0.13.0.md) · [v0.12.0](docs/v0.12.0.md) · [v0.11.1](docs/v0.11.1.md) · [v0.11.0](docs/v0.11.0.md) · [v0.10.0](docs/v0.10.0.md) · [v0.9.0](docs/v0.9.0.md) · [v0.8.1](docs/v0.8.1.md) · [v0.8.0](docs/v0.8.0.md) · [v0.7.0](docs/v0.7.0.md) · [v0.6.0](docs/v0.6.0.md) · [v0.5.0](docs/v0.5.0.md) · [v0.4.0](docs/v0.4.0.md) · [v0.3.0](docs/v0.3.0.md) +- [v0.13.1 release notes](docs/v0.13.1.md) · [v0.13.0](docs/v0.13.0.md) · [v0.12.0](docs/v0.12.0.md) · [v0.11.1](docs/v0.11.1.md) · [v0.11.0](docs/v0.11.0.md) · [v0.10.0](docs/v0.10.0.md) · [v0.9.0](docs/v0.9.0.md) · [v0.8.1](docs/v0.8.1.md) · [v0.8.0](docs/v0.8.0.md) · [v0.7.0](docs/v0.7.0.md) · [v0.6.0](docs/v0.6.0.md) · [v0.5.0](docs/v0.5.0.md) · [v0.4.0](docs/v0.4.0.md) · [v0.3.0](docs/v0.3.0.md) ## Development diff --git a/crates/copper-cloud/src/admin.rs b/crates/copper-cloud/src/admin.rs index 55bc6e5..b496a3c 100644 --- a/crates/copper-cloud/src/admin.rs +++ b/crates/copper-cloud/src/admin.rs @@ -117,6 +117,8 @@ async fn dispatch(cfg: &Config, pool: &sqlx::PgPool, cmd: AdminCommand) -> anyho if !yes && !confirm(&format!("Delete {email} and ALL their synced data? [y/N] "))? { bail!("aborted"); } + // Their agent tokens stop working at once; a running server closes their phones' + // agent streams within its device-agent user sweep (10 s). sqlx::query("DELETE FROM users WHERE id = $1") .bind(id) .execute(pool) @@ -125,6 +127,7 @@ async fn dispatch(cfg: &Config, pool: &sqlx::PgPool, cmd: AdminCommand) -> anyho } AdminCommand::DisableUser { email } => { let id = find_user(pool, &email).await?; + // As for delete-user: tokens refused at once, streams closed by the user sweep. let mut tx = pool.begin().await?; sqlx::query("UPDATE users SET disabled = true, updated_at = now() WHERE id = $1") .bind(id) diff --git a/crates/copper-cloud/src/admin_api.rs b/crates/copper-cloud/src/admin_api.rs index d5c3e32..b9b72f0 100644 --- a/crates/copper-cloud/src/admin_api.rs +++ b/crates/copper-cloud/src/admin_api.rs @@ -1004,6 +1004,10 @@ async fn update_user( // sockets there); queued in this transaction so it survives Canvas Cloud being down. provision::enqueue_user_changed(&mut tx, id, req.disabled == Some(true)).await?; tx.commit().await?; + if req.disabled == Some(true) { + // Their phones' agent streams close and calls in flight fail now. + copper_cloud_core::device_agent::revoke_user(&state, id); + } link.kick_outbox(); let mut row = user_by_id(&state.db, id).await?; with_canvas_counts(&link, std::slice::from_mut(&mut row)).await; @@ -1051,6 +1055,8 @@ async fn delete_user( .await?; provision::enqueue_user_deleted(&mut tx, id).await?; tx.commit().await?; + // Their links and agent tokens went with the user row; close the streams and calls too. + copper_cloud_core::device_agent::revoke_user(&state, id); link.kick_outbox(); Ok(ok()) } diff --git a/crates/copper-cloud/tests/device_agent.rs b/crates/copper-cloud/tests/device_agent.rs index d6d6018..97b3f02 100644 --- a/crates/copper-cloud/tests/device_agent.rs +++ b/crates/copper-cloud/tests/device_agent.rs @@ -1,6 +1,8 @@ //! Device agent relay (`device_agent`, 0.13.0): links, the frames mailbox, agent tokens, //! `/v1/agent/*` outside the gate, and MCP calls relayed to a simulated device (an SSE reader -//! for `request` / `cancel` frames plus frames POSTs for replies and progress). +//! for `request` / `cancel` frames plus frames POSTs for replies and progress). 0.13.1: +//! re-delivery to a new stream, the per-link cap, agents that never read, and disabled or +//! deleted users. #![allow(clippy::too_many_lines)] @@ -1602,3 +1604,440 @@ async fn one_log_line_per_call_never_arguments_or_results() { assert!(line["ms"].is_u64(), "{line}"); } } + +// --------------------------------------------------------------------------------------------- +// 0.13.1: re-delivery, the per-link cap, agents that never read, disabled and deleted users + +/// The next non-`ping` event within `wait` (`None`: the stream ended). +async fn next_event_within(dev: &mut SseReader, wait: Duration) -> Option { + tokio::time::timeout(wait, dev.next_event()) + .await + .expect("timed out waiting for an event") +} + +fn link_uuid(id: &str) -> uuid::Uuid { + id.parse().unwrap() +} + +#[tokio::test] +async fn a_replaced_stream_gets_the_pending_request_again() { + // A long deadline and grace: the answer must come from the re-delivery, well before either. + let s = start_with(|cfg| { + tune(cfg); + cfg.device_agent.relay_deadline = Duration::from_secs(60); + cfg.device_agent.offline_grace = Duration::from_secs(20); + }) + .await; + let p = phone(&s, "ada@example.com").await; + let id = register_ok(&s, &p).await["id"].as_str().unwrap().to_owned(); + let token = token_of(&mint(&s, &p, "phi").await); + + // Stream A gets the request, then dies without replying. + let (mut stream_a, _) = connect(&s, &p, &id).await; + let started = Instant::now(); + let args = json!({ "goal": "Find flights to Lisbon on 2026-11-20" }); + let pending = spawn_call(&s, &token, call(30, "jev_run", args.clone())); + let first = next_request(&mut stream_a).await; + let rid = first["id"].as_str().unwrap().to_owned(); + assert_eq!(first["deadline_ms"], 60_000); + drop(stream_a); + eventually("stream A detached", || async { + !s.state.device_agent.is_online(link_uuid(&id)) + }) + .await; + assert_eq!( + s.state.device_agent.in_flight(), + 1, + "the call stays pending" + ); + + // The phone reconnects: stream B gets the same request (same id, the time it has left). + let (mut stream_b, _) = connect(&s, &p, &id).await; + let again = next_request(&mut stream_b).await; + assert_eq!(again["id"], rid.as_str()); + assert_eq!(again["method"], "tools/call"); + assert_eq!(again["params"], first["params"]); + assert_eq!(again["caller"], first["caller"]); + let left = again["deadline_ms"].as_u64().unwrap(); + assert!(left > 40_000 && left < 60_000, "{left}"); + let result = + json!({ "content": [{ "type": "text", "text": "DONE: 3 flights" }], "isError": false }); + post_frames( + &s, + &p, + &id, + json!([{ "type": "reply", "id": rid, "result": result }]), + ) + .await; + assert_eq!( + pending.await.unwrap(), + json!({ "jsonrpc": "2.0", "id": 30, "result": result }) + ); + assert!( + started.elapsed() < Duration::from_secs(10), + "{:?}", + started.elapsed() + ); + assert_eq!(s.state.device_agent.in_flight(), 0); + + // Superseded while still open: the new stream gets what is still waiting, oldest first; an + // answered call is not sent again. + let one = spawn_call(&s, &token, call(31, "jev_run", json!({}))); + let two = spawn_call(&s, &token, call(32, "jev_observe", json!({}))); + let r1 = next_request(&mut stream_b).await; + let r2 = next_request(&mut stream_b).await; + let (first_id, second_id) = if r1["params"]["name"] == "jev_run" { + (r1["id"].clone(), r2["id"].clone()) + } else { + (r2["id"].clone(), r1["id"].clone()) + }; + post_frames( + &s, + &p, + &id, + json!([{ "type": "reply", "id": first_id, "result": { "content": [], "isError": false } }]), + ) + .await; + assert_eq!(one.await.unwrap()["id"], 31); + let (mut stream_c, _) = connect(&s, &p, &id).await; + assert_eq!(stream_b.next_event().await.unwrap().name, "superseded"); + let resent = next_request(&mut stream_c).await; + assert_eq!(resent["id"], second_id); + post_frames( + &s, + &p, + &id, + json!([{ "type": "reply", "id": second_id, "result": { "content": [], "isError": false } }]), + ) + .await; + assert_eq!(two.await.unwrap()["id"], 32); + assert_eq!(s.state.device_agent.in_flight(), 0); +} + +#[tokio::test] +async fn calls_past_the_per_link_cap_answer_busy_at_once() { + let s = start_with(|cfg| { + tune(cfg); + cfg.device_agent.relay_deadline = Duration::from_secs(60); + }) + .await; + let cap = s.state.cfg.device_agent.max_pending_per_link; + assert_eq!(cap, 8, "the documented default"); + let p = phone(&s, "ada@example.com").await; + let id = register_ok(&s, &p).await["id"].as_str().unwrap().to_owned(); + let token = token_of(&mint(&s, &p, "phi").await); + let other = token_of(&mint(&s, &p, "other").await); + let (mut dev, _) = connect(&s, &p, &id).await; + + let mut calls = Vec::new(); + let mut ids = Vec::new(); + for n in 0..cap { + calls.push(spawn_call( + &s, + &token, + call(100 + i64::try_from(n).unwrap(), "jev_run", json!({})), + )); + ids.push(next_request(&mut dev).await["id"].clone()); + } + let busy = format!("Copper on {DEVICE_NAME} is busy with other calls; try again shortly."); + let started = Instant::now(); + let r = rpc(&s, &token, call(200, "jev_run", json!({}))).await; + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!(r["id"], 200); + assert_eq!(r["result"]["isError"], true); + assert_eq!(text_of(&r["result"]), busy); + // Per link, whichever agent asks; as SSE too. + let mut answer = sse_call(&s, &other, call(201, "jev_run", json!({}))).await; + let msg = answer.next().await.unwrap().json(); + assert_eq!(text_of(&msg["result"]), busy); + assert!(answer.next().await.is_none()); + assert_eq!( + s.state.device_agent.in_flight(), + cap, + "nothing sent to the device" + ); + + // One finishes: room for one more. + post_frames( + &s, + &p, + &id, + json!([{ "type": "reply", "id": ids[0], "result": { "content": [], "isError": false } }]), + ) + .await; + calls.remove(0).await.unwrap(); + let next = spawn_call(&s, &other, call(202, "jev_run", json!({}))); + ids[0] = next_request(&mut dev).await["id"].clone(); + calls.insert(0, next); + let r = rpc(&s, &token, call(203, "jev_run", json!({}))).await; + assert_eq!(text_of(&r["result"]), busy); + + let replies: Vec = ids + .iter() + .map(|rid| json!({ "type": "reply", "id": rid, "result": { "content": [], "isError": false } })) + .collect(); + post_frames(&s, &p, &id, Value::Array(replies)).await; + for c in calls { + assert_eq!(c.await.unwrap()["result"]["isError"], false); + } + assert_eq!(s.state.device_agent.in_flight(), 0); +} + +#[tokio::test] +async fn an_agent_that_never_reads_its_answer_still_ends_at_the_deadline() { + use tokio::io::AsyncWriteExt as _; + + let s = start_with(|cfg| { + tune(cfg); + // Room for the flood below on a slow machine before the deadline. + cfg.device_agent.relay_deadline = Duration::from_secs(8); + }) + .await; + let p = phone(&s, "ada@example.com").await; + let id = register_ok(&s, &p).await["id"].as_str().unwrap().to_owned(); + let token = token_of(&mint(&s, &p, "phi").await); + let (mut dev, _) = connect(&s, &p, &id).await; + + // Nothing here may wait on the agent that never reads: a hang is a failure. + tokio::time::timeout(Duration::from_secs(45), async { + // The stuck agent: an SSE `tools/call` with a progress token over a connection with a tiny + // receive buffer, never read. + let socket = tokio::net::TcpSocket::new_v4().unwrap(); + socket.set_recv_buffer_size(4096).unwrap(); + let mut stuck = socket.connect(s.addr).await.unwrap(); + let body = json!({ + "jsonrpc": "2.0", "id": 40, "method": "tools/call", + "params": { "name": "jev_run", "arguments": {}, "_meta": { "progressToken": "p" } }, + }) + .to_string(); + let request = format!( + "POST /v1/agent/mcp HTTP/1.1\r\nHost: {}\r\nAuthorization: Bearer {token}\r\n\ + Accept: {SSE_ACCEPT}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + s.addr, + body.len() + ); + stuck.write_all(request.as_bytes()).await.unwrap(); + let started = Instant::now(); + let rid = next_request(&mut dev).await["id"] + .as_str() + .unwrap() + .to_owned(); + + // The device floods progress (16 MiB of 16 KiB lines, far more than the sockets buffer): + // every frames POST still returns at once. + let line = "p".repeat(16 * 1024); + for batch in 0..16_i64 { + let frames: Vec = (1..=64_i64) + .map(|n| json!({ "type": "progress", "id": rid, "seq": batch * 64 + n, "message": line })) + .collect(); + let t = Instant::now(); + post_frames(&s, &p, &id, Value::Array(frames)).await; + assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed()); + } + + assert!( + started.elapsed() < Duration::from_secs(6), + "the flood must end well before the deadline: {:?}", + started.elapsed() + ); + + // Another agent's call on the same phone goes through meanwhile. + let other = token_of(&mint(&s, &p, "other").await); + let pending = spawn_call(&s, &other, call(41, "jev_observe", json!({}))); + let rid2 = next_request(&mut dev).await["id"].clone(); + post_frames( + &s, + &p, + &id, + json!([{ "type": "reply", "id": rid2, "result": { "content": [], "isError": false } }]), + ) + .await; + assert_eq!( + pending.await.unwrap(), + json!({ "jsonrpc": "2.0", "id": 41, "result": { "content": [], "isError": false } }) + ); + + // At the deadline the stuck call is cancelled on the device and leaves the hub, though its + // agent has not read a byte. + let ev = dev.next_event().await.unwrap(); + assert_eq!( + (ev.name.as_str(), ev.json()), + ("cancel", json!({ "id": rid })) + ); + let elapsed = started.elapsed(); + assert!( + elapsed >= Duration::from_millis(7900) && elapsed < Duration::from_secs(20), + "{elapsed:?}" + ); + eventually("the stuck call left the hub", || async { + s.state.device_agent.in_flight() == 0 + }) + .await; + // A late reply is ignored; the phone keeps working. + post_frames( + &s, + &p, + &id, + json!([{ "type": "reply", "id": rid, "result": { "content": [] } }]), + ) + .await; + assert_eq!(status(&s, &token).await["online"], true); + drop(stuck); + }) + .await + .expect("a non-reading agent must not hold anything up"); +} + +/// A phone with its stream open and a call in flight (from a token it minted). +struct Busy { + p: Session, + token: String, + dev: SseReader, + call: tokio::task::JoinHandle, +} + +async fn phone_with_a_call(s: &TestServer, email: &str) -> Busy { + let p = phone(s, email).await; + let id = register_ok(s, &p).await["id"].as_str().unwrap().to_owned(); + let token = token_of(&mint(s, &p, "phi").await); + let (mut dev, _) = connect(s, &p, &id).await; + let call = spawn_call(s, &token, call(50, "jev_run", json!({}))); + next_request(&mut dev).await; + Busy { + p, + token, + dev, + call, + } +} + +fn hub_users(s: &TestServer) -> Vec { + let mut users: Vec = s + .state + .device_agent + .users() + .iter() + .map(ToString::to_string) + .collect(); + users.sort_unstable(); + users +} + +/// The phone's stream gets `revoked` within `wait` and ends, its call fails naming the account, +/// its token is refused, and the hub holds nothing of the user (stream, call or rate limit). +async fn assert_user_revoked(s: &TestServer, mut b: Busy, wait: Duration) { + let ev = next_event_within(&mut b.dev, wait) + .await + .expect("revoked event"); + assert_eq!((ev.name.as_str(), ev.data.as_str()), ("revoked", "{}")); + assert!(b.dev.next().await.is_none(), "the stream ends"); + let answer = tokio::time::timeout(Duration::from_secs(2), b.call) + .await + .expect("the call fails at once") + .unwrap(); + assert_eq!(answer["result"]["isError"], true); + assert!( + text_of(&answer["result"]).contains("was disabled or deleted while jev_run was running"), + "{answer}" + ); + let r = agent(s, Method::GET, "/v1/agent/status", &b.token) + .send() + .await + .unwrap(); + expect_error(r, 401, "agent_token").await; + assert!(!hub_users(s).contains(&b.p.user_id)); +} + +#[tokio::test] +async fn admin_api_disable_and_delete_close_the_users_streams_and_calls() { + // The default (10 s) sweep: the admin API acts at once. + let s = start_with(|cfg| { + tune(cfg); + cfg.device_agent.relay_deadline = Duration::from_secs(60); + cfg.device_agent.user_sweep = Duration::from_secs(10); + }) + .await; + s.create_admin().await; + let cookie = s.admin_login().await; + let ada = phone_with_a_call(&s, "ada@example.com").await; + let mut bob = phone_with_a_call(&s, "bob@example.com").await; + let mut both = vec![ada.p.user_id.clone(), bob.p.user_id.clone()]; + both.sort_unstable(); + assert_eq!(hub_users(&s), both); + + let r = s + .admin( + Method::PATCH, + &format!("users/{}", ada.p.user_id), + Some(&cookie), + ) + .json(&json!({ "disabled": true })) + .send() + .await + .unwrap(); + assert_eq!(r.status(), 200); + assert_user_revoked(&s, ada, Duration::from_secs(2)).await; + // Bob is untouched. + assert_eq!(hub_users(&s), std::slice::from_ref(&bob.p.user_id)); + assert_eq!(s.state.device_agent.in_flight(), 1); + assert_eq!(status(&s, &bob.token).await["online"], true); + match bob.dev.next_within(Duration::from_millis(100)).await { + Err(()) => {} + Ok(Some(ev)) => assert_eq!(ev.name, "ping", "{ev:?}"), + Ok(None) => panic!("Bob's stream ended"), + } + + let r = s + .admin( + Method::DELETE, + &format!("users/{}", bob.p.user_id), + Some(&cookie), + ) + .send() + .await + .unwrap(); + assert_eq!(r.status(), 200); + assert_user_revoked(&s, bob, Duration::from_secs(2)).await; + assert_eq!(hub_users(&s), Vec::::new()); + assert_eq!(s.state.device_agent.in_flight(), 0); +} + +#[tokio::test] +async fn cli_disable_and_delete_reach_the_running_server_through_the_sweep() { + use copper_cloud::cli::AdminCommand; + + let s = start_with(|cfg| { + tune(cfg); + cfg.device_agent.relay_deadline = Duration::from_secs(60); + cfg.device_agent.user_sweep = Duration::from_millis(200); + }) + .await; + let cfg = (*s.state.cfg).clone(); + let ada = phone_with_a_call(&s, "ada@example.com").await; + let bob = phone_with_a_call(&s, "bob@example.com").await; + + // `copper-cloud admin …` works on the database only (in production, another process). + copper_cloud::admin::run( + cfg.clone(), + AdminCommand::DisableUser { + email: "ada@example.com".into(), + }, + ) + .await + .unwrap(); + assert_user_revoked(&s, ada, Duration::from_secs(3)).await; + assert_eq!(hub_users(&s), std::slice::from_ref(&bob.p.user_id)); + + copper_cloud::admin::run( + cfg, + AdminCommand::DeleteUser { + email: "bob@example.com".into(), + yes: true, + }, + ) + .await + .unwrap(); + assert_user_revoked(&s, bob, Duration::from_secs(3)).await; + assert_eq!(hub_users(&s), Vec::::new()); + assert_eq!(s.state.device_agent.in_flight(), 0); +} diff --git a/crates/core/src/app.rs b/crates/core/src/app.rs index cc82ba4..21853fa 100644 --- a/crates/core/src/app.rs +++ b/crates/core/src/app.rs @@ -78,7 +78,8 @@ pub fn app(state: SharedState, extra_v1: Router) -> Router { /// The complete HTTP app: `/healthz`, `/v1/*` (core routes merged with `extra_v1`, e.g. the /// canvas router; unknown `/v1` paths are a JSON 404) and `root` (top-level routes and/or a /// fallback, e.g. the admin API and the portal), wrapped in the instance gate (which only -/// guards `/v1`), the auth rate limiters and request tracing/metrics. +/// guards `/v1`), the auth rate limiters and request tracing/metrics. Starts the device-agent +/// user sweep ([`crate::device_agent::start_user_sweep`]) when called inside a tokio runtime. pub fn app_with( state: SharedState, extra_v1: Router, @@ -88,6 +89,7 @@ pub fn app_with( auth: AuthRateLimiter::new(state.cfg.limits.auth_per_minute, state.cfg.trust_proxy), admin: AuthRateLimiter::new(state.cfg.limits.auth_per_minute, state.cfg.trust_proxy), }); + crate::device_agent::start_user_sweep(&state); Router::new() .route("/healthz", get(healthz)) .nest("/v1", router().merge(extra_v1).fallback(not_found)) diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 513db4e..7523aba 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -140,6 +140,12 @@ pub struct DeviceAgentTuning { pub offline_grace: std::time::Duration, /// Requests per minute per agent token on `/v1/agent/*` (600). pub agent_per_minute: u32, + /// Most calls in flight per link; one more answers "busy" at once (8; the phone runs one + /// at a time). + pub max_pending_per_link: usize, + /// How often the hub drops what it holds of users disabled or deleted outside this process, + /// e.g. by `copper-cloud admin disable-user` (10 s). + pub user_sweep: std::time::Duration, } impl Default for DeviceAgentTuning { @@ -152,6 +158,8 @@ impl Default for DeviceAgentTuning { online_window: Duration::from_secs(45), offline_grace: Duration::from_secs(15), agent_per_minute: 600, + max_pending_per_link: 8, + user_sweep: Duration::from_secs(10), } } } diff --git a/crates/core/src/device_agent/hub.rs b/crates/core/src/device_agent/hub.rs index f11456c..cf2b374 100644 --- a/crates/core/src/device_agent/hub.rs +++ b/crates/core/src/device_agent/hub.rs @@ -1,17 +1,24 @@ -//! The in-memory relay hub: which link has a frames stream open right now, and the agent calls -//! waiting for a device's reply. Single-process by design (one daemon per instance), like -//! [`crate::events`]. Every method takes the lock briefly and never awaits while holding it. +//! The in-memory relay hub: which link has a frames stream open right now, the agent calls +//! waiting for a device's reply, and each agent token's rate limit. Single-process by design +//! (one daemon per instance), like [`crate::events`]. Every method takes a lock briefly and never +//! awaits while holding it. -use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Mutex, MutexGuard, PoisonError}; use std::time::{Duration, Instant}; -use governor::{DefaultKeyedRateLimiter, Quota, RateLimiter}; +use serde::Serialize; use serde_json::value::RawValue; +use serde_json::{Map, Value}; +use tokio::sync::mpsc::error::TrySendError; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; +/// Progress frames buffered for one call; a call that falls this far behind loses the newest +/// (the frames POST never waits). +pub const PROGRESS_BUFFER: usize = 32; + /// What the cloud sends down a device's frames stream (`GET /v1/me/links/{id}/frames`). #[derive(Debug)] pub enum Down { @@ -36,6 +43,8 @@ pub enum Reply { TurnedOff, /// The agent's token was revoked while the call was running. TokenRevoked, + /// The user was disabled or deleted while the call was running. + UserRemoved, } /// One device `progress` frame, forwarded to the waiting call. @@ -45,23 +54,54 @@ pub struct Progress { pub message: String, } -/// A call handed to a device: wait on `reply`, forward `progress`. +/// A call for a device (see [`Hub::begin`]). +#[derive(Debug)] +pub struct Request { + pub link: Uuid, + pub user: Uuid, + pub token: Uuid, + /// The request id (`req_…`), also in `frame`. + pub id: String, + /// The `request` frame without `deadline_ms`, which is added each time it is sent (the time + /// left then). + pub frame: Map, + /// How long the call may run. + pub timeout: Duration, +} + +/// A call handed to a device: wait on `reply` until `deadline`, forward `progress`. pub struct Call { pub reply: oneshot::Receiver, - pub progress: mpsc::UnboundedReceiver, + pub progress: mpsc::Receiver, + pub deadline: Instant, +} + +/// Why [`Hub::begin`] did not hand a call to the device. +#[derive(Debug, PartialEq, Eq)] +pub enum Refused { + /// No open stream, or not heard from within the online window. + Offline, + /// The link already has the most calls in flight it may. + Busy, } struct Slot { stream: u64, + user: Uuid, tx: mpsc::UnboundedSender, last_heard: Instant, } struct Pending { link: Uuid, + user: Uuid, token: Uuid, + /// Begin order, so a re-delivery keeps it. + order: u64, + frame: Map, + deadline: Instant, reply: oneshot::Sender, - progress: mpsc::UnboundedSender, + progress: mpsc::Sender, last_seq: Option, } @@ -73,13 +113,25 @@ struct Inner { pending: HashMap, } +/// One agent token's rate limit (GCRA: `tat` is the theoretical arrival time). +struct Bucket { + user: Uuid, + tat: Instant, +} + /// The relay hub (lives in [`crate::state::AppState`]). pub struct Hub { inner: Mutex, + buckets: Mutex>, next_stream: AtomicU64, + next_call: AtomicU64, online_window: Duration, - limiter: DefaultKeyedRateLimiter, + max_pending: usize, + /// One request every `interval`, bursts up to `burst` ahead (one minute's worth). + interval: Duration, + burst: Duration, checks: AtomicU64, + sweeping: AtomicBool, } impl std::fmt::Debug for Hub { @@ -100,47 +152,104 @@ pub fn new_request_id() -> String { ) } +/// The `request` event data: `frame` plus `deadline_ms`. +fn render(frame: &Map, deadline_ms: u64) -> String { + #[derive(Serialize)] + struct Out<'a> { + #[serde(flatten)] + frame: &'a Map, + deadline_ms: u64, + } + serde_json::to_string(&Out { frame, deadline_ms }).unwrap_or_default() +} + +/// Whole milliseconds, rounded up. +fn ceil_ms(d: Duration) -> u64 { + u64::try_from(d.as_micros().div_ceil(1000)).unwrap_or(u64::MAX) +} + impl Hub { /// `per_minute` requests per agent token; a link is online while heard from within - /// `online_window`. - pub fn new(per_minute: u32, online_window: Duration) -> Self { - let n = std::num::NonZeroU32::new(per_minute.max(1)).expect("non-zero"); + /// `online_window`; at most `max_pending` calls in flight per link. + pub fn new(per_minute: u32, online_window: Duration, max_pending: usize) -> Self { + let per_minute = per_minute.max(1); + let interval = Duration::from_secs(60) / per_minute; Self { inner: Mutex::new(Inner::default()), + buckets: Mutex::new(HashMap::new()), next_stream: AtomicU64::new(1), + next_call: AtomicU64::new(1), online_window, - limiter: RateLimiter::keyed(Quota::per_minute(n)), + max_pending: max_pending.max(1), + interval, + burst: interval * per_minute, checks: AtomicU64::new(0), + sweeping: AtomicBool::new(false), } } + /// `true` the first time only: whoever gets it runs the user sweep. + pub(crate) fn claim_sweep(&self) -> bool { + !self.sweeping.swap(true, Ordering::SeqCst) + } + fn lock(&self) -> MutexGuard<'_, Inner> { self.inner.lock().unwrap_or_else(PoisonError::into_inner) } - /// `true` if this agent token may make another request now. - pub fn allow(&self, token: Uuid) -> bool { + fn lock_buckets(&self) -> MutexGuard<'_, HashMap> { + self.buckets.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// `true` if agent token `token` (of `user`) may make another request now. + pub fn allow(&self, user: Uuid, token: Uuid) -> bool { + let now = Instant::now(); + let mut buckets = self.lock_buckets(); if self.checks.fetch_add(1, Ordering::Relaxed) % 4096 == 4095 { - self.limiter.retain_recent(); - self.limiter.shrink_to_fit(); + // A bucket whose arrival time has passed is as good as a fresh one. + buckets.retain(|_, b| b.tat > now); + buckets.shrink_to_fit(); + } + let bucket = buckets.entry(token).or_insert(Bucket { user, tat: now }); + let tat = bucket.tat.max(now) + self.interval; + if tat > now + self.burst { + return false; } - self.limiter.check_key(&token).is_ok() + bucket.tat = tat; + true } /// A device opened its frames stream: it becomes the link's stream (the previous one, if - /// any, gets [`Down::Superseded`]). Returns the stream id (for [`Hub::detach`]) and the - /// receiving end. - pub fn attach(&self, link: Uuid) -> (u64, mpsc::UnboundedReceiver) { + /// any, gets [`Down::Superseded`]), and every call still waiting on the link is sent again + /// on it, oldest first, with the same request id and the time it has left (a device answers + /// a request it already answered with its stored reply). Returns the stream id (for + /// [`Hub::detach`]) and the receiving end. + pub fn attach(&self, link: Uuid, user: Uuid) -> (u64, mpsc::UnboundedReceiver) { let stream = self.next_stream.fetch_add(1, Ordering::Relaxed); let (tx, rx) = mpsc::unbounded_channel(); - let old = self.lock().slots.insert( - link, - Slot { - stream, - tx, - last_heard: Instant::now(), - }, - ); + let old = { + let mut inner = self.lock(); + let now = Instant::now(); + let mut waiting: Vec<&Pending> = inner + .pending + .values() + .filter(|p| p.link == link && p.deadline > now) + .collect(); + waiting.sort_unstable_by_key(|p| p.order); + for p in waiting { + let frame = render(&p.frame, ceil_ms(p.deadline - now)); + let _ = tx.send(Down::Request(frame)); + } + inner.slots.insert( + link, + Slot { + stream, + user, + tx, + last_heard: now, + }, + ) + }; if let Some(old) = old { let _ = old.tx.send(Down::Superseded); } @@ -148,7 +257,8 @@ impl Hub { } /// A frames stream ended. No-op if a newer stream already replaced it. Calls in flight stay - /// pending (the device may reconnect and still reply). + /// pending: a new stream gets them again, or they fail once the device has been offline for + /// the offline grace (see [`crate::device_agent`]'s relay). pub fn detach(&self, link: Uuid, stream: u64) -> bool { let mut inner = self.lock(); if inner.slots.get(&link).is_some_and(|s| s.stream == stream) { @@ -174,31 +284,49 @@ impl Hub { .is_some_and(|s| s.last_heard.elapsed() < self.online_window) } - /// Hand a call to the device: sends `frame` (the serialized `request` data carrying `id`) - /// down the link's stream and registers the waiter. `None` when the link is offline (the - /// caller answers at once). - pub fn begin(&self, link: Uuid, token: Uuid, id: &str, frame: String) -> Option { - let mut inner = self.lock(); + /// Hand a call to the device: sends the `request` frame down the link's stream and registers + /// the waiter. Refused when the link is offline or already has the most calls in flight + /// (the caller answers at once). + pub fn begin(&self, req: Request) -> Result { + let mut guard = self.lock(); + let inner = &mut *guard; let slot = inner .slots - .get(&link) - .filter(|s| s.last_heard.elapsed() < self.online_window)?; - slot.tx.send(Down::Request(frame)).ok()?; + .get(&req.link) + .filter(|s| s.last_heard.elapsed() < self.online_window) + .ok_or(Refused::Offline)?; + let running = inner + .pending + .values() + .filter(|p| p.link == req.link) + .count(); + if running >= self.max_pending { + return Err(Refused::Busy); + } + let deadline = Instant::now() + req.timeout; + slot.tx + .send(Down::Request(render(&req.frame, ceil_ms(req.timeout)))) + .map_err(|_| Refused::Offline)?; let (reply_tx, reply_rx) = oneshot::channel(); - let (progress_tx, progress_rx) = mpsc::unbounded_channel(); + let (progress_tx, progress_rx) = mpsc::channel(PROGRESS_BUFFER); inner.pending.insert( - id.to_owned(), + req.id, Pending { - link, - token, + link: req.link, + user: req.user, + token: req.token, + order: self.next_call.fetch_add(1, Ordering::Relaxed), + frame: req.frame, + deadline, reply: reply_tx, progress: progress_tx, last_seq: None, }, ); - Some(Call { + Ok(Call { reply: reply_rx, progress: progress_rx, + deadline, }) } @@ -216,7 +344,8 @@ impl Hub { } /// A device's progress frame. `seq` must grow (a stale or repeated one is ignored); a frame - /// without one counts as the next. `false` when ignored. + /// without one counts as the next. Never waits: a call whose buffer is full loses it. + /// `false` when ignored or dropped. pub fn progress(&self, link: Uuid, id: &str, seq: Option, message: String) -> bool { let mut inner = self.lock(); let Some(p) = inner.pending.get_mut(id).filter(|p| p.link == link) else { @@ -227,7 +356,10 @@ impl Hub { return false; } p.last_seq = Some(seq); - p.progress.send(Progress { seq, message }).is_ok() + match p.progress.try_send(Progress { seq, message }) { + Ok(()) => true, + Err(TrySendError::Full(_) | TrySendError::Closed(_)) => false, + } } /// Stop waiting for a call: forget it and send the device a `cancel` if its stream is @@ -243,71 +375,142 @@ impl Hub { true } + /// The calls in flight, across every link. + pub fn in_flight(&self) -> usize { + self.lock().pending.len() + } + /// The link was turned off (revoked, or its device deleted): close its stream with /// `revoked` and fail its calls in flight. pub fn revoke_link(&self, link: Uuid) { let (slot, failed) = { let mut inner = self.lock(); let slot = inner.slots.remove(&link); - let ids: Vec = inner - .pending - .iter() - .filter(|(_, p)| p.link == link) - .map(|(id, _)| id.clone()) - .collect(); - let failed: Vec = ids - .iter() - .filter_map(|id| inner.pending.remove(id)) - .collect(); - (slot, failed) + (slot, take_pending(&mut inner, |p| p.link == link)) }; if let Some(slot) = slot { let _ = slot.tx.send(Down::Revoked); } - for p in failed { + for (_, p) in failed { let _ = p.reply.send(Reply::TurnedOff); } } - /// An agent token was revoked: fail its calls in flight and tell the devices to stop them. + /// An agent token was revoked: fail its calls in flight, tell the devices to stop them and + /// forget its rate limit. pub fn revoke_token(&self, token: Uuid) { let failed = { let mut inner = self.lock(); - let ids: Vec = inner - .pending - .iter() - .filter(|(_, p)| p.token == token) - .map(|(id, _)| id.clone()) - .collect(); - let mut failed = Vec::with_capacity(ids.len()); - for id in ids { - if let Some(p) = inner.pending.remove(&id) { - if let Some(slot) = inner.slots.get(&p.link) { - let _ = slot.tx.send(Down::Cancel(id)); - } - failed.push(p); + let failed = take_pending(&mut inner, |p| p.token == token); + for (id, p) in &failed { + if let Some(slot) = inner.slots.get(&p.link) { + let _ = slot.tx.send(Down::Cancel(id.clone())); } } failed }; - for p in failed { + self.lock_buckets().remove(&token); + for (_, p) in failed { let _ = p.reply.send(Reply::TokenRevoked); } } + + /// A user was disabled or deleted: close their streams with `revoked`, fail their calls in + /// flight and forget their tokens' rate limits. `true` if the hub held anything of theirs. + pub fn revoke_user(&self, user: Uuid) -> bool { + let (slots, failed) = { + let mut inner = self.lock(); + let links: Vec = inner + .slots + .iter() + .filter(|(_, s)| s.user == user) + .map(|(link, _)| *link) + .collect(); + let slots: Vec = links + .iter() + .filter_map(|link| inner.slots.remove(link)) + .collect(); + (slots, take_pending(&mut inner, |p| p.user == user)) + }; + let buckets = { + let mut buckets = self.lock_buckets(); + let before = buckets.len(); + buckets.retain(|_, b| b.user != user); + before - buckets.len() + }; + let touched = !slots.is_empty() || !failed.is_empty() || buckets > 0; + for slot in slots { + let _ = slot.tx.send(Down::Revoked); + } + for (_, p) in failed { + let _ = p.reply.send(Reply::UserRemoved); + } + touched + } + + /// Every user with an open stream, a call in flight or a live rate limit here. + pub fn users(&self) -> Vec { + let mut users: HashSet = { + let inner = self.lock(); + inner + .slots + .values() + .map(|s| s.user) + .chain(inner.pending.values().map(|p| p.user)) + .collect() + }; + users.extend(self.lock_buckets().values().map(|b| b.user)); + users.into_iter().collect() + } +} + +/// Remove and return the calls matching `pred`. +fn take_pending(inner: &mut Inner, pred: impl Fn(&Pending) -> bool) -> Vec<(String, Pending)> { + let ids: Vec = inner + .pending + .iter() + .filter(|(_, p)| pred(p)) + .map(|(id, _)| id.clone()) + .collect(); + ids.into_iter() + .filter_map(|id| inner.pending.remove(&id).map(|p| (id, p))) + .collect() } #[cfg(test)] mod tests { use super::*; + use serde_json::json; fn hub() -> Hub { - Hub::new(3, Duration::from_secs(45)) + Hub::new(3, Duration::from_secs(45), 8) } fn raw(s: &str) -> Box { RawValue::from_string(s.to_owned()).unwrap() } + fn req(link: Uuid, user: Uuid, token: Uuid, id: &str) -> Request { + let Value::Object(frame) = json!({ "id": id, "method": "tools/call" }) else { + unreachable!() + }; + Request { + link, + user, + token, + id: id.to_owned(), + frame, + timeout: Duration::from_secs(200), + } + } + + fn request_of(down: Option) -> Value { + match down { + Some(Down::Request(f)) => serde_json::from_str(&f).unwrap(), + other => panic!("not a request: {other:?}"), + } + } + #[test] fn request_ids() { let id = new_request_id(); @@ -318,11 +521,11 @@ mod tests { #[tokio::test] async fn attach_supersedes_and_detach_only_removes_its_own_stream() { let h = hub(); - let link = Uuid::now_v7(); + let (link, user) = (Uuid::now_v7(), Uuid::now_v7()); assert!(!h.is_online(link)); - let (first, mut rx1) = h.attach(link); + let (first, mut rx1) = h.attach(link, user); assert!(h.is_online(link)); - let (second, _rx2) = h.attach(link); + let (second, _rx2) = h.attach(link, user); assert!(matches!(rx1.recv().await, Some(Down::Superseded))); assert!( !h.detach(link, first), @@ -336,16 +539,21 @@ mod tests { #[tokio::test] async fn calls_go_to_the_link_and_only_it_may_answer() { let h = hub(); - let (link, other, token) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); - assert!( - h.begin(link, token, "req_a", "{}".into()).is_none(), - "offline" + let (link, other, user, token) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + assert_eq!( + h.begin(req(link, user, token, "req_a")).err(), + Some(Refused::Offline) ); - let (_, mut rx) = h.attach(link); - let mut call = h - .begin(link, token, "req_a", r#"{"id":"req_a"}"#.into()) - .unwrap(); - assert!(matches!(rx.recv().await, Some(Down::Request(f)) if f.contains("req_a"))); + let (_, mut rx) = h.attach(link, user); + let mut call = h.begin(req(link, user, token, "req_a")).unwrap(); + let sent = request_of(rx.recv().await); + assert_eq!(sent["id"], "req_a"); + assert_eq!(sent["deadline_ms"], 200_000); assert!(!h.reply(other, "req_a", Reply::Error("x".into()))); assert!(h.progress(link, "req_a", Some(2), "two".into())); assert!( @@ -359,6 +567,7 @@ mod tests { assert!(h.progress(link, "req_a", None, "next".into())); assert_eq!(call.progress.recv().await.unwrap().seq, 2); assert_eq!(call.progress.recv().await.unwrap().seq, 3); + assert_eq!(h.in_flight(), 1); assert!(h.reply(link, "req_a", Reply::Result(raw(r#"{"content":[]}"#)))); assert!(matches!(call.reply.await, Ok(Reply::Result(r)) if r.get() == r#"{"content":[]}"#)); assert!( @@ -366,20 +575,97 @@ mod tests { "finished" ); assert!(!h.cancel("req_a"), "finished"); + assert_eq!(h.in_flight(), 0); + } + + #[tokio::test] + async fn a_new_stream_gets_the_calls_still_waiting() { + let h = hub(); + let (link, user, token) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (first, mut rx1) = h.attach(link, user); + let _a = h.begin(req(link, user, token, "req_1")).unwrap(); + let _b = h.begin(req(link, user, token, "req_2")).unwrap(); + let _c = h.begin(req(link, user, token, "req_3")).unwrap(); + for id in ["req_1", "req_2", "req_3"] { + assert_eq!(request_of(rx1.recv().await)["id"], id); + } + assert!(h.cancel("req_2")); + // The stream dies without a reply; the calls stay pending. + assert!(h.detach(link, first)); + assert_eq!(h.in_flight(), 2); + let (_, mut rx2) = h.attach(link, user); + for id in ["req_1", "req_3"] { + let sent = request_of(rx2.recv().await); + assert_eq!(sent["id"], id); + assert_eq!(sent["method"], "tools/call"); + let left = sent["deadline_ms"].as_u64().unwrap(); + assert!(left > 190_000 && left <= 200_000, "{left}"); + } + assert!(h.reply(link, "req_1", Reply::Error("done".into()))); + // Superseded while still open: the new stream gets what is left. + let (_, mut rx3) = h.attach(link, user); + assert!(matches!(rx2.recv().await, Some(Down::Superseded))); + assert_eq!(request_of(rx3.recv().await)["id"], "req_3"); + assert!(rx3.try_recv().is_err()); + } + + #[tokio::test] + async fn busy_past_the_per_link_cap() { + let h = Hub::new(600, Duration::from_secs(45), 2); + let (link, other, user, token) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + let (_, _rx) = h.attach(link, user); + let (_, _rx_other) = h.attach(other, user); + let _a = h.begin(req(link, user, token, "req_a")).unwrap(); + let _b = h.begin(req(link, user, token, "req_b")).unwrap(); + assert_eq!( + h.begin(req(link, user, token, "req_c")).err(), + Some(Refused::Busy) + ); + assert!( + h.begin(req(other, user, token, "req_o")).is_ok(), + "per link" + ); + assert!(h.cancel("req_a")); + assert!(h.begin(req(link, user, token, "req_c")).is_ok()); + } + + #[tokio::test] + async fn progress_never_waits_for_a_slow_call() { + let h = hub(); + let (link, user, token) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (_, _rx) = h.attach(link, user); + let mut call = h.begin(req(link, user, token, "req_p")).unwrap(); + let mut sent = 0; + for seq in 1..=i64::try_from(PROGRESS_BUFFER + 10).unwrap() { + sent += usize::from(h.progress(link, "req_p", Some(seq), "x".into())); + } + assert_eq!(sent, PROGRESS_BUFFER, "the overflow is dropped"); + assert_eq!(call.progress.recv().await.unwrap().seq, 1); + assert!(h.progress(link, "req_p", None, "room again".into())); } #[tokio::test] async fn cancel_revoke_and_token_revoke() { let h = hub(); - let (link, token, other_token) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); - let (_, mut rx) = h.attach(link); - let _c = h.begin(link, token, "req_c", "{}".into()).unwrap(); + let (link, user, token, other_token) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + let (_, mut rx) = h.attach(link, user); + let _c = h.begin(req(link, user, token, "req_c")).unwrap(); rx.recv().await.unwrap(); assert!(h.cancel("req_c")); assert!(matches!(rx.recv().await, Some(Down::Cancel(id)) if id == "req_c")); - let t = h.begin(link, token, "req_t", "{}".into()).unwrap(); - let o = h.begin(link, other_token, "req_o", "{}".into()).unwrap(); + let t = h.begin(req(link, user, token, "req_t")).unwrap(); + let o = h.begin(req(link, user, other_token, "req_o")).unwrap(); rx.recv().await.unwrap(); rx.recv().await.unwrap(); h.revoke_token(token); @@ -392,17 +678,51 @@ mod tests { assert!(!h.is_online(link)); } + #[tokio::test] + async fn revoke_user_closes_streams_fails_calls_and_forgets_rate_limits() { + let h = hub(); + let (ada, bob) = (Uuid::now_v7(), Uuid::now_v7()); + let (ada_phone, ada_pad, bob_phone) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (ada_token, bob_token) = (Uuid::now_v7(), Uuid::now_v7()); + let (_, mut rx_phone) = h.attach(ada_phone, ada); + let (_, mut rx_pad) = h.attach(ada_pad, ada); + let (_, mut rx_bob) = h.attach(bob_phone, bob); + assert!(h.allow(ada, ada_token) && h.allow(bob, bob_token)); + let a = h.begin(req(ada_phone, ada, ada_token, "req_a")).unwrap(); + let b = h.begin(req(bob_phone, bob, bob_token, "req_b")).unwrap(); + rx_phone.recv().await.unwrap(); + rx_bob.recv().await.unwrap(); + let mut users = h.users(); + users.sort_unstable(); + let mut both = vec![ada, bob]; + both.sort_unstable(); + assert_eq!(users, both); + + assert!(h.revoke_user(ada)); + assert!(matches!(rx_phone.recv().await, Some(Down::Revoked))); + assert!(matches!(rx_pad.recv().await, Some(Down::Revoked))); + assert!(matches!(a.reply.await, Ok(Reply::UserRemoved))); + assert!(!h.is_online(ada_phone) && !h.is_online(ada_pad)); + assert_eq!(h.users(), vec![bob]); + assert!(!h.revoke_user(ada), "nothing left"); + // Bob is untouched. + assert!(h.is_online(bob_phone)); + assert!(h.reply(bob_phone, "req_b", Reply::Error("fine".into()))); + assert!(matches!(b.reply.await, Ok(Reply::Error(_)))); + } + #[test] fn online_needs_a_recent_heartbeat() { - let h = Hub::new(3, Duration::from_millis(30)); - let link = Uuid::now_v7(); - let _stream = h.attach(link); + let h = Hub::new(3, Duration::from_millis(30), 8); + let (link, user) = (Uuid::now_v7(), Uuid::now_v7()); + let _stream = h.attach(link, user); assert!(h.is_online(link)); std::thread::sleep(Duration::from_millis(40)); assert!(!h.is_online(link)); - assert!(h - .begin(link, Uuid::now_v7(), "req_x", "{}".into()) - .is_none()); + assert_eq!( + h.begin(req(link, user, Uuid::now_v7(), "req_x")).err(), + Some(Refused::Offline) + ); h.heard(link); assert!(h.is_online(link)); } @@ -410,9 +730,20 @@ mod tests { #[test] fn rate_limit_per_token() { let h = hub(); - let (a, b) = (Uuid::now_v7(), Uuid::now_v7()); - assert!(h.allow(a) && h.allow(a) && h.allow(a)); - assert!(!h.allow(a)); - assert!(h.allow(b)); + let (user, a, b) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + assert!(h.allow(user, a) && h.allow(user, a) && h.allow(user, a)); + assert!(!h.allow(user, a)); + assert!(h.allow(user, b)); + h.revoke_token(a); + assert!(h.allow(user, a), "a revoked token's bucket is forgotten"); + } + + #[test] + fn rate_limit_refills() { + let h = Hub::new(600, Duration::from_secs(45), 8); + let (user, token) = (Uuid::now_v7(), Uuid::now_v7()); + assert_eq!((0..700).filter(|_| h.allow(user, token)).count(), 600); + std::thread::sleep(Duration::from_millis(250)); + assert!(h.allow(user, token), "one request every 100 ms comes back"); } } diff --git a/crates/core/src/device_agent/mcp.rs b/crates/core/src/device_agent/mcp.rs index dfc7084..2f519d9 100644 --- a/crates/core/src/device_agent/mcp.rs +++ b/crates/core/src/device_agent/mcp.rs @@ -1,12 +1,17 @@ //! The agent side: `GET /v1/agent/status` and `POST /v1/agent/mcp` — MCP Streamable HTTP, //! stateless — authenticated by a `cat_` token instead of the instance gate. //! -//! `tools/call` is relayed to the device: offline, turned off or an unknown tool answer at +//! `tools/call` is relayed to the device: offline, turned off, busy or an unknown tool answer at //! once; otherwise a `request` frame goes down the device's stream and the reply comes back as //! the JSON-RPC result. With `Accept: text/event-stream` the answer is an SSE stream (keepalive //! comments, `notifications/progress` when the call carried a progress token, then the //! response); otherwise one JSON response. A deadline miss, or the agent going away, sends the //! device a `cancel`. +//! +//! Each relayed call waits in its own task, apart from the agent's response: the deadline ends it +//! whether or not the agent reads its answer, and a slow reader only loses progress lines +//! (up to [`AGENT_BUFFER`] wait for it, later ones are dropped) — it never holds up the device's +//! frames POST or anyone else's calls. use std::convert::Infallible; use std::time::{Duration, Instant}; @@ -18,14 +23,15 @@ use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use axum::response::sse::Event as SseEvent; use axum::response::{IntoResponse, Response}; use axum::Json; -use futures::{Stream, StreamExt as _}; +use futures::Stream; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use serde_json::{json, Value}; +use serde_json::{json, Map, Value}; use time::OffsetDateTime; +use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use super::hub::{new_request_id, Call, Progress, Reply}; +use super::hub::{self, new_request_id, Call, Progress, Refused, Reply}; use super::{plausible_agent_token, sse_response}; use crate::error::{ApiError, ApiResult}; use crate::extract::{check_content_length, read_body}; @@ -106,7 +112,7 @@ impl FromRequestParts for AgentCaller { return Err(bad_token()); }; tracing::Span::current().record("user_id", tracing::field::display(user_id)); - if !state.device_agent.allow(token_id) { + if !state.device_agent.allow(user_id, token_id) { metrics::counter!("agent_rate_limited_total").increment(1); tracing::warn!(%token_id, "agent rate limit exceeded"); return Err(ApiError::RateLimited); @@ -471,6 +477,10 @@ fn turned_off_text(device: &str) -> String { ) } +fn busy_text(device: &str) -> String { + format!("Copper on {device} is busy with other calls; try again shortly.") +} + fn unknown_tool_text(tool: &str, device: &str) -> String { format!( "Copper on {device} has no tool named \"{tool}\". Call tools/list for the tools it offers." @@ -485,11 +495,6 @@ struct RelayInfo { progress_token: Option, } -enum CallEvent { - Progress(String), - Done(String), -} - fn progress_notification(token: &Value, p: &Progress) -> String { json!({ "jsonrpc": "2.0", @@ -527,6 +532,12 @@ fn outcome( )), "token_revoked", ), + Ok(Reply::UserRemoved) => ( + tool_error(&format!( + "The account {device} is signed in to was disabled or deleted while {tool} was running; the call was stopped." + )), + "user_revoked", + ), Err(_) => ( tool_error(&format!("{tool} ended on {device} without a reply.")), "error", @@ -534,79 +545,114 @@ fn outcome( } } -/// Wait for the device: progress events while it works, then exactly one `Done` with the -/// JSON-RPC response. The guard travels inside, so dropping the stream cancels the call. -// `state` and `info` are moved into the stream; clippy cannot see through `stream!`. -#[allow(clippy::needless_pass_by_value)] -fn relay( +/// Where a relayed call's answer goes: `done` gets the JSON-RPC response; `progress` (SSE with a +/// progress token only) gets `notifications/progress` lines, best effort. +struct Answer { + progress: Option>, + done: oneshot::Sender, +} + +/// Progress lines buffered for the agent; a reader this far behind loses the newest. +const AGENT_BUFFER: usize = 32; + +/// Wait for the device in a task of its own: forward progress while it works, then send the +/// JSON-RPC response to `answer.done`. The deadline, shutdown, an offline device and the agent +/// going away (its end of `done` dropped) all end the wait, whether or not the agent is reading. +async fn relay( state: SharedState, call: Call, mut guard: CallGuard, info: RelayInfo, -) -> impl Stream { - let tuning = state.cfg.device_agent.clone(); + mut answer: Answer, +) { + let tuning = &state.cfg.device_agent; let mut shutdown = crate::shutdown::subscribe(); - async_stream::stream! { - let Call { mut reply, mut progress } = call; - let deadline = tokio::time::sleep(tuning.relay_deadline); - tokio::pin!(deadline); - let mut watch = tokio::time::interval( - WATCH_EVERY.min(tuning.offline_grace / 4).max(Duration::from_millis(10)), - ); - watch.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut offline_since: Option = None; - let mut progress_open = true; - let (result, status) = loop { - tokio::select! { - // The deadline and shutdown first (a flood of progress cannot hold them off), - // then progress before the reply, so frames uploaded just before it still go - // out. - biased; - () = &mut deadline => { - state.device_agent.cancel(&guard.request_id); - let text = format!( - "{} timed out: {} did not answer within {} s, so the call was cancelled. \ - Try again, or ask the person to check Copper on their iPhone.", - info.tool, info.device_name, tuning.relay_deadline.as_secs_f64().ceil() - ); - break (tool_error(&text), "timeout"); + let Call { + mut reply, + mut progress, + deadline, + } = call; + let deadline = tokio::time::sleep_until(deadline.into()); + tokio::pin!(deadline); + let mut watch = tokio::time::interval( + WATCH_EVERY + .min(tuning.offline_grace / 4) + .max(Duration::from_millis(10)), + ); + watch.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut offline_since: Option = None; + let mut progress_open = true; + let (result, status) = loop { + tokio::select! { + // The deadline and shutdown first (a flood of progress cannot hold them off), then + // progress before the reply, so frames uploaded just before it still go out. + biased; + () = &mut deadline => { + state.device_agent.cancel(&guard.request_id); + let text = format!( + "{} timed out: {} did not answer within {} s, so the call was cancelled. \ + Try again, or ask the person to check Copper on their iPhone.", + info.tool, info.device_name, tuning.relay_deadline.as_secs_f64().ceil() + ); + break (tool_error(&text), "timeout"); + } + () = crate::shutdown::wait(&mut shutdown) => { + state.device_agent.cancel(&guard.request_id); + let text = format!( + "Copper Cloud is restarting, so {} was stopped. Try again in a moment.", + info.tool + ); + break (tool_error(&text), "shutdown"); + } + // The agent went away: dropping the guard cancels the call on the device. + () = answer.done.closed() => return, + p = progress.recv(), if progress_open => match p { + Some(p) => { + if let (Some(out), Some(token)) = (&answer.progress, &info.progress_token) { + // Never wait for the agent: a full buffer drops this line. + let _ = out.try_send(progress_notification(token, &p)); + } } - () = crate::shutdown::wait(&mut shutdown) => { + None => progress_open = false, + }, + r = &mut reply => break outcome(r, &info), + _ = watch.tick() => { + if state.device_agent.is_online(info.link) { + offline_since = None; + } else if offline_since.get_or_insert_with(Instant::now).elapsed() + >= tuning.offline_grace + { state.device_agent.cancel(&guard.request_id); let text = format!( - "Copper Cloud is restarting, so {} was stopped. Try again in a moment.", - info.tool + "Copper closed on {} before {} finished (it is no longer open on \ + their screen). Ask the person to open Copper again, then try again.", + info.device_name, info.tool ); - break (tool_error(&text), "shutdown"); - } - p = progress.recv(), if progress_open => match p { - Some(p) => { - if let Some(token) = &info.progress_token { - yield CallEvent::Progress(progress_notification(token, &p)); - } - } - None => progress_open = false, - }, - r = &mut reply => break outcome(r, &info), - _ = watch.tick() => { - if state.device_agent.is_online(info.link) { - offline_since = None; - } else if offline_since.get_or_insert_with(Instant::now).elapsed() - >= tuning.offline_grace - { - state.device_agent.cancel(&guard.request_id); - let text = format!( - "Copper closed on {} before {} finished (it is no longer open on \ - their screen). Ask the person to open Copper again, then try again.", - info.device_name, info.tool - ); - break (tool_error(&text), "gone_offline"); - } + break (tool_error(&text), "gone_offline"); } } - }; - guard.finish(status); - yield CallEvent::Done(rpc_result(&info.id, &result)); + } + }; + guard.finish(status); + // Progress first, then the response: the agent's stream drains the lines, then reads this. + drop(answer.progress); + let _ = answer.done.send(rpc_result(&info.id, &result)); +} + +/// An SSE answer: the progress lines, then the response. +fn answer_stream( + progress: Option>, + done: oneshot::Receiver, +) -> impl Stream> { + async_stream::stream! { + if let Some(mut progress) = progress { + while let Some(line) = progress.recv().await { + yield Ok(message_event(line)); + } + } + if let Ok(body) = done.await { + yield Ok(message_event(body)); + } } } @@ -682,22 +728,35 @@ async fn tools_call( } let tuning = &state.cfg.device_agent; let request_id = new_request_id(); - let frame = json!({ - "id": request_id, - "method": "tools/call", - "params": { "name": name, "arguments": arguments }, - "caller": { "token_id": caller.token_id, "label": caller.label }, - "deadline_ms": u64::try_from(tuning.relay_deadline.as_millis()).unwrap_or(u64::MAX), - }) - .to_string(); - let Some(call) = state - .device_agent - .begin(link.id, caller.token_id, &request_id, frame) - else { - audit.log("offline"); - return answer_now(&id, &tool_error(&offline_text(device)), wants_sse); + let mut frame = Map::new(); + frame.insert("id".into(), Value::from(request_id.as_str())); + frame.insert("method".into(), Value::from("tools/call")); + frame.insert( + "params".into(), + json!({ "name": name, "arguments": arguments }), + ); + frame.insert( + "caller".into(), + json!({ "token_id": caller.token_id, "label": caller.label }), + ); + let call = match state.device_agent.begin(hub::Request { + link: link.id, + user: caller.user_id, + token: caller.token_id, + id: request_id.clone(), + frame, + timeout: tuning.relay_deadline, + }) { + Ok(call) => call, + Err(Refused::Offline) => { + audit.log("offline"); + return answer_now(&id, &tool_error(&offline_text(device)), wants_sse); + } + Err(Refused::Busy) => { + audit.log("busy"); + return answer_now(&id, &tool_error(&busy_text(device)), wants_sse); + } }; - let keepalive = tuning.agent_keepalive; let info = RelayInfo { id, tool: name.to_owned(), @@ -710,22 +769,44 @@ async fn tools_call( request_id, audit: Some(audit), }; - let events = relay(state, call, guard, info); + relayed(state, call, guard, info, wants_sse).await +} + +/// Start the call's [`relay`] task and answer the agent from it: an SSE stream, or one JSON +/// response once it is done. +async fn relayed( + state: SharedState, + call: Call, + guard: CallGuard, + info: RelayInfo, + wants_sse: bool, +) -> Response { + let keepalive = state.cfg.device_agent.agent_keepalive; + let (progress_tx, progress_rx) = if wants_sse && info.progress_token.is_some() { + let (tx, rx) = mpsc::channel(AGENT_BUFFER); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + let (done_tx, done_rx) = oneshot::channel(); + tokio::spawn(relay( + state, + call, + guard, + info, + Answer { + progress: progress_tx, + done: done_tx, + }, + )); if wants_sse { - let stream = events.map(|ev| match ev { - CallEvent::Progress(body) | CallEvent::Done(body) => { - Ok::<_, Infallible>(message_event(body)) - } - }); - return sse_response(stream, Some(keepalive)); + return sse_response(answer_stream(progress_rx, done_rx), Some(keepalive)); } - let mut events = std::pin::pin!(events); - while let Some(ev) = events.next().await { - if let CallEvent::Done(body) = ev { - return json_answer(StatusCode::OK, body); - } + // Dropping this future (the agent went away) drops `done_rx`, which ends the call. + match done_rx.await { + Ok(body) => json_answer(StatusCode::OK, body), + Err(_) => ApiError::internal("relay ended without an answer").into_response(), } - ApiError::internal("relay ended without an answer").into_response() } #[cfg(test)] diff --git a/crates/core/src/device_agent/mod.rs b/crates/core/src/device_agent/mod.rs index ae40e27..7321adf 100644 --- a/crates/core/src/device_agent/mod.rs +++ b/crates/core/src/device_agent/mod.rs @@ -17,7 +17,12 @@ //! `POST /v1/agent/mcp` (MCP Streamable HTTP, stateless; [`mcp`]). 600 requests/minute per //! token. //! * Which link is online and the calls in flight live in the in-memory [`Hub`] (one daemon per -//! instance); links and tokens persist (`0016_device_agent.sql`). +//! instance); links and tokens persist (`0016_device_agent.sql`). A stream that (re)attaches +//! gets every call still waiting on its link again (same request id); at most +//! [`crate::config::DeviceAgentTuning::max_pending_per_link`] calls are in flight per link. +//! * Disabling or deleting a user (admin API) closes their streams with `revoked` and fails +//! their calls at once; a change made outside this process (`copper-cloud admin …`) reaches +//! the hub through [`start_user_sweep`]. //! * Logs: one line per agent call (token id, device id, tool, ms, status), never arguments or //! results. Token mints/revokes and link registrations/revokes are logged (like pairing codes, //! these user actions are not in the admin audit log). @@ -27,6 +32,7 @@ pub(crate) mod mcp; use std::collections::HashSet; use std::convert::Infallible; +use std::sync::Arc; use std::time::Duration; use axum::extract::{Path, Request, State}; @@ -51,7 +57,7 @@ use crate::config::{Config, TlsMode}; use crate::error::{ApiError, ApiResult}; use crate::extract::{check_content_length, read_body, JsonBody, OptionalJsonBody}; use crate::ids; -use crate::state::SharedState; +use crate::state::{AppState, SharedState}; /// Prefix of every agent token. pub const AGENT_TOKEN_PREFIX: &str = "cat_"; @@ -488,7 +494,7 @@ async fn open_frames( let attached = if row.revoked { None } else { - let (stream, rx) = state.device_agent.attach(id); + let (stream, rx) = state.device_agent.attach(id, user.user_id); let guard = StreamGuard::new(state.clone(), id, stream); // Re-checked after attaching: a revoke that committed since `own_link` found no stream // to close, so this one must not stay attached. @@ -786,6 +792,64 @@ impl RevokedDevice { } } +// --------------------------------------------------------------------------------------------- +// Disabled and deleted users + +/// A user was disabled or deleted (after the transaction committed): close their streams with +/// `revoked`, fail their calls in flight and forget their tokens' rate limits. +pub fn revoke_user(state: &AppState, user_id: Uuid) { + if state.device_agent.revoke_user(user_id) { + tracing::info!(%user_id, "device agent: user disabled or deleted; streams closed, calls failed"); + } +} + +/// The hub's users that are missing or disabled in the database are revoked +/// ([`revoke_user`]); returns how many. +pub async fn sweep_users(state: &AppState) -> ApiResult { + let users = state.device_agent.users(); + if users.is_empty() { + return Ok(0); + } + let gone: Vec = sqlx::query_scalar( + "SELECT u FROM unnest($1::uuid[]) AS u + WHERE NOT EXISTS (SELECT 1 FROM users WHERE id = u AND NOT disabled)", + ) + .bind(&users) + .fetch_all(&state.db) + .await?; + for user in &gone { + revoke_user(state, *user); + } + Ok(gone.len()) +} + +/// Users disabled or deleted outside this process (`copper-cloud admin disable-user` / +/// `delete-user` work on the database directly) are dropped from the hub within +/// [`crate::config::DeviceAgentTuning::user_sweep`]; their agent tokens stop working at once +/// regardless. Runs once per hub, inside a tokio runtime; ends with the state or on shutdown. +pub fn start_user_sweep(state: &SharedState) { + if tokio::runtime::Handle::try_current().is_err() || !state.device_agent.claim_sweep() { + return; + } + let weak = Arc::downgrade(state); + let every = state.cfg.device_agent.user_sweep; + tokio::spawn(async move { + let mut tick = tokio::time::interval(every); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut shutdown = crate::shutdown::subscribe(); + loop { + tokio::select! { + _ = tick.tick() => {} + () = crate::shutdown::wait(&mut shutdown) => return, + } + let Some(state) = weak.upgrade() else { return }; + if let Err(err) = sweep_users(&state).await { + tracing::warn!(error = %err, "device agent user sweep failed"); + } + } + }); +} + /// Revoke the link and every agent token of device `device_id` under `users` (inside the /// transaction that deletes the device). pub async fn revoke_device( diff --git a/crates/core/src/state.rs b/crates/core/src/state.rs index a068369..0e2d604 100644 --- a/crates/core/src/state.rs +++ b/crates/core/src/state.rs @@ -31,6 +31,7 @@ impl AppState { let device_agent = crate::device_agent::Hub::new( cfg.device_agent.agent_per_minute, cfg.device_agent.online_window, + cfg.device_agent.max_pending_per_link, ); Arc::new(Self { db, diff --git a/docs/admin-api.md b/docs/admin-api.md index ebfcebb..ba35a90 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -539,7 +539,9 @@ key you gave them (directory mode) or the instance link code (open mode). Disabling revokes all of the user's sessions immediately, disconnects their live canvas peers (queued to the linked Canvas Cloud: `disconnect_user`, plus `upsert_user` with -`disabled: true` — [provisioning](canvas-link.md#user-provisioning)) and blocks sign-in (`401 account_disabled` in Copper). `display_name` must not be empty. +`disabled: true` — [provisioning](canvas-link.md#user-provisioning)) and blocks sign-in (`401 account_disabled` in Copper). Their phones' [device agent](api.md#device-agent-relay) +streams close with `revoked` and their agent calls in flight fail (0.13.1); their agent tokens +are refused until they are enabled again. `display_name` must not be empty. Unknown id (or not a UUID) → `404`. ### `DELETE /admin/api/users/{id}` @@ -549,7 +551,8 @@ pairing codes, access keys bound to their email, their legacy (pre-0.9) canvas r legacy invites addressed to their email. On the linked Canvas Cloud (queued in the same transaction, `delete_user`) their app user is deleted together with the canvases they own (Personal and shared — including other members' access to them) and their memberships; -Canvas Cloud closes those rooms. +Canvas Cloud closes those rooms. Their device agent links and tokens go with the user; their +phones' agent streams close with `revoked` and calls in flight fail at once (0.13.1). ```json { "ok": true } diff --git a/docs/api.md b/docs/api.md index 531bb13..8ad2ba7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -896,11 +896,19 @@ is one JSON object: | `cancel` | `{"id": "req_…"}` — the agent went away, timed out or was revoked; stop that call | | `ping` | `{}` every 15 s | | `superseded` | `{}`, then the stream closes; do not reconnect from this stream | -| `revoked` | `{}`, then the stream closes; the user turned the link off | +| `revoked` | `{}`, then the stream closes; the user turned the link off, or the account was disabled or deleted | Opening a revoked link's stream answers `hello` (`"revoked": true`), then `revoked`, then closes. The stream also ends on server shutdown and when the session is revoked (checked every 5 min). +**Re-delivery (0.13.1).** A call stays pending when the stream its request went down ends +without a reply. Every stream the device opens (a reconnect, or one that supersedes a stream +still open) gets, right after `hello`, every request still pending on its link, oldest first, +with the same `id`, `method`, `params` and `caller`, and `deadline_ms` = the time the call has +left. The device must treat a request id it has already seen as the same call: answer a finished +one with its stored reply, ignore one it is still running. A pending call with no stream fails +after 15 s ([below](#post-v1agentmcp--mcp-streamable-http-stateless)). + #### `POST /v1/me/links/{id}/frames` From the link's own device (`403` / `404` as above): `{"frames": [ … ]}`, body up to 32 MiB. @@ -990,13 +998,18 @@ answer `405 method_not_allowed` with `Allow: POST`: there is no server-initiated - **Offline**: at once, `{"content": [{"type": "text", "text": "Copper isn't open on . Ask the person to open Copper on their iPhone and keep it on screen (Settings › Agents › Let agents use this iPhone must be on), then try again."}], "isError": true}`. +- **Busy** (0.13.1): when the device already has 8 calls in flight (from any of its agents), at + once, `isError: true` with "Copper on is busy with other calls; try again + shortly." Nothing is sent to the device. - Otherwise the device gets a `request` frame (deadline 200 s) and the server waits for its reply. A reply's `result` is the JSON-RPC result as sent; a reply `error` becomes `isError: true` with its message. A deadline miss sends `cancel` to the device and answers - `isError: true` ("… timed out …"). If the device is offline for 15 s while the call runs - (Copper left the screen), the call is cancelled and answers `isError: true` then, instead of - waiting out the deadline. Revoking the link or the token mid-call, or a server shutdown, - answers `isError: true` too. + `isError: true` ("… timed out …"). If the device's stream drops mid-call, the request is sent + again on the stream it opens next ([re-delivery](#get-v1melinksidframes--sse-mailbox)); if + the device stays offline for 15 s while the call runs (Copper left the screen), the call is + cancelled and answers `isError: true` then, instead of waiting out the deadline. Revoking the + link or the token mid-call, disabling or deleting the account, or a server shutdown, answers + `isError: true` too. - With `text/event-stream` in the request's `Accept`, the answer is an SSE stream (`Cache-Control: no-store`, `X-Accel-Buffering: no`): a `: keepalive` comment after every 10 s of silence (proxies drop idle requests at 60 s), an `event: message` with @@ -1005,6 +1018,9 @@ answer `405 method_not_allowed` with `Allow: POST`: there is no server-initiated `event: message` with the JSON-RPC response, and the stream ends. Otherwise one `application/json` response (progress is not delivered). Every other method answers `application/json`. +- The call ends at its deadline (or any of the above) whether or not the agent reads its + answer, and an agent that reads slowly holds up nothing else: up to 32 progress notifications + wait for it, later ones are dropped until it catches up (the response is never dropped). - The agent disconnecting mid-call (closing the request or the stream) sends `cancel` to the device. - One log line per call — token id, device id, tool name, duration (`ms`), outcome (`status`) — diff --git a/docs/architecture.md b/docs/architecture.md index a4f3e1b..1c13338 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -135,12 +135,20 @@ An agent calls a person's Copper for iOS through `POST /v1/agent/mcp` (ungated, the phone holds `GET /v1/me/links/{id}/frames` open while Copper is on screen and posts replies to the same path ([api.md](api.md#device-agent-relay)). The in-process `device_agent::Hub` (on `AppState`, like `Events`) maps each link to its current frames stream (`mpsc` sender, stream id, -last heartbeat) and each request id (`req_…`) to the waiting call (`oneshot` for the reply, -`mpsc` for progress). A call is handed down only if the link is online (stream open, heard from -within 45 s); otherwise the agent is answered at once. The waiting side is a stream that owns a -guard: dropping it — the JSON handler future or the SSE body, when the agent goes away — sends -the device a `cancel`. Deadline (200 s), offline grace (15 s), revoke and shutdown end the call -with an `isError` result. Links, manifests and token hashes are in Postgres; nothing about calls +user, last heartbeat), each request id (`req_…`) to the waiting call (the `request` frame and its +deadline, a `oneshot` for the reply, a bounded `mpsc` for progress) and each agent token to its +rate limit. A call is handed down only if the link is online (stream open, heard from within +45 s) and has fewer than 8 calls in flight; otherwise the agent is answered at once. When a +stream (re)attaches, every call still waiting on its link is sent on it again (same id, the time +left as `deadline_ms`); the phone answers a request it already answered with its stored reply. +Each call waits in a task of its own, apart from the agent's response: the agent's JSON handler +or SSE body only holds the receiving ends, and dropping them (the agent went away) sends the +device a `cancel`. Deadline (200 s), offline grace (15 s), revoke and shutdown end the call with +an `isError` result whether or not the agent reads; a slow SSE reader loses progress lines (32 +buffered), never the response, and a frames POST never waits. Disabling or deleting a user +through the admin API revokes everything of theirs in the hub at once; a background sweep (every +10 s, only while the hub holds anything) does the same for users changed outside the process +(`copper-cloud admin …`). Links, manifests and token hashes are in Postgres; nothing about calls is stored, and the per-call log line never carries arguments or results. ## Encryption at rest diff --git a/docs/install.md b/docs/install.md index cc6fd56..7118450 100644 --- a/docs/install.md +++ b/docs/install.md @@ -12,7 +12,7 @@ curl -fsSL https://raw.githubusercontent.com/copper-browser/copper-cloud/main/in From a release tarball (files at the archive root): ```sh -tar xzf copper-cloud-0.13.0-linux-aarch64.tar.gz +tar xzf copper-cloud-0.13.1-linux-aarch64.tar.gz sudo ./install.sh ``` @@ -67,7 +67,7 @@ data stay. | `COPPER_CLOUD_MASTER_KEY` | generated | base64url of exactly 32 bytes | | `COPPER_CLOUD_BINARY` | — | local binary or release tarball | | `COPPER_CLOUD_BINARY_URL` | — | URL of a binary or release tarball | -| `COPPER_CLOUD_VERSION` | `latest` | GitHub release to fetch (e.g. `0.13.0`) | +| `COPPER_CLOUD_VERSION` | `latest` | GitHub release to fetch (e.g. `0.13.1`) | | `GITHUB_TOKEN` | — | for a private GitHub repository | `sudo -E` (or `sudo VAR=… sh install.sh`) is needed to pass variables through sudo. diff --git a/docs/operations.md b/docs/operations.md index 63daa4d..bb66a9c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -109,7 +109,7 @@ curl -s http://127.0.0.1:9464/metrics | `sse_subscribers` | gauge | — (open event streams) | | `auth_rate_limited_total` | counter | — | | `device_agent_streams` | gauge | — (open device-agent frames streams: phones with Copper on screen) | -| `device_agent_calls_total` | counter | `status` (`ok`, `tool_error`, `error`, `offline`, `turned_off`, `unknown_tool`, `timeout`, `gone_offline`, `revoked`, `token_revoked`, `cancelled`, `shutdown`) | +| `device_agent_calls_total` | counter | `status` (`ok`, `tool_error`, `error`, `offline`, `busy`, `turned_off`, `unknown_tool`, `timeout`, `gone_offline`, `revoked`, `token_revoked`, `user_revoked`, `cancelled`, `shutdown`) | | `agent_rate_limited_total` | counter | — (`/v1/agent/*` requests refused by the per-token limit) | | `db_pool_size`, `db_pool_idle` | gauge | — (refreshed on scrape) | | `process_uptime_seconds` | gauge | — | @@ -216,6 +216,10 @@ sudo copper-cloud admin enable-signup `--password` also works but lands in shell history; prefer `--password-stdin`. +These commands work on the database, so a running server learns of a disabled or deleted user +within 10 s: their device-agent tokens are refused at once, and their phones' agent streams close +and calls in flight fail at the next sweep (the admin API does all of it at once). + ### Synced history People delete their own synced history from Copper (`DELETE /v1/sync/history`); choosing diff --git a/docs/security.md b/docs/security.md index 1ff3ce7..1a5a5a9 100644 --- a/docs/security.md +++ b/docs/security.md @@ -334,7 +334,9 @@ signed-in tabs, so it is as powerful as the phone's browser while Copper is open and the person has turned agents on (the phone holds the stream); every call is announced on the page with Stop. Revoking a token, revoking the link or deleting the device takes effect at once: calls in flight fail and the device gets a `cancel`. A disabled account's tokens stop - working. + working; disabling or deleting a user also closes their phones' streams and fails their calls + (at once from the admin API, within 10 s from `copper-cloud admin`). At most 8 calls are in + flight per phone. - **Devices only feed their own link.** Only the link's own device (its session) may open or post to its frames; a reply is matched to the request id and the link it was sent to. - **Nothing about calls is stored.** One log line per call (token id, device id, tool, duration, diff --git a/docs/v0.13.1.md b/docs/v0.13.1.md new file mode 100644 index 0000000..41ef5fc --- /dev/null +++ b/docs/v0.13.1.md @@ -0,0 +1,21 @@ +# copper-cloud v0.13.1 + +Fixes in the device agent relay ([api.md](api.md#device-agent-relay)). No new migrations, no API +removed: an upgrade from 0.13.0 is a binary swap (`sudo ./install.sh` from the tarball), and a +rollback to 0.13.0 is too. + +- **A dropped stream no longer loses a call.** A request sent down a stream that died before the + phone answered waited out the whole 200 s deadline. Now every stream the phone opens gets the + requests still pending on its link again, right after `hello` (same id, `deadline_ms` = the + time left), and a call stays pending until a new stream re-delivers it or the phone has been + offline for 15 s. The phone answers a request it already answered with its stored reply. +- **At most 8 calls in flight per phone.** One more answers at once with `isError: true`: + "Copper on is busy with other calls; try again shortly." (status `busy`). +- **An agent that does not read its answer holds nothing up.** Each call now waits in a task of + its own: it ends at its deadline (and on cancel, revoke or shutdown) whether or not the agent + reads its SSE answer, a slow reader loses progress notifications beyond 32 buffered (never the + response), and a frames POST never waits on any agent. +- **Disabling or deleting a user stops their agents' calls.** From the admin API, their phones' + streams close with `revoked`, their calls in flight fail (status `user_revoked`) and their + rate-limit state is dropped at once; `copper-cloud admin disable-user` / `delete-user` (which + work on the database) reach a running server within 10 s.